diff --git a/Framework/KubernetesWorkflow/LogHandler.cs b/Framework/KubernetesWorkflow/LogHandler.cs index af77ef75..44adcc05 100644 --- a/Framework/KubernetesWorkflow/LogHandler.cs +++ b/Framework/KubernetesWorkflow/LogHandler.cs @@ -40,6 +40,9 @@ namespace KubernetesWorkflow protected override void ProcessLine(string line) { + if (line.Contains("Received JSON-RPC response")) return; + if (line.Contains("object field not marked with serialize, skipping")) return; + LogFile.WriteRaw(line); } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 25a028dc..32dfc5d0 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -47,7 +47,7 @@ namespace CodexContractsPlugin.ChainMonitor handler = changeHandler; this.doProofPeriodMonitoring = doProofPeriodMonitoring; TotalSpan = new TimeRange(startUtc, startUtc); - PeriodMonitor = new PeriodMonitor(this.log, contracts); + PeriodMonitor = new PeriodMonitor(contracts); } public TimeRange TotalSpan { get; private set; } @@ -78,7 +78,7 @@ namespace CodexContractsPlugin.ChainMonitor throw new Exception(msg); } - log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events."); + log.Debug($"ChainState updating: {events.BlockInterval} = {events.All.Length} events."); // Run through each block and apply the events to the state in order. var span = events.BlockInterval.TimeRange.Duration; diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs index 538ae124..2fbb5ff3 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs @@ -1,18 +1,15 @@ -using Logging; -using Utils; +using Utils; namespace CodexContractsPlugin.ChainMonitor { public class PeriodMonitor { - private readonly ILog log; private readonly ICodexContracts contracts; private readonly List reports = new List(); private ulong? currentPeriod = null; - public PeriodMonitor(ILog log, ICodexContracts contracts) + public PeriodMonitor(ICodexContracts contracts) { - this.log = log; this.contracts = contracts; } @@ -39,8 +36,6 @@ namespace CodexContractsPlugin.ChainMonitor private void CreateReportForPeriod(ulong lastBlockInPeriod, ulong periodNumber, IChainStateRequest[] requests) { - log.Log("Creating report for period " + periodNumber); - ulong total = 0; ulong required = 0; var missed = new List(); @@ -87,6 +82,8 @@ namespace CodexContractsPlugin.ChainMonitor private void CalcStats() { IsEmpty = Reports.All(r => r.TotalProofsRequired == 0); + if (Reports.Length == 0) return; + PeriodLow = Reports.Min(r => r.PeriodNumber); PeriodHigh = Reports.Max(r => r.PeriodNumber); AverageNumSlots = Reports.Average(r => Convert.ToSingle(r.TotalNumSlots)); diff --git a/Tests/CodexReleaseTests/MarketTests/ChainMonitor.cs b/Tests/CodexReleaseTests/MarketTests/ChainMonitor.cs new file mode 100644 index 00000000..74f6a96a --- /dev/null +++ b/Tests/CodexReleaseTests/MarketTests/ChainMonitor.cs @@ -0,0 +1,69 @@ +using CodexContractsPlugin; +using CodexContractsPlugin.ChainMonitor; +using Logging; + +namespace CodexReleaseTests.MarketTests +{ + public class ChainMonitor + { + private readonly ILog log; + private readonly ICodexContracts contracts; + private readonly DateTime startUtc; + private readonly TimeSpan updateInterval; + private CancellationTokenSource cts = new CancellationTokenSource(); + private Task worker = Task.CompletedTask; + + public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc) + : this(log, contracts, startUtc, TimeSpan.FromSeconds(3.0)) + { + } + + public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval) + { + this.log = log; + this.contracts = contracts; + this.startUtc = startUtc; + this.updateInterval = updateInterval; + } + + public void Start() + { + cts = new CancellationTokenSource(); + worker = Task.Run(Worker); + } + + public void Stop() + { + cts.Cancel(); + worker.Wait(); + if (worker.Exception != null) throw worker.Exception; + } + + private void Worker() + { + var state = new ChainState(log, contracts, new DoNothingChainEventHandler(), startUtc, doProofPeriodMonitoring: true); + Thread.Sleep(updateInterval); + + while (!cts.IsCancellationRequested) + { + UpdateChainState(state); + + cts.Token.WaitHandle.WaitOne(updateInterval); + } + } + + private void UpdateChainState(ChainState state) + { + state.Update(); + + var reports = state.PeriodMonitor.GetAndClearReports(); + if (reports.IsEmpty) return; + + var slots = reports.Reports.Sum(r => Convert.ToInt32(r.TotalNumSlots)); + var required = reports.Reports.Sum(r => Convert.ToInt32(r.TotalProofsRequired)); + var missed = reports.Reports.Sum(r => Convert.ToInt32(r.MissedProofs)); + + log.Log($"Proof report: Slots={slots} Required={required} Missed={missed}"); + } + } +} diff --git a/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs index 8431d121..66fd6542 100644 --- a/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs @@ -4,16 +4,31 @@ using Utils; namespace CodexReleaseTests.MarketTests { - [TestFixture] + [TestFixture(6, 3, 1)] + [TestFixture(6, 4, 2)] + [TestFixture(8, 5, 2)] + [TestFixture(8, 6, 3)] + [TestFixture(12, 8, 1)] + [TestFixture(12, 8, 4)] public class ContractSuccessfulTest : MarketplaceAutoBootstrapDistTest { - private const int FilesizeMb = 10; + public ContractSuccessfulTest(int hosts, int slots, int tolerance) + { + this.hosts = hosts; + this.slots = slots; + this.tolerance = tolerance; + } - protected override int NumberOfHosts => 6; + private const int FilesizeMb = 10; + private readonly TestToken pricePerBytePerSecond = 10.TstWei(); + private readonly int hosts; + private readonly int slots; + private readonly int tolerance; + + protected override int NumberOfHosts => hosts; protected override int NumberOfClients => 1; protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB(); protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration(); - private readonly TestToken pricePerBytePerSecond = 10.TstWei(); [Test] public void ContractSuccessful() @@ -44,11 +59,8 @@ namespace CodexReleaseTests.MarketTests { Duration = GetContractDuration(), Expiry = GetContractExpiry(), - // TODO: this should work with NumberOfHosts, but - // an ongoing issue makes hosts sometimes not pick up slots. - // When it's resolved, we can reduce the number of hosts and slim down this test. - MinRequiredNumberOfNodes = 3, - NodeFailureTolerance = 1, + MinRequiredNumberOfNodes = (uint)slots, + NodeFailureTolerance = (uint)tolerance, PricePerBytePerSecond = pricePerBytePerSecond, ProofProbability = 20, CollateralPerByte = 100.TstWei() diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index e5ad35a4..b0db0702 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -5,6 +5,7 @@ using CodexPlugin; using CodexTests; using DistTestCore; using GethPlugin; +using Logging; using Nethereum.Hex.HexConvertors.Extensions; using Utils; @@ -21,11 +22,15 @@ namespace CodexReleaseTests.MarketTests base.LifecycleStart(lifecycle); var geth = StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); - handles.Add(lifecycle, new MarketplaceHandle(geth, contracts)); + var monitor = SetupChainMonitor(lifecycle.Log, contracts, lifecycle.TestStartUtc); + handles.Add(lifecycle, new MarketplaceHandle(geth, contracts, monitor)); } protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) { + var handle = handles[lifecycle]; + if (handle.ChainMonitor != null) handle.ChainMonitor.Stop(); + handles.Remove(lifecycle); base.LifecycleStop(lifecycle, result); } @@ -50,6 +55,7 @@ namespace CodexReleaseTests.MarketTests protected abstract int NumberOfClients { get; } protected abstract ByteSize HostAvailabilitySize { get; } protected abstract TimeSpan HostAvailabilityMaxDuration { get; } + protected virtual bool MonitorChainState { get; } = true; public ICodexNodeGroup StartHosts() { @@ -112,6 +118,15 @@ namespace CodexReleaseTests.MarketTests }); } + private ChainMonitor? SetupChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc) + { + if (!MonitorChainState) return null; + + var result = new ChainMonitor(log, contracts, startUtc); + result.Start(); + return result; + } + private Retry GetBalanceAssertRetry() { return new Retry("AssertBalance", @@ -330,14 +345,16 @@ namespace CodexReleaseTests.MarketTests private class MarketplaceHandle { - public MarketplaceHandle(IGethNode geth, ICodexContracts contracts) + public MarketplaceHandle(IGethNode geth, ICodexContracts contracts, ChainMonitor? chainMonitor) { Geth = geth; Contracts = contracts; + ChainMonitor = chainMonitor; } public IGethNode Geth { get; } public ICodexContracts Contracts { get; } + public ChainMonitor? ChainMonitor { get; } } } } diff --git a/Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs b/Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs index 6ad10643..d91dec64 100644 --- a/Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs @@ -1,4 +1,5 @@ using CodexClient; +using CodexPlugin; using NUnit.Framework; using Utils; @@ -16,13 +17,25 @@ namespace CodexReleaseTests.MarketTests private readonly TestToken pricePerBytePerSecond = 10.TstWei(); [Test] - [Ignore("TODO - Test where multiple successful contracts are run simultaenously")] - public void MultipleSuccessfulContracts() + [Ignore("TODO - wip")] + [Combinatorial] + public void MultipleContractGenerations( + [Values(1, 5, 10)] int numGenerations) { var hosts = StartHosts(); + + for (var i = 0; i < numGenerations; i++) + { + Log("Generation: " + i); + Generation(hosts); + } + } + + private void Generation(ICodexNodeGroup hosts) + { var clients = StartClients(); - var requests = clients.Select(c => CreateStorageRequest(c)).ToArray(); + var requests = clients.Select(CreateStorageRequest).ToArray(); All(requests, r => { @@ -32,26 +45,17 @@ namespace CodexReleaseTests.MarketTests All(requests, r => r.WaitForStorageContractStarted()); All(requests, r => AssertContractSlotsAreFilledByHosts(r, hosts)); - All(requests, r => r.WaitForStorageContractFinished()); - - // todo: removed from codexclient: - //contracts.WaitUntilNextPeriod(); - //contracts.WaitUntilNextPeriod(); - - //var blocks = 3; - //Log($"Waiting {blocks} blocks for nodes to process payouts..."); - //Thread.Sleep(GethContainerRecipe.BlockInterval * blocks); - - // todo: - //AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts); - //AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts); - //AssertHostsCollateralsAreUnchanged(hosts); } private void All(IStoragePurchaseContract[] requests, Action action) { - foreach (var r in requests) action(r); + var tasks = requests.Select(r => Task.Run(() => action(r))).ToArray(); + Task.WaitAll(tasks); + foreach(var t in tasks) + { + if (t.Exception != null) throw t.Exception; + } } private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) @@ -62,8 +66,8 @@ namespace CodexReleaseTests.MarketTests { Duration = GetContractDuration(), Expiry = GetContractExpiry(), - MinRequiredNumberOfNodes = (uint)NumberOfHosts, - NodeFailureTolerance = (uint)(NumberOfHosts / 2), + MinRequiredNumberOfNodes = (uint)NumberOfHosts / 2, + NodeFailureTolerance = (uint)(NumberOfHosts / 4), PricePerBytePerSecond = pricePerBytePerSecond, ProofProbability = 20, CollateralPerByte = 1.Tst() diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 534eb74a..104bf8d8 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -164,7 +164,7 @@ namespace DistTestCore protected TimeRange GetTestRunTimeRange() { - return new TimeRange(Get().TestStart, DateTime.UtcNow); + return new TimeRange(Get().TestStartUtc, DateTime.UtcNow); } protected virtual void Initialize(FixtureLog fixtureLog) diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 3d642d20..53d0f177 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -26,7 +26,7 @@ namespace DistTestCore WebCallTimeSet = webCallTimeSet; K8STimeSet = k8sTimeSet; TestNamespace = testNamespace; - TestStart = DateTime.UtcNow; + TestStartUtc = DateTime.UtcNow; entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(k8sTimeSet, this, testNamespace), configuration.GetFileManagerFolder(), webCallTimeSet, k8sTimeSet); metadata = entryPoint.GetPluginMetadata(); @@ -36,7 +36,7 @@ namespace DistTestCore log.WriteLogTag(); } - public DateTime TestStart { get; } + public DateTime TestStartUtc { get; } public TestLog Log { get; } public Configuration Configuration { get; } public IWebCallTimeSet WebCallTimeSet { get; } @@ -76,7 +76,7 @@ namespace DistTestCore public TimeSpan GetTestDuration() { - return DateTime.UtcNow - TestStart; + return DateTime.UtcNow - TestStartUtc; } public void OnContainersStarted(RunningPod rc)