diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs b/Framework/BlockchainUtils/BlockCache.cs similarity index 96% rename from Framework/NethereumWorkflow/BlockUtils/BlockCache.cs rename to Framework/BlockchainUtils/BlockCache.cs index c902eda0..01226440 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs +++ b/Framework/BlockchainUtils/BlockCache.cs @@ -1,4 +1,4 @@ -namespace NethereumWorkflow.BlockUtils +namespace BlockchainUtils { public class BlockCache { diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockTimeEntry.cs b/Framework/BlockchainUtils/BlockTimeEntry.cs similarity index 90% rename from Framework/NethereumWorkflow/BlockUtils/BlockTimeEntry.cs rename to Framework/BlockchainUtils/BlockTimeEntry.cs index 8846b93e..54a720b7 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockTimeEntry.cs +++ b/Framework/BlockchainUtils/BlockTimeEntry.cs @@ -1,4 +1,4 @@ -namespace NethereumWorkflow.BlockUtils +namespace BlockchainUtils { public class BlockTimeEntry { diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs b/Framework/BlockchainUtils/BlockTimeFinder.cs similarity index 97% rename from Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs rename to Framework/BlockchainUtils/BlockTimeFinder.cs index c174de99..f7b4bd50 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs +++ b/Framework/BlockchainUtils/BlockTimeFinder.cs @@ -1,6 +1,6 @@ using Logging; -namespace NethereumWorkflow.BlockUtils +namespace BlockchainUtils { public class BlockTimeFinder { @@ -87,6 +87,8 @@ namespace NethereumWorkflow.BlockUtils private bool HighestBeforeSelector(DateTime target, BlockTimeEntry entry) { + if (entry.BlockNumber == bounds.Current.BlockNumber) return true; + var next = GetBlock(entry.BlockNumber + 1); return entry.Utc <= target && diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockchainBounds.cs b/Framework/BlockchainUtils/BlockchainBounds.cs similarity index 94% rename from Framework/NethereumWorkflow/BlockUtils/BlockchainBounds.cs rename to Framework/BlockchainUtils/BlockchainBounds.cs index 92841b37..27328669 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockchainBounds.cs +++ b/Framework/BlockchainUtils/BlockchainBounds.cs @@ -1,5 +1,11 @@ -namespace NethereumWorkflow.BlockUtils +namespace BlockchainUtils { + public interface IWeb3Blocks + { + ulong GetCurrentBlockNumber(); + DateTime? GetTimestampForBlock(ulong blockNumber); + } + public class BlockchainBounds { private readonly BlockCache cache; diff --git a/Framework/BlockchainUtils/BlockchainUtils.csproj b/Framework/BlockchainUtils/BlockchainUtils.csproj new file mode 100644 index 00000000..7ad10d4e --- /dev/null +++ b/Framework/BlockchainUtils/BlockchainUtils.csproj @@ -0,0 +1,16 @@ + + + + net8.0 + enable + enable + + + + + + + + + + diff --git a/Framework/NethereumWorkflow/ConversionExtensions.cs b/Framework/BlockchainUtils/ConversionExtensions.cs similarity index 96% rename from Framework/NethereumWorkflow/ConversionExtensions.cs rename to Framework/BlockchainUtils/ConversionExtensions.cs index 22daa7c7..439eb27f 100644 --- a/Framework/NethereumWorkflow/ConversionExtensions.cs +++ b/Framework/BlockchainUtils/ConversionExtensions.cs @@ -1,7 +1,7 @@ using Nethereum.Hex.HexTypes; using System.Numerics; -namespace NethereumWorkflow +namespace BlockchainUtils { public static class ConversionExtensions { diff --git a/Framework/NethereumWorkflow/NethereumInteraction.cs b/Framework/NethereumWorkflow/NethereumInteraction.cs index 01f26bf5..4b543e9f 100644 --- a/Framework/NethereumWorkflow/NethereumInteraction.cs +++ b/Framework/NethereumWorkflow/NethereumInteraction.cs @@ -1,17 +1,16 @@ -using Logging; +using BlockchainUtils; +using Logging; using Nethereum.ABI.FunctionEncoding.Attributes; using Nethereum.Contracts; using Nethereum.RPC.Eth.DTOs; using Nethereum.Web3; -using NethereumWorkflow.BlockUtils; using Utils; namespace NethereumWorkflow { public class NethereumInteraction { - // BlockCache is a static instance: It stays alive for the duration of the application runtime. - private readonly static BlockCache blockCache = new BlockCache(); + private readonly static BlockCache blockCache = new BlockCache(); // WRONG: parallel environments! private readonly ILog log; private readonly Web3 web3; diff --git a/Framework/NethereumWorkflow/NethereumWorkflow.csproj b/Framework/NethereumWorkflow/NethereumWorkflow.csproj index ebd214b0..a0e8175d 100644 --- a/Framework/NethereumWorkflow/NethereumWorkflow.csproj +++ b/Framework/NethereumWorkflow/NethereumWorkflow.csproj @@ -12,6 +12,7 @@ + diff --git a/Framework/NethereumWorkflow/Web3Wrapper.cs b/Framework/NethereumWorkflow/Web3Wrapper.cs index 985bed9f..a68ad2e4 100644 --- a/Framework/NethereumWorkflow/Web3Wrapper.cs +++ b/Framework/NethereumWorkflow/Web3Wrapper.cs @@ -1,16 +1,11 @@ -using Logging; +using BlockchainUtils; +using Logging; using Nethereum.RPC.Eth.DTOs; using Nethereum.Web3; using Utils; namespace NethereumWorkflow { - public interface IWeb3Blocks - { - ulong GetCurrentBlockNumber(); - DateTime? GetTimestampForBlock(ulong blockNumber); - } - public class Web3Wrapper : IWeb3Blocks { private readonly Web3 web3; diff --git a/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs b/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs index 24566980..487cea43 100644 --- a/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs +++ b/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs @@ -15,6 +15,7 @@ namespace CodexPlugin void WaitForStorageContractSubmitted(); void WaitForStorageContractStarted(); void WaitForStorageContractFinished(ICodexContracts contracts); + void WaitForContractFailed(); } public class StoragePurchaseContract : IStoragePurchaseContract @@ -85,6 +86,17 @@ namespace CodexPlugin Thread.Sleep(GethContainerRecipe.BlockInterval * blocks); } + public void WaitForContractFailed() + { + if (!contractStartedUtc.HasValue) + { + WaitForStorageContractStarted(); + } + var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value; + var timeout = (Purchase.Duration - currentContractTime) + gracePeriod; + WaitForStorageContractState(timeout, "failed"); + } + public StoragePurchase GetPurchaseStatus(string purchaseId) { return codexAccess.GetPurchaseStatus(purchaseId); diff --git a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs index 2d30584c..3e025d4e 100644 --- a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs @@ -1,20 +1,112 @@ -using CodexTests; +using CodexContractsPlugin; +using CodexContractsPlugin.Marketplace; +using CodexPlugin; +using CodexTests; using NUnit.Framework; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using Utils; namespace CodexReleaseTests.MarketTests { - public class ContractFailedTest : CodexDistTest + public class ContractFailedTest : MarketplaceAutoBootstrapDistTest { + protected override int NumberOfHosts => 4; + protected override int NumberOfClients => 1; + protected override ByteSize HostAvailabilitySize => 1.GB(); + protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(1.0); + private readonly TestToken pricePerSlotPerSecond = 10.TstWei(); + [Test] [Ignore("TODO - Test in which hosts are punished for failing a contract")] public void ContractFailed() { + var hosts = StartHosts(); + var client = StartClients().Single(); + StartValidator(); + var request = CreateStorageRequest(client); + + request.WaitForStorageContractSubmitted(); + AssertContractIsOnChain(request); + + request.WaitForStorageContractStarted(); + AssertContractSlotsAreFilledByHosts(request, hosts); + + hosts.BringOffline(waitTillStopped: true); + + WaitForSlotFreedEvents(); + + request.WaitForContractFailed(); + } + + private void WaitForSlotFreedEvents() + { + Log(nameof(WaitForSlotFreedEvents)); + + var start = DateTime.UtcNow; + var timeout = CalculateContractFailTimespan(); + + while (DateTime.UtcNow < start + timeout) + { + var events = GetContracts().GetEvents(GetTestRunTimeRange()); + var slotFreed = events.GetSlotFreedEvents(); + if (slotFreed.Length == NumberOfHosts) + { + Log($"{nameof(WaitForSlotFreedEvents)} took {Time.FormatDuration(DateTime.UtcNow - start)}"); + return; + } + GetContracts().WaitUntilNextPeriod(); + } + Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}"); + } + + private TimeSpan CalculateContractFailTimespan() + { + var config = GetContracts().Deployment.Config; + var maxSlashesBeforeSlotFreed = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes); + var numProofsMissedBeforeSlash = Convert.ToInt32(config.Collateral.SlashCriterion); + + var periodDuration = GetPeriodDuration(); + var requiredNumMissedProofs = maxSlashesBeforeSlotFreed * numProofsMissedBeforeSlash; + + // Each host could miss 1 proof per period, + // so the time we should wait is period time * requiredNum of missed proofs. + // Except: the proof requirement has a concept of "downtime": + // a segment of time where proof is not required. + // We calculate the probability of downtime and extend the waiting + // timeframe by a factor, such that all hosts are highly likely to have + // failed a sufficient number of proofs. + + float n = requiredNumMissedProofs; + return periodDuration * n * GetDowntimeFactor(config); + } + + private float GetDowntimeFactor(MarketplaceConfig config) + { + byte numBlocksInDowntimeSegment = config.Proofs.Downtime; + float downtime = numBlocksInDowntimeSegment; + float window = 256.0f; + var chanceOfDowntime = downtime / window; + return 1.0f + chanceOfDowntime + chanceOfDowntime; + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) + { + var cid = client.UploadFile(GenerateTestFile(5.MB())); + return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + Duration = TimeSpan.FromHours(1.0), + Expiry = TimeSpan.FromHours(0.2), + MinRequiredNumberOfNodes = (uint)NumberOfHosts, + NodeFailureTolerance = (uint)(NumberOfHosts / 2), + PricePerSlotPerSecond = pricePerSlotPerSecond, + ProofProbability = 1, // Require a proof every period + RequiredCollateral = 1.Tst() + }); } } } diff --git a/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs index 989da95d..b007945d 100644 --- a/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs @@ -11,7 +11,7 @@ namespace CodexReleaseTests.MarketTests { private const int FilesizeMb = 10; - protected override int NumberOfHosts => 4; + protected override int NumberOfHosts => 6; protected override int NumberOfClients => 1; protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB(); protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration(); @@ -46,8 +46,11 @@ namespace CodexReleaseTests.MarketTests { Duration = GetContractDuration(), Expiry = GetContractExpiry(), - MinRequiredNumberOfNodes = (uint)NumberOfHosts, - NodeFailureTolerance = (uint)(NumberOfHosts / 2), + // TODO: this should work with NumberOfHosts, but + // an ongoing issue makes hosts sometimes not pick up slots. + // When it's resolved, we can reduce the number of hosts and slim down this test. + MinRequiredNumberOfNodes = 3, + NodeFailureTolerance = 1, PricePerSlotPerSecond = pricePerSlotPerSecond, ProofProbability = 20, RequiredCollateral = 1.Tst() @@ -66,8 +69,7 @@ namespace CodexReleaseTests.MarketTests private TimeSpan Get8TimesConfiguredPeriodDuration() { - var config = GetContracts().Deployment.Config; - return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0); + return GetPeriodDuration() * 8.0; } } } diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index e2594edb..79e10377 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -40,6 +40,12 @@ namespace CodexReleaseTests.MarketTests return handles[Get()].Contracts; } + protected TimeSpan GetPeriodDuration() + { + var config = GetContracts().Deployment.Config; + return TimeSpan.FromSeconds(((double)config.Proofs.Period)); + } + protected abstract int NumberOfHosts { get; } protected abstract int NumberOfClients { get; } protected abstract ByteSize HostAvailabilitySize { get; } @@ -101,6 +107,17 @@ namespace CodexReleaseTests.MarketTests ); } + public ICodexNode StartValidator() + { + return StartCodex(s => s + .WithName("validator") + .EnableMarketplace(GetGeth(), GetContracts(), m => m + .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst()) + .AsValidator() + ) + ); + } + public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts, string purchaseId) { var fills = GetOnChainSlotFills(possibleHosts); @@ -177,9 +194,17 @@ namespace CodexReleaseTests.MarketTests private DateTime GetContractOnChainSubmittedUtc(IStoragePurchaseContract contract) { - var events = GetContracts().GetEvents(GetTestRunTimeRange()); - var submitEvent = events.GetStorageRequests().Single(e => e.RequestId.ToHex(false) == contract.PurchaseId); - return submitEvent.Block.Utc; + return Time.Retry(() => + { + var events = GetContracts().GetEvents(GetTestRunTimeRange()); + var submitEvent = events.GetStorageRequests().SingleOrDefault(e => e.RequestId.ToHex(false) == contract.PurchaseId); + if (submitEvent == null) + { + // We're too early. + throw new TimeoutException(nameof(GetContractOnChainSubmittedUtc) + "StorageRequest not found on-chain."); + } + return submitEvent.Block.Utc; + }, nameof(GetContractOnChainSubmittedUtc)); } private TestToken GetContractCostPerSlot(TestToken pricePerSlotPerSecond, TimeSpan slotDuration) diff --git a/Tests/CodexReleaseTests/Parallelism.cs b/Tests/CodexReleaseTests/Parallelism.cs index a1b26c73..cd0a4299 100644 --- a/Tests/CodexReleaseTests/Parallelism.cs +++ b/Tests/CodexReleaseTests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -[assembly: LevelOfParallelism(1)] +[assembly: LevelOfParallelism(10)] namespace CodexReleaseTests.DataTests { } diff --git a/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs b/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs index 17f5ee68..59539a80 100644 --- a/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs +++ b/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs @@ -1,4 +1,5 @@ -using Logging; +using BlockchainUtils; +using Logging; using Moq; using NethereumWorkflow; using NethereumWorkflow.BlockUtils; diff --git a/Tools/TestNetRewarder/RewardCheck.cs b/Tools/TestNetRewarder/RewardCheck.cs index abfafda8..b7f6b73b 100644 --- a/Tools/TestNetRewarder/RewardCheck.cs +++ b/Tools/TestNetRewarder/RewardCheck.cs @@ -1,4 +1,5 @@ -using CodexContractsPlugin.ChainMonitor; +using BlockchainUtils; +using CodexContractsPlugin.ChainMonitor; using DiscordRewards; using GethPlugin; using NethereumWorkflow; diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index b95938f2..7ba4155b 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -80,6 +80,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexReleaseTests", "Tests\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExperimentalTests", "Tests\ExperimentalTests\ExperimentalTests.csproj", "{BA7369CD-7C2F-4075-8E35-98BCC19EE203}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BlockchainUtils", "Framework\BlockchainUtils\BlockchainUtils.csproj", "{4648B5AA-A0A7-44BA-89BC-2FD57370943C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -214,6 +216,10 @@ Global {BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Debug|Any CPU.Build.0 = Debug|Any CPU {BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Release|Any CPU.ActiveCfg = Release|Any CPU {BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Release|Any CPU.Build.0 = Release|Any CPU + {4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -251,6 +257,7 @@ Global {6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} {639A0603-4E80-465B-BB59-AB02F1DEEF5A} = {88C2A621-8A98-4D07-8625-7900FC8EF89E} {BA7369CD-7C2F-4075-8E35-98BCC19EE203} = {88C2A621-8A98-4D07-8625-7900FC8EF89E} + {4648B5AA-A0A7-44BA-89BC-2FD57370943C} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}