diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index fc4d5f62..d96b25f1 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -33,10 +33,10 @@ public class Program } Console.WriteLine("Using images:" + nl + - $"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl + - $"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl + - $"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl + - $"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl); + $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl + + $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl + + $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl + + $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl); if (!args.Any(a => a == "-y")) { diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 86500a24..09170f7a 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -60,40 +60,5 @@ // file.AssertIsEqual(result); // }); // } - -// private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId) -// { -// var lastState = ""; -// var waitStart = DateTime.UtcNow; -// var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024); -// var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); - -// Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}"); -// while (lastState != "started") -// { -// CancelToken.ThrowIfCancellationRequested(); - -// var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); -// var statusJson = JsonConvert.SerializeObject(purchaseStatus); -// if (purchaseStatus != null && purchaseStatus.state != lastState) -// { -// lastState = purchaseStatus.state; -// Log.Log("Purchase status: " + statusJson); -// } - -// Thread.Sleep(2000); - -// if (lastState == "errored") -// { -// Assert.Fail("Contract start failed: " + statusJson); -// } - -// if (DateTime.UtcNow - waitStart > maxWaitTime) -// { -// Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}"); -// } -// } -// Log.Log("Contract started."); -// } // } //} diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index cc79b1d9..4039d541 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -5,12 +5,8 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { -#if Arm64 - public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a"; -#else - //public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah"; - public const string DockerImage = "codexstorage/nim-codex:sha-14c5270"; -#endif + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-3e80de3"; + public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; @@ -18,15 +14,11 @@ namespace DistTestCore.Codex public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); - public static string DockerImageOverride = string.Empty; + public override string Image { get; } - protected override string Image + public CodexContainerRecipe() { - get - { - if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride; - return DockerImage; - } + Image = GetDockerImage(); } protected override void Initialize(StartupConfig startupConfig) @@ -92,5 +84,12 @@ namespace DistTestCore.Codex if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value; return Index; } + + private string GetDockerImage() + { + var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE"); + if (!string.IsNullOrEmpty(image)) return image; + return DefaultDockerImage; + } } } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 3560d5d0..252a2f42 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -28,7 +28,7 @@ namespace DistTestCore var logConfig = configuration.GetLogConfig(); var startTime = DateTime.UtcNow; fixtureLog = new FixtureLog(logConfig, startTime); - statusLog = new StatusLog(logConfig, startTime, CodexContainerRecipe.DockerImage); + statusLog = new StatusLog(logConfig, startTime, new CodexContainerRecipe().Image); PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this); PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this); @@ -40,6 +40,12 @@ namespace DistTestCore [OneTimeSetUp] public void GlobalSetup() { + fixtureLog.Log($"Codex Distributed Tests are starting..."); + fixtureLog.Log($"Codex image: '{new CodexContainerRecipe().Image}'"); + fixtureLog.Log($"CodexContracts image: '{new CodexContractsContainerRecipe().Image}'"); + fixtureLog.Log($"Prometheus image: '{new PrometheusContainerRecipe().Image}'"); + fixtureLog.Log($"Geth image: '{new GethContainerRecipe().Image}'"); + // Previous test run may have been interrupted. // Begin by cleaning everything up. try @@ -58,9 +64,6 @@ namespace DistTestCore } fixtureLog.Log("Global setup cleanup successful"); - fixtureLog.Log($"Codex image: '{CodexContainerRecipe.DockerImage}'"); - fixtureLog.Log($"Prometheus image: '{PrometheusContainerRecipe.DockerImage}'"); - fixtureLog.Log($"Geth image: '{GethContainerRecipe.DockerImage}'"); } [SetUp] diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index 1ff0c06d..51e62d59 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -4,15 +4,15 @@ namespace DistTestCore.Marketplace { public class CodexContractsContainerRecipe : ContainerRecipeFactory { - #if Arm64 - public const string DockerImage = "emizzle/codex-contracts-deployment:latest"; - #else - public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint2"; - #endif - public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json"; - public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json"; + public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json"; + public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; - protected override string Image => DockerImage; + public override string Image { get; } + + public CodexContractsContainerRecipe() + { + Image = "codexstorage/dist-tests-codex-contracts-eth:sha-b4e4897"; + } protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index a377d69f..6dc62e92 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -4,19 +4,18 @@ namespace DistTestCore.Marketplace { public class GethContainerRecipe : ContainerRecipeFactory { - #if Arm64 - public const string DockerImage = "emizzle/geth-confenv:latest"; - #else - public const string DockerImage = "thatbenbierens/geth-confenv:onethousand"; - #endif + private const string defaultArgs = "--ipcdisable --syncmode full"; public const string HttpPortTag = "http_port"; public const string DiscoveryPortTag = "disc_port"; - private const string defaultArgs = "--ipcdisable --syncmode full"; - public const string AccountsFilename = "accounts.csv"; - protected override string Image => DockerImage; + public override string Image { get; } + + public GethContainerRecipe() + { + Image = "codexstorage/dist-tests-geth:sha-b788a2d"; + } protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 8e3fa89d..06715826 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -1,5 +1,7 @@ using DistTestCore.Codex; using DistTestCore.Helpers; +using Logging; +using Newtonsoft.Json; using NUnit.Framework; using NUnit.Framework.Constraints; using System.Numerics; @@ -10,7 +12,7 @@ namespace DistTestCore.Marketplace public interface IMarketplaceAccess { string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration); - string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); + StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); void AssertThatBalance(IResolveConstraint constraint, string message = ""); TestToken GetBalance(); } @@ -30,7 +32,7 @@ namespace DistTestCore.Marketplace this.codexAccess = codexAccess; } - public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) + public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) { var request = new CodexSalesRequestStorageRequest { @@ -57,9 +59,9 @@ namespace DistTestCore.Marketplace throw new InvalidOperationException(response); } - Log($"Storage requested successfully. PurchaseId: {response}"); + Log($"Storage requested successfully. PurchaseId: '{response}'."); - return response; + return new StoragePurchaseContract(lifecycle.Log, codexAccess, response, duration); } public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration) @@ -121,10 +123,10 @@ namespace DistTestCore.Marketplace public class MarketplaceUnavailable : IMarketplaceAccess { - public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) + public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) { Unavailable(); - return string.Empty; + return null!; } public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration) @@ -150,4 +152,92 @@ namespace DistTestCore.Marketplace throw new InvalidOperationException(); } } + + public class StoragePurchaseContract + { + private readonly BaseLog log; + private readonly CodexAccess codexAccess; + private DateTime? contractStartUtc; + + public StoragePurchaseContract(BaseLog log, CodexAccess codexAccess, string purchaseId, TimeSpan contractDuration) + { + this.log = log; + this.codexAccess = codexAccess; + PurchaseId = purchaseId; + ContractDuration = contractDuration; + } + + public string PurchaseId { get; } + public TimeSpan ContractDuration { get; } + + public void WaitForStorageContractStarted() + { + WaitForStorageContractStarted(TimeSpan.FromSeconds(30)); + } + + public void WaitForStorageContractFinished() + { + if (!contractStartUtc.HasValue) + { + WaitForStorageContractStarted(); + } + var gracePeriod = TimeSpan.FromSeconds(10); + var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value; + var timeout = (ContractDuration - currentContractTime) + gracePeriod; + WaitForStorageContractState(timeout, "finished"); + } + + /// + /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files. + /// + public void WaitForStorageContractStarted(ByteSize contractFileSize) + { + var filesizeInMb = contractFileSize.SizeInBytes / (1024 * 1024); + var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); + + WaitForStorageContractStarted(maxWaitTime); + } + + public void WaitForStorageContractStarted(TimeSpan timeout) + { + WaitForStorageContractState(timeout, "started"); + contractStartUtc = DateTime.UtcNow; + } + + private void WaitForStorageContractState(TimeSpan timeout, string desiredState) + { + var lastState = ""; + var waitStart = DateTime.UtcNow; + + log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'."); + while (lastState != desiredState) + { + var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId); + var statusJson = JsonConvert.SerializeObject(purchaseStatus); + if (purchaseStatus != null && purchaseStatus.state != lastState) + { + lastState = purchaseStatus.state; + log.Debug("Purchase status: " + statusJson); + } + + Thread.Sleep(1000); + + if (lastState == "errored") + { + Assert.Fail("Contract errored: " + statusJson); + } + + if (DateTime.UtcNow - waitStart > timeout) + { + Assert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}"); + } + } + log.Log($"Contract '{desiredState}'."); + } + + public CodexStoragePurchase GetPurchaseStatus(string purchaseId) + { + return codexAccess.GetPurchaseStatus(purchaseId); + } + } } diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs index 46587cf9..b7022225 100644 --- a/DistTestCore/Metrics/PrometheusContainerRecipe.cs +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -4,9 +4,12 @@ namespace DistTestCore.Metrics { public class PrometheusContainerRecipe : ContainerRecipeFactory { - public const string DockerImage = "thatbenbierens/prometheus-envconf:latest"; + public override string Image { get; } - protected override string Image => DockerImage; + public PrometheusContainerRecipe() + { + Image = "codexstorage/dist-tests-prometheus:sha-f97d7fd"; + } protected override void Initialize(StartupConfig startupConfig) { diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 07dc58e9..340ed2da 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -27,7 +27,7 @@ return recipe; } - protected abstract string Image { get; } + public abstract string Image { get; } protected int ContainerNumber { get; private set; } = 0; protected int Index { get; private set; } = 0; protected abstract void Initialize(StartupConfig config); diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 3007ae48..420b6fad 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,4 +1,7 @@ using DistTestCore; +using DistTestCore.Codex; +using DistTestCore.Marketplace; +using Newtonsoft.Json; using NUnit.Framework; using Utils; @@ -44,6 +47,7 @@ namespace Tests.BasicTests { var sellerInitialBalance = 234.TestTokens(); var buyerInitialBalance = 1000.TestTokens(); + var fileSize = 10.MB(); var seller = SetupCodexNode(s => s .WithStorageQuota(11.GB()) @@ -56,25 +60,27 @@ namespace Tests.BasicTests maxCollateral: 20.TestTokens(), maxDuration: TimeSpan.FromMinutes(3)); - var testFile = GenerateTestFile(10.MB()); + var testFile = GenerateTestFile(fileSize); var buyer = SetupCodexNode(s => s .WithBootstrapNode(seller) .EnableMarketplace(buyerInitialBalance)); + + buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance)); var contentId = buyer.UploadFile(testFile); - buyer.Marketplace.RequestStorage(contentId, + var purchaseContract = buyer.Marketplace.RequestStorage(contentId, pricePerSlotPerSecond: 2.TestTokens(), requiredCollateral: 10.TestTokens(), minRequiredNumberOfNodes: 1, proofProbability: 5, duration: TimeSpan.FromMinutes(1)); - Time.Sleep(TimeSpan.FromSeconds(10)); + purchaseContract.WaitForStorageContractStarted(fileSize); seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed."); - Time.Sleep(TimeSpan.FromMinutes(1)); + purchaseContract.WaitForStorageContractFinished(); seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage."); buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage."); diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 00000000..98af2465 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + dist-test-run: + build: + context: .. + dockerfile: docker/Dockerfile + environment: + - CODEXDOCKERIMAGE=codexstorage/nim-codex:sha-14c5270 + - BRANCH="feature/docker-image-testruns" + - KUBECONFIG=/opt/kubeconfig + - LOGPATH=/opt/logs + - RUNNERLOCATION=ExternalToCluster