diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 75711c8f..972a539c 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -35,7 +35,7 @@ namespace DistTestCore catch (Exception ex) { GlobalTestFailure.HasFailed = true; - Error($"Global setup cleanup failed with: {ex}"); + fixtureLog.Error($"Global setup cleanup failed with: {ex}"); throw; } @@ -67,7 +67,7 @@ namespace DistTestCore } catch (Exception ex) { - Error("Cleanup failed: " + ex.Message); + fixtureLog.Error("Cleanup failed: " + ex.Message); GlobalTestFailure.HasFailed = true; } } @@ -82,36 +82,6 @@ namespace DistTestCore return new CodexSetup(lifecycle.CodexStarter, numberOfNodes); } - private void IncludeLogsAndMetricsOnTestFailure() - { - var result = TestContext.CurrentContext.Result; - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - fixtureLog.MarkAsFailed(); - - if (IsDownloadingLogsAndMetricsEnabled()) - { - Log("Downloading all CodexNode logs and metrics because of test failure..."); - DownloadAllLogs(); - DownloadAllMetrics(); - } - else - { - Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - } - } - } - - private void Log(string msg) - { - lifecycle.Log.Log(msg); - } - - private void Error(string msg) - { - lifecycle.Log.Error(msg); - } - private void CreateNewTestLifecycle() { Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () => @@ -133,6 +103,26 @@ namespace DistTestCore }); } + private void IncludeLogsAndMetricsOnTestFailure() + { + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + fixtureLog.MarkAsFailed(); + + if (IsDownloadingLogsAndMetricsEnabled()) + { + lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure..."); + DownloadAllLogs(); + DownloadAllMetrics(); + } + else + { + lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + } + } + } + private string GetTestDuration() { var testDuration = DateTime.UtcNow - testStart; diff --git a/DistTestCore/Marketplace/CodexContractsContainerConfig.cs b/DistTestCore/Marketplace/CodexContractsContainerConfig.cs new file mode 100644 index 00000000..3b669a4b --- /dev/null +++ b/DistTestCore/Marketplace/CodexContractsContainerConfig.cs @@ -0,0 +1,16 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class CodexContractsContainerConfig + { + public CodexContractsContainerConfig(string bootstrapNodeIp, Port jsonRpcPort) + { + BootstrapNodeIp = bootstrapNodeIp; + JsonRpcPort = jsonRpcPort; + } + + public string BootstrapNodeIp { get; } + public Port JsonRpcPort { get; } + } +} diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs new file mode 100644 index 00000000..63d6f306 --- /dev/null +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -0,0 +1,23 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class CodexContractsContainerRecipe : ContainerRecipeFactory + { + public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; + + protected override string Image => DockerImage; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + var ip = config.BootstrapNodeIp; + var port = config.JsonRpcPort.Number; + + AddEnvVar("DISTTEST_NETWORK_URL", $"http://{ip}:{port}"); + AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork"); + AddEnvVar("KEEP_ALIVE", "1"); + } + } +} diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs new file mode 100644 index 00000000..a6539bb8 --- /dev/null +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -0,0 +1,68 @@ +using KubernetesWorkflow; +using Utils; + +namespace DistTestCore.Marketplace +{ + public class CodexContractsStarter + { + private const string readyString = "Done! Sleeping indefinitely..."; + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public void Start(RunningContainer bootstrapContainer) + { + var workflow = workflowCreator.CreateWorkflow(); + var startupConfig = CreateStartupConfig(bootstrapContainer); + + lifecycle.Log.Log("Deploying Codex contracts..."); + var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig); + if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure."); + var container = containers.Containers[0]; + + WaitUntil(() => + { + var logHandler = new ContractsReadyLogHandler(readyString); + workflow.DownloadContainerLog(container, logHandler); + return logHandler.Found; + }); + + lifecycle.Log.Log("Contracts deployed."); + } + + private void WaitUntil(Func predicate) + { + Time.WaitUntil(predicate, TimeSpan.FromMinutes(2), TimeSpan.FromSeconds(1)); + } + + private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer) + { + var startupConfig = new StartupConfig(); + var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag)); + startupConfig.Add(contractsConfig); + return startupConfig; + } + } + + public class ContractsReadyLogHandler : LogHandler + { + private readonly string targetString; + + public ContractsReadyLogHandler(string targetString) + { + this.targetString = targetString; + } + + public bool Found { get; private set; } + + protected override void ProcessLine(string line) + { + if (line.Contains(targetString)) Found = true; + } + } +} diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 2868752e..bc5d4aab 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -7,11 +7,13 @@ namespace DistTestCore.Marketplace private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; private readonly TestLifecycle lifecycle; private readonly WorkflowCreator workflowCreator; + private readonly CodexContractsStarter codexContractsStarter; public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { this.lifecycle = lifecycle; this.workflowCreator = workflowCreator; + codexContractsStarter = new CodexContractsStarter(lifecycle, workflowCreator); } public GethBootstrapNodeInfo StartGethBootstrapNode() @@ -22,18 +24,26 @@ namespace DistTestCore.Marketplace var workflow = workflowCreator.CreateWorkflow(); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); + var bootstrapContainer = containers.Containers[0]; - var extractor = new GethInfoExtractor(workflow, containers.Containers[0]); + var extractor = new GethInfoExtractor(workflow, bootstrapContainer); var account = extractor.ExtractAccount(); var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); var pubKey = extractor.ExtractPubKey(); - var discoveryPort = containers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); + var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); Log($"Geth bootstrap node started with account '{account}'"); + DeployCodexContracts(bootstrapContainer); + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort); } + private void DeployCodexContracts(RunningContainer bootstrapContainer) + { + codexContractsStarter.Start(bootstrapContainer); + } + private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index fad225fc..3cb7d417 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -29,7 +29,7 @@ namespace DistTestCore.Marketplace if (config.IsBootstrapNode) { AddEnvVar("IS_BOOTSTRAP", "1"); - var exposedPort = AddExposedPort(); + var exposedPort = AddExposedPort(tag: HttpPortTag); return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover"; } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 6ce366ab..611c8fec 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -1,5 +1,6 @@ using k8s; using k8s.Models; +using Utils; namespace KubernetesWorkflow { @@ -345,18 +346,7 @@ namespace KubernetesWorkflow private void WaitUntil(Func predicate) { - var start = DateTime.UtcNow; - var state = predicate(); - while (!state) - { - if (DateTime.UtcNow - start > cluster.K8sOperationTimeout()) - { - throw new TimeoutException("K8s operation timed out."); - } - - cluster.WaitForK8sServiceDelay(); - state = predicate(); - } + Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.WaitForK8sServiceDelay()); } #endregion diff --git a/Utils/Time.cs b/Utils/Time.cs index afe4f29f..0f4f71b6 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -22,5 +22,21 @@ result += $"{d.Seconds} secs"; return result; } + + public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryTime) + { + var start = DateTime.UtcNow; + var state = predicate(); + while (!state) + { + if (DateTime.UtcNow - start > timeout) + { + throw new TimeoutException("Operation timed out."); + } + + Sleep(retryTime); + state = predicate(); + } + } } }