From 43aaed225d67f003db64cc70f185fd3681dcb88c Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 3 Apr 2024 20:40:13 +0200 Subject: [PATCH 1/4] Updates codex image --- ProjectPlugins/CodexPlugin/ApiChecker.cs | 2 +- .../CodexPlugin/CodexContainerRecipe.cs | 2 +- ProjectPlugins/CodexPlugin/openapi.yaml | 53 ++++++++++++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 498aeb1..212fa19 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -9,7 +9,7 @@ namespace CodexPlugin public class ApiChecker { // - private const string OpenApiYamlHash = "63-7F-46-5E-2C-60-7A-BD-0C-EC-32-87-61-1B-79-FA-C2-EF-73-81-BA-FA-28-77-33-02-81-30-80-5D-00-97"; + private const string OpenApiYamlHash = "5A-B0-2A-AC-42-B1-A2-49-6F-9D-4E-D8-56-40-10-A6-67-F4-0D-2A-9F-E0-84-5C-EB-B8-2D-4F-D8-56-79-6C"; private const string OpenApiFilePath = "/codex/openapi.yaml"; private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 4e2cc8f..7203aa3 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexPlugin { public class CodexContainerRecipe : ContainerRecipeFactory { - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-f2f1dd5-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1524803-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; public const string MetricsPortTag = "codex_metrics_port"; diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml index 6099ed5..cf9b6b6 100644 --- a/ProjectPlugins/CodexPlugin/openapi.yaml +++ b/ProjectPlugins/CodexPlugin/openapi.yaml @@ -65,6 +65,22 @@ components: description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data. default: 10 minutes + SPR: + type: string + description: Signed Peer Record (libp2p) + + SPRRead: + type: object + properties: + spr: + $ref: "#/components/schemas/SPR" + + PeerIdRead: + type: object + properties: + id: + $ref: "#/components/schemas/PeerId" + ErasureParameters: type: object properties: @@ -106,8 +122,7 @@ components: type: string description: Path of the data repository where all nodes data are stored spr: - type: string - description: Signed Peer Record to advertise DHT connection information + $ref: "#/components/schemas/SPR" SalesAvailability: type: object @@ -685,6 +700,40 @@ paths: "503": description: Purchasing is unavailable + "/node/spr": + get: + summary: "Get Node's SPR" + operationId: getSPR + tags: [ Node ] + responses: + "200": + description: Node's SPR + content: + plain/text: + schema: + $ref: "#/components/schemas/SPR" + application/json: + schema: + $ref: "#/components/schemas/SPRRead" + "503": + description: Node SPR not ready, try again later + + "/node/peerid": + get: + summary: "Get Node's PeerID" + operationId: getPeerId + tags: [ Node ] + responses: + "200": + description: Node's Peer ID + content: + plain/text: + schema: + $ref: "#/components/schemas/PeerId" + application/json: + schema: + $ref: "#/components/schemas/PeerIdRead" + "/debug/chronicles/loglevel": post: summary: "Set log level at run time" From 69666d3fee389332022d2821dbd943253d647f31 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 9 Apr 2024 09:30:45 +0200 Subject: [PATCH 2/4] Setting up future-containers --- Framework/KubernetesWorkflow/K8sController.cs | 6 +++++- .../KubernetesWorkflow/StartupWorkflow.cs | 21 ++++++++++++++----- .../Types/FutureContainers.cs | 20 ++++++++++++++++++ ProjectPlugins/CodexPlugin/CodexStarter.cs | 1 + 4 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 Framework/KubernetesWorkflow/Types/FutureContainers.cs diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index b14267d..80b15d1 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -43,6 +43,11 @@ namespace KubernetesWorkflow return new StartResult(cluster, containerRecipes, deployment, internalService, externalService); } + public void WaitUntilOnline(RunningContainer container) + { + WaitUntilDeploymentOnline(container.Recipe.Name); + } + public PodInfo GetPodInfo(RunningDeployment deployment) { var pod = GetPodForDeployment(deployment); @@ -372,7 +377,6 @@ namespace KubernetesWorkflow }; client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace)); - WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); var name = deploymentSpec.Metadata.Name; return new RunningDeployment(name, podLabel); diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index 4d1a2f6..b8f7d77 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -9,8 +9,8 @@ namespace KubernetesWorkflow public interface IStartupWorkflow { IKnownLocations GetAvailableLocations(); - RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); - RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); + FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); + FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); PodInfo GetPodInfo(RunningContainer container); PodInfo GetPodInfo(RunningContainers containers); CrashWatcher CreateCrashWatcher(RunningContainer container); @@ -45,12 +45,12 @@ namespace KubernetesWorkflow return locationProvider.GetAvailableLocations(); } - public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig); } - public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { return K8s(controller => { @@ -67,7 +67,18 @@ namespace KubernetesWorkflow { componentFactory.Update(controller); } - return rc; + return new FutureContainers(rc, this); + }); + } + + public void WaitUntilOnline(RunningContainers rc) + { + K8s(controller => + { + foreach (var c in rc.Containers) + { + controller.WaitUntilOnline(c); + } }); } diff --git a/Framework/KubernetesWorkflow/Types/FutureContainers.cs b/Framework/KubernetesWorkflow/Types/FutureContainers.cs new file mode 100644 index 0000000..262eac4 --- /dev/null +++ b/Framework/KubernetesWorkflow/Types/FutureContainers.cs @@ -0,0 +1,20 @@ +namespace KubernetesWorkflow.Types +{ + public class FutureContainers + { + private readonly RunningContainers runningContainers; + private readonly StartupWorkflow workflow; + + public FutureContainers(RunningContainers runningContainers, StartupWorkflow workflow) + { + this.runningContainers = runningContainers; + this.workflow = workflow; + } + + public RunningContainers WaitForOnline() + { + workflow.WaitUntilOnline(runningContainers); + return runningContainers; + } + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index 05db6bd..2a8f1bb 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -93,6 +93,7 @@ namespace CodexPlugin var workflow = pluginTools.CreateWorkflow(); result.Add(workflow.Start(1, location, recipe, startupConfig)); } + return result.ToArray(); } From 0182ce134fe22de33da22ed1c2d40a3a4e9a29fe Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 9 Apr 2024 10:23:07 +0200 Subject: [PATCH 3/4] Applies future container to existing plugins --- .../CodexContractsStarter.cs | 2 +- .../CodexDiscordBotPlugin.cs | 4 +-- ProjectPlugins/CodexPlugin/CodexStarter.cs | 8 ++++-- .../DeployAndRunPlugin/DeployAndRunPlugin.cs | 2 +- ProjectPlugins/GethPlugin/GethStarter.cs | 2 +- .../MetricsPlugin/PrometheusStarter.cs | 2 +- .../CodexTests/BasicTests/MarketplaceTests.cs | 28 +++++++++---------- 7 files changed, 25 insertions(+), 23 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index a6f6641..0137a17 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -24,7 +24,7 @@ namespace CodexContractsPlugin var startupConfig = CreateStartupConfig(gethNode); startupConfig.NameOverride = "codex-contracts"; - var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig); + var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline(); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index cad50c1..fd082cf 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -46,14 +46,14 @@ namespace CodexDiscordBotPlugin var startupConfig = new StartupConfig(); startupConfig.NameOverride = config.Name; startupConfig.Add(config); - return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig); + return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline(); } private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) { var startupConfig = new StartupConfig(); startupConfig.Add(config); - return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig); + return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline(); } } } diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index 2a8f1bb..6fcc4bd 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -87,14 +87,16 @@ namespace CodexPlugin private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location) { - var result = new List(); + var futureContainers = new List(); for (var i = 0; i < numberOfNodes; i++) { var workflow = pluginTools.CreateWorkflow(); - result.Add(workflow.Start(1, location, recipe, startupConfig)); + futureContainers.Add(workflow.Start(1, location, recipe, startupConfig)); } - return result.ToArray(); + return futureContainers + .Select(f => f.WaitForOnline()) + .ToArray(); } private PodInfo GetPodInfo(RunningContainers rc) diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs index 070b9fe..c290c0d 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs @@ -30,7 +30,7 @@ namespace DeployAndRunPlugin startupConfig.Add(config); var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd"); - var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig); + var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline(); return containers.Containers.Single(); } } diff --git a/ProjectPlugins/GethPlugin/GethStarter.cs b/ProjectPlugins/GethPlugin/GethStarter.cs index c5ad801..edfba4e 100644 --- a/ProjectPlugins/GethPlugin/GethStarter.cs +++ b/ProjectPlugins/GethPlugin/GethStarter.cs @@ -21,7 +21,7 @@ namespace GethPlugin startupConfig.NameOverride = gethStartupConfig.NameOverride; var workflow = tools.CreateWorkflow(); - var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig); + var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline(); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs index 53f784e..f63f97d 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs @@ -25,7 +25,7 @@ namespace MetricsPlugin startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets))); var workflow = tools.CreateWorkflow(); - var runningContainers = workflow.Start(1, recipe, startupConfig); + var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline(); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); Log("Metrics server started."); diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index 7474696..630ff80 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -19,22 +19,22 @@ namespace CodexTests.BasicTests var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); + + var numberOfHosts = 5; + var hosts = AddCodex(numberOfHosts, s => s + .WithName("Host") + .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) + { + ContractClock = CodexLogLevel.Trace, + }) + .WithStorageQuota(11.GB()) + .EnableMarketplace(geth, contracts, m => m + .WithInitial(10.Eth(), hostInitialBalance) + .AsStorageNode() + .AsValidator())); - var numberOfHosts = 3; - for (var i = 0; i < numberOfHosts; i++) + foreach (var host in hosts) { - var host = AddCodex(s => s - .WithName("Host") - .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) - { - ContractClock = CodexLogLevel.Trace, - }) - .WithStorageQuota(11.GB()) - .EnableMarketplace(geth, contracts, m => m - .WithInitial(10.Eth(), hostInitialBalance) - .AsStorageNode() - .AsValidator())); - AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance)); var availability = new StorageAvailability( From b7ab2f994ed67288e3f48baadf827ffcbd8010cc Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 9 Apr 2024 10:57:02 +0200 Subject: [PATCH 4/4] Fixes market test --- Tests/CodexTests/BasicTests/MarketplaceTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index 630ff80..d5befe0 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -33,9 +33,10 @@ namespace CodexTests.BasicTests .AsStorageNode() .AsValidator())); + var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens(); foreach (var host in hosts) { - AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance)); + AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance)); var availability = new StorageAvailability( totalSpace: 10.GB(),