From 703f410161b5f69a232e5d696345af339fd843fe Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:38:19 +0200 Subject: [PATCH 01/15] Fix reports workflows scheduler (#81) --- .github/workflows/report-HoldMyBeerTest.yaml | 2 +- .github/workflows/report-PeersTest.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/report-HoldMyBeerTest.yaml b/.github/workflows/report-HoldMyBeerTest.yaml index 7bd7890..c4ae260 100644 --- a/.github/workflows/report-HoldMyBeerTest.yaml +++ b/.github/workflows/report-HoldMyBeerTest.yaml @@ -3,7 +3,7 @@ name: Report - HoldMyBeerTest on: schedule: - - cron: '30 */49 * * *' + - cron: '30 * */2 * *' workflow_dispatch: jobs: diff --git a/.github/workflows/report-PeersTest.yaml b/.github/workflows/report-PeersTest.yaml index 0f1166b..b063a02 100644 --- a/.github/workflows/report-PeersTest.yaml +++ b/.github/workflows/report-PeersTest.yaml @@ -3,7 +3,7 @@ name: Report - PeersTest on: schedule: - - cron: '30 */49 * * *' + - cron: '30 * */2 * *' workflow_dispatch: jobs: From 06ff7c97608eae9747df74a99a3bb88b2039f764 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Wed, 22 Nov 2023 21:04:59 +0200 Subject: [PATCH 02/15] Add CODEXDOCKERIMAGE input to C-Tests workflow (#82) --- .github/workflows/continuous-tests.yaml | 11 +++++++++++ docker/continuous-tests-job.yaml | 2 ++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/continuous-tests.yaml b/.github/workflows/continuous-tests.yaml index 6eab327..b4b5a1f 100644 --- a/.github/workflows/continuous-tests.yaml +++ b/.github/workflows/continuous-tests.yaml @@ -12,6 +12,10 @@ on: description: Branch with tests (master) required: false type: string + codexdockerimage: + description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests) + required: false + type: string nameprefix: description: Resources prefix (c-tests) required: false @@ -38,6 +42,10 @@ on: description: Branch with tests (master) required: false type: string + codexdockerimage: + description: Codex Docker tag (codexstorage/nim-codex:latest-dist-tests) + required: false + type: string nameprefix: description: Resources prefix (c-tests) required: false @@ -59,6 +67,7 @@ on: env: SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }} BRANCH: ${{ github.ref_name }} + CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests NAMEPREFIX: c-tests NAMESPACE: default TESTS_TARGET_DURATION: 2d @@ -84,6 +93,7 @@ jobs: echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV" [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV" [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}-${RUNID}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}-${RUNID}" >>"$GITHUB_ENV" [[ -n "${{ inputs.nameprefix }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ inputs.nameprefix }}-${RUNID}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.NAMEPREFIX }}-${RUNID}" >>"$GITHUB_ENV" [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV" @@ -115,6 +125,7 @@ jobs: echo "Runner namespace: ${{ env.NAMESPACE }}" echo "----" echo "Tests runid: ${{ env.RUNID }}" + echo "Tests codexdockerimage: ${{ env.CODEXDOCKERIMAGE }}" echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}" echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}" echo "Tests filter: ${{ env.TESTS_FILTER }}" diff --git a/docker/continuous-tests-job.yaml b/docker/continuous-tests-job.yaml index f439981..97eea31 100644 --- a/docker/continuous-tests-job.yaml +++ b/docker/continuous-tests-job.yaml @@ -39,6 +39,8 @@ spec: value: "${SOURCE}" - name: RUNID value: "${RUNID}" + - name: CODEXDOCKERIMAGE + value: "${CODEXDOCKERIMAGE}" - name: TESTID value: "${TESTID}" - name: DEPLOYMENT_NAMESPACE From 6bd7098cfea719ade939896836f560fce9620649 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Nov 2023 13:32:58 +0100 Subject: [PATCH 03/15] Fixes download endpoint. --- ProjectPlugins/CodexPlugin/CodexAccess.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 9fa1683..d1b276b 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -68,7 +68,7 @@ namespace CodexPlugin public Stream DownloadFile(string contentId) { - return Http().HttpGetStream("data/" + contentId); + return Http().HttpGetStream("data/" + contentId + "/network"); } public CodexLocalDataResponse[] LocalFiles() From 1a86d3459d13b1e8a0d60392d5082fa880ffcfbe Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Nov 2023 13:43:19 +0100 Subject: [PATCH 04/15] Fixes local files endpoint --- ProjectPlugins/CodexPlugin/CodexAccess.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index d1b276b..a305d76 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -73,7 +73,7 @@ namespace CodexPlugin public CodexLocalDataResponse[] LocalFiles() { - return Http().HttpGetJson("local"); + return Http().HttpGetJson("data"); } public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request) From b61f5d835c4ca3b48a22a9ce32aea9914a7e482b Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Nov 2023 14:10:59 +0100 Subject: [PATCH 05/15] Fixes request-storage endpoint --- .../CodexPlugin/MarketplaceAccess.cs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs b/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs index 9edea6e..d910471 100644 --- a/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs +++ b/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs @@ -10,6 +10,7 @@ namespace CodexPlugin { string MakeStorageAvailable(ByteSize size, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration); StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); + StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry); } public class MarketplaceAccess : IMarketplaceAccess @@ -25,13 +26,20 @@ namespace CodexPlugin public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) { + return RequestStorage(contentId, pricePerSlotPerSecond, requiredCollateral, minRequiredNumberOfNodes, proofProbability, duration, TimeSpan.FromMinutes(5)); + } + + public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry) + { + var expireUtc = DateTimeOffset.UtcNow.ToUnixTimeSeconds() + expiry.TotalSeconds; + var request = new CodexSalesRequestStorageRequest { duration = ToDecInt(duration.TotalSeconds), proofProbability = ToDecInt(proofProbability), reward = ToDecInt(pricePerSlotPerSecond), collateral = ToDecInt(requiredCollateral), - expiry = null, + expiry = ToDecInt(expireUtc), nodes = minRequiredNumberOfNodes, tolerance = null, }; @@ -41,11 +49,14 @@ namespace CodexPlugin $"requiredCollateral: {requiredCollateral}, " + $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " + $"proofProbability: {proofProbability}, " + + $"expiry: {Time.FormatDuration(expiry)}, " + $"duration: {Time.FormatDuration(duration)})"); var response = codexAccess.RequestStorage(request, contentId.Id); - if (response == "Purchasing not available") + if (response == "Purchasing not available" || + response == "Expiry required" || + response == "Expiry needs to be in future") { throw new InvalidOperationException(response); } @@ -104,6 +115,12 @@ namespace CodexPlugin return null!; } + public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry) + { + Unavailable(); + throw new NotImplementedException(); + } + public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration) { Unavailable(); From ff52e8e84147871b5d99388c1e0c48a9764d2d78 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Nov 2023 14:15:37 +0100 Subject: [PATCH 06/15] Defines public IP fetching service --- Framework/KubernetesWorkflow/K8sController.cs | 3 ++- Framework/KubernetesWorkflow/PublicIpService.cs | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 Framework/KubernetesWorkflow/PublicIpService.cs diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 2251336..9cde3fb 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -664,7 +664,8 @@ namespace KubernetesWorkflow private V1Pod GetPodForDeployment(RunningDeployment deployment) { return Time.Retry(() => GetPodForDeplomentInternal(deployment), - maxRetries: 2, + // We will wait up to 1 minute, k8s might be moving pods around. + maxRetries: 6, retryTime: TimeSpan.FromSeconds(10), description: "Find pod by label for deployment."); } diff --git a/Framework/KubernetesWorkflow/PublicIpService.cs b/Framework/KubernetesWorkflow/PublicIpService.cs new file mode 100644 index 0000000..6e7f414 --- /dev/null +++ b/Framework/KubernetesWorkflow/PublicIpService.cs @@ -0,0 +1,7 @@ +namespace KubernetesWorkflow +{ + public static class PublicIpService + { + public static string Address { get; } = "ip.codex.storage"; + } +} From 3761e236a330453000b69fbd65e7db70244d0f16 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Nov 2023 14:50:54 +0100 Subject: [PATCH 07/15] Sets node critical priority for codex and geth nodes --- Framework/KubernetesWorkflow/K8sController.cs | 10 ++++++++++ Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs | 4 +++- .../Recipe/ContainerRecipeFactory.cs | 9 ++++++++- ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs | 1 + ProjectPlugins/GethPlugin/GethContainerRecipe.cs | 1 + 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 9cde3fb..e8c28ef 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -335,6 +335,7 @@ namespace KubernetesWorkflow }, Spec = new V1PodSpec { + PriorityClassName = GetPriorityClassName(containerRecipes), Affinity = CreatePodAffinity(containerRecipes), NodeSelector = CreateNodeSelector(location), Containers = CreateDeploymentContainers(containerRecipes), @@ -410,6 +411,15 @@ namespace KubernetesWorkflow return l.NodeLabel; } + private string GetPriorityClassName(ContainerRecipe[] containerRecipes) + { + if (containerRecipes.Any(c => c.SetCriticalPriority)) + { + return "system-node-critical"; + } + return null!; + } + private IDictionary GetSelector(ContainerRecipe[] containerRecipes) { return containerRecipes.First().PodLabels.GetLabels(); diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs index 1865dc7..fb7c8a4 100644 --- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs +++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs @@ -2,13 +2,14 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals) + public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals) { Number = number; NameOverride = nameOverride; Image = image; Resources = resources; SchedulingAffinity = schedulingAffinity; + SetCriticalPriority = setCriticalPriority; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; @@ -34,6 +35,7 @@ public string? NameOverride { get; } public ContainerResources Resources { get; } public SchedulingAffinity SchedulingAffinity { get; } + public bool SetCriticalPriority { get; } public string Image { get; } public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs index 3b2b941..c2ba8d8 100644 --- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs @@ -14,6 +14,7 @@ namespace KubernetesWorkflow.Recipe private RecipeComponentFactory factory = null!; private ContainerResources resources = new ContainerResources(); private SchedulingAffinity schedulingAffinity = new SchedulingAffinity(); + private bool setCriticalPriority; public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config) { @@ -23,7 +24,7 @@ namespace KubernetesWorkflow.Recipe Initialize(config); - var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, + var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, setCriticalPriority, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), @@ -42,6 +43,7 @@ namespace KubernetesWorkflow.Recipe this.factory = null!; resources = new ContainerResources(); schedulingAffinity = new SchedulingAffinity(); + setCriticalPriority = false; return recipe; } @@ -128,6 +130,11 @@ namespace KubernetesWorkflow.Recipe schedulingAffinity = new SchedulingAffinity(notIn); } + protected void SetSystemCriticalPriority() + { + setCriticalPriority = true; + } + // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is // called for if they have resource limits defined. //protected void SetResourceLimits(int milliCPUs, ByteSize memory) diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 744ae37..5298279 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -30,6 +30,7 @@ namespace CodexPlugin //SetResourceLimits(milliCPUs: 4000, memory: 12.GB()); SetSchedulingAffinity(notIn: "tests-runners"); + SetSystemCriticalPriority(); var config = startupConfig.Get(); diff --git a/ProjectPlugins/GethPlugin/GethContainerRecipe.cs b/ProjectPlugins/GethPlugin/GethContainerRecipe.cs index 35b994c..9db6876 100644 --- a/ProjectPlugins/GethPlugin/GethContainerRecipe.cs +++ b/ProjectPlugins/GethPlugin/GethContainerRecipe.cs @@ -25,6 +25,7 @@ namespace GethPlugin var args = CreateArgs(config); SetSchedulingAffinity(notIn: "tests-runners"); + SetSystemCriticalPriority(); AddEnvVar("GETH_ARGS", args); } From 00207de04d1bb1f9b5badc70a2c0c38387cdcb82 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Fri, 24 Nov 2023 20:29:50 +0200 Subject: [PATCH 08/15] Auto cleanup old Continuous tests Jobs (#83) --- docker/continuous-tests-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/continuous-tests-job.yaml b/docker/continuous-tests-job.yaml index 97eea31..0f5f7aa 100644 --- a/docker/continuous-tests-job.yaml +++ b/docker/continuous-tests-job.yaml @@ -7,6 +7,7 @@ metadata: name: ${NAMEPREFIX} runid: ${RUNID} spec: + ttlSecondsAfterFinished: 86400 backoffLimit: 0 template: metadata: From b57a7271958edec81e139b8c9dc1851d308359a9 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Sun, 26 Nov 2023 21:20:04 +0200 Subject: [PATCH 09/15] Adjust C-Tests for autoruns (#84) --- .github/workflows/continuous-tests.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-tests.yaml b/.github/workflows/continuous-tests.yaml index b4b5a1f..74334e8 100644 --- a/.github/workflows/continuous-tests.yaml +++ b/.github/workflows/continuous-tests.yaml @@ -30,6 +30,7 @@ on: type: string tests_cleanup: description: Runner tests cleanup + required: false type: boolean default: true workflow_call: @@ -43,7 +44,7 @@ on: required: false type: string codexdockerimage: - description: Codex Docker tag (codexstorage/nim-codex:latest-dist-tests) + description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests) required: false type: string nameprefix: @@ -62,6 +63,12 @@ on: description: Runner tests cleanup required: false type: boolean + default: true + workflow_source: + description: Workflow source + required: false + type: string + default: '' env: @@ -80,11 +87,13 @@ env: jobs: run_tests: - name: Run Tests + name: Run Continuous Tests ${{ inputs.tests_filter }} runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 + with: + repository: ${{ inputs.workflow_source }} - name: Variables run: | @@ -94,8 +103,9 @@ jobs: [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV" [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV" [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV" - [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}-${RUNID}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}-${RUNID}" >>"$GITHUB_ENV" - [[ -n "${{ inputs.nameprefix }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ inputs.nameprefix }}-${RUNID}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.NAMEPREFIX }}-${RUNID}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`" + echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV" + echo "DEPLOYMENT_NAMESPACE=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV" [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV" [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV" [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV" From 4c46a708abb8059d549e1611aa017d062c8ecb81 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Mon, 27 Nov 2023 19:06:04 +0200 Subject: [PATCH 10/15] Change affinity label (#85) https://github.com/codex-storage/infra-codex/issues/100 --- Framework/KubernetesWorkflow/K8sController.cs | 2 +- .../CodexContractsPlugin/CodexContractsContainerRecipe.cs | 2 +- .../CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs | 2 +- ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs | 2 +- ProjectPlugins/GethPlugin/GethContainerRecipe.cs | 2 +- ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index e8c28ef..b994b75 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -393,7 +393,7 @@ namespace KubernetesWorkflow { new V1NodeSelectorRequirement { - Key = "workload-type", + Key = "allow-tests-pods", OperatorProperty = "NotIn", Values = notIns } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index 3e5faa1..88faab2 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -21,7 +21,7 @@ namespace CodexContractsPlugin var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag); - SetSchedulingAffinity(notIn: "tests-runners"); + SetSchedulingAffinity(notIn: "false"); AddEnvVar("DISTTEST_NETWORK_URL", address.ToString()); AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork"); diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs index e1596b4..3ca6669 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs @@ -13,7 +13,7 @@ namespace CodexDiscordBotPlugin { var config = startupConfig.Get(); - SetSchedulingAffinity(notIn: "tests-runners"); + SetSchedulingAffinity(notIn: "false"); AddEnvVar("TOKEN", config.Token); AddEnvVar("SERVERNAME", config.ServerName); diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 5298279..e6315d2 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -29,7 +29,7 @@ namespace CodexPlugin SetResourcesRequest(milliCPUs: 100, memory: 100.MB()); //SetResourceLimits(milliCPUs: 4000, memory: 12.GB()); - SetSchedulingAffinity(notIn: "tests-runners"); + SetSchedulingAffinity(notIn: "false"); SetSystemCriticalPriority(); var config = startupConfig.Get(); diff --git a/ProjectPlugins/GethPlugin/GethContainerRecipe.cs b/ProjectPlugins/GethPlugin/GethContainerRecipe.cs index 9db6876..75acd69 100644 --- a/ProjectPlugins/GethPlugin/GethContainerRecipe.cs +++ b/ProjectPlugins/GethPlugin/GethContainerRecipe.cs @@ -24,7 +24,7 @@ namespace GethPlugin var args = CreateArgs(config); - SetSchedulingAffinity(notIn: "tests-runners"); + SetSchedulingAffinity(notIn: "false"); SetSystemCriticalPriority(); AddEnvVar("GETH_ARGS", args); diff --git a/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs b/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs index e132a8f..23ebf3a 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs @@ -14,7 +14,7 @@ namespace MetricsPlugin { var config = startupConfig.Get(); - SetSchedulingAffinity(notIn: "tests-runners"); + SetSchedulingAffinity(notIn: "false"); AddExposedPortAndVar("PROM_PORT", PortTag); AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); From 12ea002660249fdefd73e253f4952f4d5e2170c9 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Mon, 27 Nov 2023 19:22:28 +0200 Subject: [PATCH 11/15] Update Runners Jobs nodeSelector (#86) https://github.com/codex-storage/infra-codex/issues/96 --- docker/continuous-tests-job.yaml | 2 +- docker/dist-tests-job.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/continuous-tests-job.yaml b/docker/continuous-tests-job.yaml index 0f5f7aa..ecf8ea8 100644 --- a/docker/continuous-tests-job.yaml +++ b/docker/continuous-tests-job.yaml @@ -19,7 +19,7 @@ spec: spec: priorityClassName: system-node-critical nodeSelector: - workload-type: "tests-runners" + workload-type: "tests-runners-ci" containers: - name: ${NAMEPREFIX}-runner image: codexstorage/cs-codex-dist-tests:latest diff --git a/docker/dist-tests-job.yaml b/docker/dist-tests-job.yaml index 2eaed22..b6fdb72 100644 --- a/docker/dist-tests-job.yaml +++ b/docker/dist-tests-job.yaml @@ -18,7 +18,7 @@ spec: spec: priorityClassName: system-node-critical nodeSelector: - workload-type: "tests-runners" + workload-type: "tests-runners-ci" containers: - name: ${NAMEPREFIX}-runner image: codexstorage/cs-codex-dist-tests:latest From ad05dc07f017220d9a63875f239387cd6a70b3a6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 5 Dec 2023 08:31:28 +0100 Subject: [PATCH 12/15] wip report for november --- .../CodexTestNetReport-November2023.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md diff --git a/Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md b/Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md new file mode 100644 index 0000000..ebe4b82 --- /dev/null +++ b/Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md @@ -0,0 +1,41 @@ +# Codex Continuous Test-net Report +Date: 05-12-2023 + +Report for: 11-2023 + + +## Continuous test-net Status +Continuous test runs (which can take many hours or days) can easily be started by team member from the github actions UI. Results are collected and displayed in Grafana. For the time being, we're suspending the effort to have a network of Codex nodes "always online" and continuously being tested, until overal reliability improves. + +## Deployment Configuration +Continous Test-net is deployed to the kubernetes cluster with the following configuration: + +5x Codex Nodes: +- Log-level: Trace +- Storage quota: 20480 MB +- Storage sell: 1024 MB +- Min price: 1024 +- Max collateral: 1024 +- Max duration: 3600000 seconds +- Block-TTL*: 99999999 seconds +- Block-MI*: 99999999 seconds +- Block-MN*: 100 blocks +3 of these 5 nodes have: +- Validator: true + +## Test Overview +| Changes | Test | Description | Status | Results | +|---------|------------------|--------------------------------|---------|----------------------| +| todo | Two-client test | See report for July 2023. | Faulted | Test reliably fails. | +| todo | Two-client test* | See report for September 2023. | Faulted | Test reliably fails. | +| todo | HoldMyBeer test | See report for August 2023. | todo | todo | +| todo | Peers test | See report for August 2023. | todo | todo | + +## Resulting changes +As a result of the testing efforts in 11-2023, these changes were made: +1. todo + +## Action Points +- Debugging efforts continuou +- + From 260bc93414fcc59a645f4a63a368165c92dd9fdb Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 6 Dec 2023 09:34:22 +0100 Subject: [PATCH 13/15] Updates contracts image to latest --- .../CodexContractsPlugin/CodexContractsContainerRecipe.cs | 2 +- ProjectPlugins/CodexPlugin/MarketplaceAccess.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index 88faab2..e381414 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexContractsPlugin { public class CodexContractsContainerRecipe : ContainerRecipeFactory { - public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests"; + public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests"; public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; diff --git a/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs b/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs index d910471..cde4eb6 100644 --- a/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs +++ b/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs @@ -26,7 +26,7 @@ namespace CodexPlugin public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) { - return RequestStorage(contentId, pricePerSlotPerSecond, requiredCollateral, minRequiredNumberOfNodes, proofProbability, duration, TimeSpan.FromMinutes(5)); + return RequestStorage(contentId, pricePerSlotPerSecond, requiredCollateral, minRequiredNumberOfNodes, proofProbability, duration, duration / 2); } public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry) From 074f5ebfaeded1eca0f166dd09e7911040619ea4 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 6 Dec 2023 09:59:45 +0100 Subject: [PATCH 14/15] Sets up transferSpeed class --- Framework/FileUtils/TrackedFile.cs | 5 ++ Framework/Logging/Stopwatch.cs | 22 ++++++-- Framework/Utils/ByteSize.cs | 7 +++ ProjectPlugins/CodexPlugin/CodexNode.cs | 12 ++++- ProjectPlugins/CodexPlugin/TransferSpeeds.cs | 52 +++++++++++++++++++ .../Helpers/PeerDownloadTestHelpers.cs | 6 +-- 6 files changed, 93 insertions(+), 11 deletions(-) create mode 100644 ProjectPlugins/CodexPlugin/TransferSpeeds.cs diff --git a/Framework/FileUtils/TrackedFile.cs b/Framework/FileUtils/TrackedFile.cs index 5f9b04a..694408b 100644 --- a/Framework/FileUtils/TrackedFile.cs +++ b/Framework/FileUtils/TrackedFile.cs @@ -37,6 +37,11 @@ namespace FileUtils return $"'{Filename}'{sizePostfix}"; } + public ByteSize GetFilesize() + { + return new ByteSize(GetFileSize()); + } + private void AssertEqual(TrackedFile? actual) { if (actual == null) FrameworkAssert.Fail("TestFile is null."); diff --git a/Framework/Logging/Stopwatch.cs b/Framework/Logging/Stopwatch.cs index fddd8b5..d2f13b1 100644 --- a/Framework/Logging/Stopwatch.cs +++ b/Framework/Logging/Stopwatch.cs @@ -16,19 +16,19 @@ namespace Logging this.debug = debug; } - public static void Measure(ILog log, string name, Action action, bool debug = false) + public static TimeSpan Measure(ILog log, string name, Action action, bool debug = false) { var sw = Begin(log, name, debug); action(); - sw.End(); + return sw.End(); } - public static T Measure(ILog log, string name, Func action, bool debug = false) + public static StopwatchResult Measure(ILog log, string name, Func action, bool debug = false) { var sw = Begin(log, name, debug); var result = action(); - sw.End(); - return result; + var duration = sw.End(); + return new StopwatchResult(result, duration); } public static Stopwatch Begin(ILog log) @@ -68,4 +68,16 @@ namespace Logging return duration; } } + + public class StopwatchResult + { + public StopwatchResult(T value, TimeSpan duration) + { + Value = value; + Duration = duration; + } + + public T Value { get; } + public TimeSpan Duration { get; } + } } diff --git a/Framework/Utils/ByteSize.cs b/Framework/Utils/ByteSize.cs index 170aaf7..69d5f91 100644 --- a/Framework/Utils/ByteSize.cs +++ b/Framework/Utils/ByteSize.cs @@ -46,6 +46,13 @@ } } + public class BytesPerSecond : ByteSize + { + public BytesPerSecond(long sizeInBytes) : base(sizeInBytes) + { + } + } + public static class ByteSizeIntExtensions { private const long Kilo = 1024; diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index d0a0636..2c79459 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -25,6 +25,7 @@ namespace CodexPlugin IMarketplaceAccess Marketplace { get; } CrashWatcher CrashWatcher { get; } PodInfo GetPodInfo(); + ITransferSpeeds TransferSpeeds { get; } void Stop(); } @@ -34,6 +35,7 @@ namespace CodexPlugin private const string UploadFailedMessage = "Unable to store block"; private readonly IPluginTools tools; private readonly EthAddress? ethAddress; + private readonly TransferSpeeds transferSpeeds; public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress) { @@ -43,6 +45,7 @@ namespace CodexPlugin Group = group; Marketplace = marketplaceAccess; Version = new CodexDebugVersionResponse(); + transferSpeeds = new TransferSpeeds(); } public RunningContainer Container { get { return CodexAccess.Container; } } @@ -51,6 +54,7 @@ namespace CodexPlugin public CodexNodeGroup Group { get; } public IMarketplaceAccess Marketplace { get; } public CodexDebugVersionResponse Version { get; private set; } + public ITransferSpeeds TransferSpeeds { get => transferSpeeds; } public IMetricsScrapeTarget MetricsScrapeTarget { get @@ -101,11 +105,14 @@ namespace CodexPlugin var logMessage = $"Uploading file {file.Describe()}..."; Log(logMessage); - var response = Stopwatch.Measure(tools.GetLog(), logMessage, () => + var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => { return CodexAccess.UploadFile(fileStream); }); + var response = measurement.Value; + transferSpeeds.AddUploadSample(file.GetFilesize(), measurement.Duration); + if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response."); if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block."); @@ -118,7 +125,8 @@ namespace CodexPlugin var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; Log(logMessage); var file = tools.GetFileManager().CreateEmptyFile(fileLabel); - Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file)); + var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file)); + transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement); Log($"Downloaded file {file.Describe()} to '{file.Filename}'."); return file; } diff --git a/ProjectPlugins/CodexPlugin/TransferSpeeds.cs b/ProjectPlugins/CodexPlugin/TransferSpeeds.cs new file mode 100644 index 0000000..ae1f291 --- /dev/null +++ b/ProjectPlugins/CodexPlugin/TransferSpeeds.cs @@ -0,0 +1,52 @@ +using Utils; + +namespace CodexPlugin +{ + public interface ITransferSpeeds + { + BytesPerSecond GetUploadSpeed(); + BytesPerSecond GetDownloadSpeed(); + } + + public class TransferSpeeds : ITransferSpeeds + { + private readonly List uploads = new List(); + private readonly List downloads = new List(); + + public void AddUploadSample(ByteSize bytes, TimeSpan duration) + { + uploads.Add(Convert(bytes, duration)); + } + + public void AddDownloadSample(ByteSize bytes, TimeSpan duration) + { + downloads.Add(Convert(bytes, duration)); + } + + public BytesPerSecond GetUploadSpeed() + { + return Average(uploads); + } + + public BytesPerSecond GetDownloadSpeed() + { + return Average(downloads); + } + + private static BytesPerSecond Convert(ByteSize size, TimeSpan duration) + { + double bytes = size.SizeInBytes; + double seconds = duration.TotalSeconds; + + return new BytesPerSecond(System.Convert.ToInt64(Math.Round(bytes / seconds))); + } + + private static BytesPerSecond Average(List list) + { + double sum = list.Sum(i => i.SizeInBytes); + double num = list.Count; + + return new BytesPerSecond(System.Convert.ToInt64(Math.Round(sum / num))); + } + } +} diff --git a/Tests/CodexTests/Helpers/PeerDownloadTestHelpers.cs b/Tests/CodexTests/Helpers/PeerDownloadTestHelpers.cs index fab2a23..6dab12a 100644 --- a/Tests/CodexTests/Helpers/PeerDownloadTestHelpers.cs +++ b/Tests/CodexTests/Helpers/PeerDownloadTestHelpers.cs @@ -9,7 +9,6 @@ namespace CodexTests.Helpers public class PeerDownloadTestHelpers : IFullConnectivityImplementation { private readonly FullConnectivityHelper helper; - private readonly ILog log; private readonly IFileManager fileManager; private ByteSize testFileSize; @@ -17,7 +16,6 @@ namespace CodexTests.Helpers { helper = new FullConnectivityHelper(log, this); testFileSize = 1.MB(); - this.log = log; this.fileManager = fileManager; } @@ -45,11 +43,11 @@ namespace CodexTests.Helpers private PeerConnectionState CheckConnectivity(Entry from, Entry to) { var expectedFile = GenerateTestFile(from.Node, to.Node); - var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(expectedFile)); + var contentId = from.Node.UploadFile(expectedFile); try { - var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded")); + var downloadedFile = DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"); expectedFile.AssertIsEqual(downloadedFile); return PeerConnectionState.Connection; } From 09554da362f72fd2c474fc24513997d0ba0241d9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 6 Dec 2023 10:50:02 +0100 Subject: [PATCH 15/15] Adds transfer speeds to status logs. --- Framework/Utils/ByteSize.cs | 5 +++ ProjectPlugins/CodexPlugin/TransferSpeeds.cs | 31 ++++++++++++++----- Tests/CodexContinuousTests/SingleTestRun.cs | 5 +++ Tests/CodexTests/CodexDistTest.cs | 32 ++++++++++++++------ Tests/DistTestCore/DistTest.cs | 18 ++++++++++- 5 files changed, 72 insertions(+), 19 deletions(-) diff --git a/Framework/Utils/ByteSize.cs b/Framework/Utils/ByteSize.cs index 69d5f91..a5204c7 100644 --- a/Framework/Utils/ByteSize.cs +++ b/Framework/Utils/ByteSize.cs @@ -51,6 +51,11 @@ public BytesPerSecond(long sizeInBytes) : base(sizeInBytes) { } + + public override string ToString() + { + return base.ToString() + "/s"; + } } public static class ByteSizeIntExtensions diff --git a/ProjectPlugins/CodexPlugin/TransferSpeeds.cs b/ProjectPlugins/CodexPlugin/TransferSpeeds.cs index ae1f291..b745f64 100644 --- a/ProjectPlugins/CodexPlugin/TransferSpeeds.cs +++ b/ProjectPlugins/CodexPlugin/TransferSpeeds.cs @@ -4,8 +4,8 @@ namespace CodexPlugin { public interface ITransferSpeeds { - BytesPerSecond GetUploadSpeed(); - BytesPerSecond GetDownloadSpeed(); + BytesPerSecond? GetUploadSpeed(); + BytesPerSecond? GetDownloadSpeed(); } public class TransferSpeeds : ITransferSpeeds @@ -23,14 +23,16 @@ namespace CodexPlugin downloads.Add(Convert(bytes, duration)); } - public BytesPerSecond GetUploadSpeed() + public BytesPerSecond? GetUploadSpeed() { - return Average(uploads); + if (!uploads.Any()) return null; + return uploads.Average(); } - public BytesPerSecond GetDownloadSpeed() + public BytesPerSecond? GetDownloadSpeed() { - return Average(downloads); + if (!downloads.Any()) return null; + return downloads.Average(); } private static BytesPerSecond Convert(ByteSize size, TimeSpan duration) @@ -40,13 +42,26 @@ namespace CodexPlugin return new BytesPerSecond(System.Convert.ToInt64(Math.Round(bytes / seconds))); } + } - private static BytesPerSecond Average(List list) + public static class ListExtensions + { + public static BytesPerSecond Average(this List list) { double sum = list.Sum(i => i.SizeInBytes); double num = list.Count; - return new BytesPerSecond(System.Convert.ToInt64(Math.Round(sum / num))); + return new BytesPerSecond(Convert.ToInt64(Math.Round(sum / num))); + } + + public static BytesPerSecond? OptionalAverage(this List? list) + { + if (list == null || !list.Any() || !list.Any(i => i != null)) return null; + var values = list.Where(i => i != null).Cast().ToArray(); + double sum = values.Sum(i => i.SizeInBytes); + double num = values.Length; + + return new BytesPerSecond(Convert.ToInt64(Math.Round(sum / num))); } } } diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 2ab5a33..39b45e5 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -197,6 +197,11 @@ namespace ContinuousTests if (error.Contains(":")) error = error.Substring(1 + error.LastIndexOf(":")); result.Add("error", error); + var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage(); + var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage(); + if (upload != null) result.Add("avgupload", upload.ToString()); + if (download != null) result.Add("avgdownload", download.ToString()); + return result; } diff --git a/Tests/CodexTests/CodexDistTest.cs b/Tests/CodexTests/CodexDistTest.cs index 4d6dab2..a8b406c 100644 --- a/Tests/CodexTests/CodexDistTest.cs +++ b/Tests/CodexTests/CodexDistTest.cs @@ -6,14 +6,13 @@ using Core; using DistTestCore; using DistTestCore.Helpers; using DistTestCore.Logs; -using NUnit.Framework; using NUnit.Framework.Constraints; namespace CodexTests { public class CodexDistTest : DistTest { - private readonly List onlineCodexNodes = new List(); + private readonly Dictionary> onlineCodexNodes = new Dictionary>(); public CodexDistTest() { @@ -23,12 +22,6 @@ namespace CodexTests ProjectPlugin.Load(); } - [TearDown] - public void TearDownCodexFixture() - { - onlineCodexNodes.Clear(); - } - protected override void Initialize(FixtureLog fixtureLog) { var localBuilder = new LocalCodexBuilder(fixtureLog); @@ -36,6 +29,16 @@ namespace CodexTests localBuilder.Build(); } + protected override void LifecycleStart(TestLifecycle lifecycle) + { + onlineCodexNodes.Add(lifecycle, new List()); + } + + protected override void LifecycleStop(TestLifecycle lifecycle) + { + onlineCodexNodes.Remove(lifecycle); + } + public ICodexNode AddCodex() { return AddCodex(s => { }); @@ -58,7 +61,7 @@ namespace CodexTests setup(s); OnCodexSetup(s); }); - onlineCodexNodes.AddRange(group); + onlineCodexNodes[Get()].AddRange(group); return group; } @@ -74,7 +77,7 @@ namespace CodexTests public IEnumerable GetAllOnlineCodexNodes() { - return onlineCodexNodes; + return onlineCodexNodes[Get()]; } public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "") @@ -85,5 +88,14 @@ namespace CodexTests protected virtual void OnCodexSetup(ICodexSetup setup) { } + + protected override void CollectStatusLogData(TestLifecycle lifecycle, Dictionary data) + { + var nodes = onlineCodexNodes[lifecycle]; + var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage(); + var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage(); + if (upload != null) data.Add("avgupload", upload.ToString()); + if (download != null) data.Add("avgdownload", download.ToString()); + } } } diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 74898d5..06014e0 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -147,6 +147,18 @@ namespace DistTestCore { } + protected virtual void LifecycleStart(TestLifecycle lifecycle) + { + } + + protected virtual void LifecycleStop(TestLifecycle lifecycle) + { + } + + protected virtual void CollectStatusLogData(TestLifecycle lifecycle, Dictionary data) + { + } + protected TestLifecycle Get() { lock (lifecycleLock) @@ -166,6 +178,7 @@ namespace DistTestCore var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace); lifecycles.Add(testName, lifecycle); + LifecycleStart(lifecycle); } }); } @@ -175,13 +188,16 @@ namespace DistTestCore var lifecycle = Get(); var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); + var data = lifecycle.GetPluginMetadata(); + CollectStatusLogData(lifecycle, data); fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})"); - statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetPluginMetadata()); + statusLog.ConcludeTest(testResult, testDuration, data); Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => { WriteEndTestLog(lifecycle.Log); IncludeLogsOnTestFailure(lifecycle); + LifecycleStop(lifecycle); lifecycle.DeleteAllResources(); lifecycle = null!; });