Merge branch 'master' into feature/public-testnet-deploying

This commit is contained in:
benbierens 2023-11-20 11:17:17 +01:00
commit 005c9b4259
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
17 changed files with 237 additions and 71 deletions

View File

@ -2,16 +2,6 @@ name: Run Continuous Tests
on:
# push:
# branches:
# - master
# tags:
# - 'v*.*.*'
# paths-ignore:
# - '**/*.md'
# - '.gitignore'
# - 'docker/**'
# - '!docker/continuous-tests-job.yaml'
workflow_dispatch:
inputs:
source:
@ -23,15 +13,11 @@ on:
required: false
type: string
nameprefix:
description: Runner name prefix (c-tests-runner)
required: false
type: string
namespace:
description: Runner namespace (default)
description: Resources prefix (c-tests)
required: false
type: string
tests_target_duration:
description: Runner target duration (172800 = 48h)
description: Runner target duration (2d)
required: false
type: string
tests_filter:
@ -40,24 +26,42 @@ on:
type: string
tests_cleanup:
description: Runner tests cleanup
type: choice
options:
- true
- false
type: boolean
default: true
deployment_namespace:
description: Deployment namespace (c-tests-$runid)
workflow_call:
inputs:
source:
description: Repository with tests (current)
required: false
type: string
branch:
description: Branch with tests (master)
required: false
type: string
nameprefix:
description: Resources prefix (c-tests)
required: false
type: string
tests_target_duration:
description: Runner target duration (2d)
required: false
type: string
tests_filter:
description: Runner tests filter ("")
required: false
type: string
tests_cleanup:
description: Runner tests cleanup
required: false
type: boolean
env:
BRANCH: ${{ github.ref_name }}
SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
NAMEPREFIX: c-tests-runner
BRANCH: ${{ github.ref_name }}
NAMEPREFIX: c-tests
NAMESPACE: default
DEPLOYMENT_NAMESPACE: c-tests
TESTS_TARGET_DURATION: 172800
TESTS_TARGET_DURATION: 2d
TESTS_FILTER: ""
TESTS_CLEANUP: true
JOB_MANIFEST: docker/continuous-tests-job.yaml
@ -78,14 +82,14 @@ jobs:
RUNID=$(date +%Y%m%d-%H%M%S)
echo "RUNID=${RUNID}" >> $GITHUB_ENV
echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
[[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ github.event.inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ github.event.inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ github.event.inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}-${RUNID}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}-${RUNID}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}-${RUNID}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.nameprefix }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ inputs.nameprefix }}-${RUNID}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.NAMEPREFIX }}-${RUNID}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
- name: Kubectl - Install ${{ env.KUBE_VERSION }}
uses: azure/setup-kubectl@v3
@ -106,7 +110,8 @@ jobs:
echo "----"
echo "Repository: ${{ env.SOURCE }}"
echo "Branch: ${{ env.BRANCH }}"
echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
echo "Runner job: ${{ env.NAMEPREFIX }}"
echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`"
echo "Runner namespace: ${{ env.NAMESPACE }}"
echo "----"
echo "Tests runid: ${{ env.RUNID }}"
@ -115,3 +120,30 @@ jobs:
echo "Tests filter: ${{ env.TESTS_FILTER }}"
echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
echo "----"
- name: Show Runner logs
if: false
run: |
# Variables
# We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node
duration=600
namespace="${{ env.NAMESPACE }}"
pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
# Check Pod status
WAIT=120
SECONDS=0
sleep=1
while (( SECONDS < WAIT )); do
phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}")
[[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; }
sleep $sleep
done
# Get logs
timeout $duration \
kubectl logs $pod \
-n $namespace \
-f \
--tail=-1 \
--timestamps || true

View File

@ -2,16 +2,6 @@ name: Run Dist Tests
on:
# push:
# branches:
# - master
# tags:
# - 'v*.*.*'
# paths-ignore:
# - '**/*.md'
# - '.gitignore'
# - 'docker/**'
# - '!docker/dist-tests-job.yaml'
workflow_dispatch:
inputs:
source:
@ -37,8 +27,8 @@ on:
env:
BRANCH: ${{ github.ref_name }}
SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
BRANCH: ${{ github.ref_name }}
NAMEPREFIX: d-tests-runner
NAMESPACE: default
COMMAND: dotnet test Tests/CodexTests
@ -57,14 +47,14 @@ jobs:
- name: Variables
run: |
[[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
[[ -n "${{ github.event.inputs.command }}" ]] && COMMAND="${{ github.event.inputs.command }}" || COMMAND="${{ env.COMMAND }}"
echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
[[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
- name: Kubectl - Install ${{ env.KUBE_VERSION }}
uses: azure/setup-kubectl@v3
@ -85,7 +75,7 @@ jobs:
echo "----"
echo "Repository: ${{ env.SOURCE }}"
echo "Branch: ${{ env.BRANCH }}"
echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
echo "Runner namespace: ${{ env.NAMESPACE }}"
echo "----"
echo "Tests runid: ${{ env.RUNID }}"

View File

@ -0,0 +1,20 @@
name: Report - HoldMyBeerTest
on:
schedule:
- cron: '30 */49 * * *'
workflow_dispatch:
jobs:
run_tests:
name: Call runner
uses: ./.github/workflows/continuous-tests.yaml
with:
source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
branch: master
nameprefix: c-tests-report-holdmybeertest
tests_target_duration: 48h
tests_filter: HoldMyBeerTest
tests_cleanup: true
secrets: inherit

20
.github/workflows/report-PeersTest.yaml vendored Normal file
View File

@ -0,0 +1,20 @@
name: Report - PeersTest
on:
schedule:
- cron: '30 */49 * * *'
workflow_dispatch:
jobs:
run_tests:
name: Call runner
uses: ./.github/workflows/continuous-tests.yaml
with:
source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
branch: master
nameprefix: c-tests-report-peerstest
tests_target_duration: 48h
tests_filter: PeersTest
tests_cleanup: true
secrets: inherit

View File

@ -23,6 +23,35 @@
return result;
}
public static TimeSpan ParseTimespan(string span)
{
span = span.Replace(" ", "").Replace(",", "");
var result = TimeSpan.Zero;
var number = "";
foreach (var c in span)
{
if (char.IsNumber(c)) number += c;
else
{
var value = Convert.ToInt32(number);
number = "";
if (c == 'd') result += TimeSpan.FromDays(value);
else if (c == 'h') result += TimeSpan.FromHours(value);
else if (c == 'm') result += TimeSpan.FromMinutes(value);
else if (c == 's') result += TimeSpan.FromSeconds(value);
else throw new Exception("Unknown time modifier: " + c);
}
}
if (!string.IsNullOrEmpty(number))
{
var value = Convert.ToInt32(number);
result += TimeSpan.FromSeconds(value);
}
return result;
}
public static void WaitUntil(Func<bool> predicate)
{
WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));

View File

@ -24,8 +24,8 @@ namespace ContinuousTests
[Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
public int StopOnFailure { get; set; } = 0;
[Uniform("target-duration", "td", "TARGETDURATION", false, "If greater than zero, runner will run for this many seconds before stopping.")]
public int TargetDurationSeconds { get; set; } = 0;
[Uniform("target-duration", "td", "TARGETDURATION", false, "If set, runner will run for this length of time before stopping. Supports seconds, or '1d2h3m4s' format.")]
public string TargetDurationSeconds { get; set; } = string.Empty;
[Uniform("filter", "f", "FILTER", false, "If set, runs only tests whose names contain any of the filter strings. Comma-separated. Case sensitive.")]
public string Filter { get; set; } = string.Empty;

View File

@ -98,13 +98,13 @@ namespace ContinuousTests
private void WaitUntilFinished(LogSplitter overviewLog, StatusLog statusLog, DateTime startTime, TestLoop[] testLoops)
{
var testDuration = Time.FormatDuration(DateTime.UtcNow - startTime);
var testDuration = (DateTime.UtcNow - startTime).TotalSeconds.ToString();
var testData = FormatTestRuns(testLoops);
overviewLog.Log("Total duration: " + testDuration);
if (config.TargetDurationSeconds > 0)
if (!string.IsNullOrEmpty(config.TargetDurationSeconds))
{
var targetDuration = TimeSpan.FromSeconds(config.TargetDurationSeconds);
var targetDuration = Time.ParseTimespan(config.TargetDurationSeconds);
var wasCancelled = cancelToken.WaitHandle.WaitOne(targetDuration);
if (!wasCancelled)
{

View File

@ -16,7 +16,14 @@ public class Program
Cancellation.Cts.Cancel();
};
try
{
runner.Run();
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
Console.WriteLine("Done.");
}
}

View File

@ -59,7 +59,7 @@ namespace ContinuousTests
overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
Environment.Exit(-1);
}
});
}, nameof(SingleTestRun));
}
private void RunTest(Action<bool> resultHandler)
@ -191,6 +191,12 @@ namespace ContinuousTests
result.Add("teststart", testStart.ToString("o"));
result.Add("testname", testName);
result.Add("message", message);
result.Add("involvedpods", string.Join(",", nodes.Select(n => n.GetName())));
var error = message.Split(Environment.NewLine).First();
if (error.Contains(":")) error = error.Substring(1 + error.LastIndexOf(":"));
result.Add("error", error);
return result;
}

View File

@ -5,11 +5,23 @@
private readonly object taskLock = new();
private readonly List<Task> activeTasks = new List<Task>();
public void Run(Action action)
public void Run(Action action, string name)
{
lock (taskLock)
{
activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
activeTasks.Add(Task.Run(() => CatchException(action, name)).ContinueWith(CleanupTask, null));
}
}
private void CatchException(Action action, string name)
{
try
{
action();
}
catch (Exception ex)
{
Console.WriteLine($"Exception in task '{name}': " + ex);
}
}

View File

@ -69,7 +69,7 @@ namespace ContinuousTests
overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
Environment.Exit(-1);
}
});
}, nameof(TestLoop));
}
private void StartTest()

View File

@ -1,6 +1,6 @@
using Logging;
using Newtonsoft.Json;
using Utils;
using System.Globalization;
namespace DistTestCore.Logs
{
@ -20,7 +20,7 @@ namespace DistTestCore.Logs
public void ConcludeTest(string resultStatus, TimeSpan testDuration, Dictionary<string, string> data)
{
ConcludeTest(resultStatus, Time.FormatDuration(testDuration), data);
ConcludeTest(resultStatus, testDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), data);
}
public void ConcludeTest(string resultStatus, string testDuration, Dictionary<string, string> data)

View File

@ -58,10 +58,9 @@ namespace DistTestCore
return entryPoint.GetPluginMetadata();
}
public string GetTestDuration()
public TimeSpan GetTestDuration()
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
return DateTime.UtcNow - testStart;
}
public void OnContainersStarted(RunningContainers rc)

View File

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="nunit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,25 @@
using NUnit.Framework;
using Utils;
namespace FrameworkTests.Utils
{
[TestFixture]
public class TimeTests
{
[Test]
public void Timespan()
{
Assert.That(Time.ParseTimespan("10"), Is.EqualTo(TimeSpan.FromSeconds(10)));
Assert.That(Time.ParseTimespan("10s"), Is.EqualTo(TimeSpan.FromSeconds(10)));
Assert.That(Time.ParseTimespan("10m"), Is.EqualTo(TimeSpan.FromMinutes(10)));
Assert.That(Time.ParseTimespan("10d"), Is.EqualTo(TimeSpan.FromDays(10)));
Assert.That(Time.ParseTimespan("120s"), Is.EqualTo(TimeSpan.FromSeconds(120)));
Assert.That(Time.ParseTimespan("2d14h6m28s"), Is.EqualTo(
TimeSpan.FromDays(2) +
TimeSpan.FromHours(14) +
TimeSpan.FromMinutes(6) +
TimeSpan.FromSeconds(28)
));
}
}
}

View File

@ -51,6 +51,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestClusterStarter", "Tools
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeployAndRunPlugin", "ProjectPlugins\DeployAndRunPlugin\DeployAndRunPlugin.csproj", "{1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrameworkTests", "Tests\FrameworkTests\FrameworkTests.csproj", "{25E72004-4D71-4D1E-A193-FC125D12FF96}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -137,6 +139,10 @@ Global
{1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}.Release|Any CPU.Build.0 = Release|Any CPU
{25E72004-4D71-4D1E-A193-FC125D12FF96}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{25E72004-4D71-4D1E-A193-FC125D12FF96}.Debug|Any CPU.Build.0 = Debug|Any CPU
{25E72004-4D71-4D1E-A193-FC125D12FF96}.Release|Any CPU.ActiveCfg = Release|Any CPU
{25E72004-4D71-4D1E-A193-FC125D12FF96}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -162,6 +168,7 @@ Global
{FB96A58B-F7F0-490A-9A85-72A96A018042} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124}
{3E38A906-C2FC-43DC-8CA2-FC07C79CF3CA} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
{1CC5AF82-8924-4C7E-BFF1-3125D86E53FB} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124}
{25E72004-4D71-4D1E-A193-FC125D12FF96} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}

View File

@ -1,10 +1,10 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ${NAMEPREFIX}-${RUNID}
name: ${NAMEPREFIX}
namespace: ${NAMESPACE}
labels:
name: ${NAMEPREFIX}-${RUNID}
name: ${NAMEPREFIX}
runid: ${RUNID}
spec:
backoffLimit: 0
@ -13,7 +13,7 @@ spec:
name: ${NAMEPREFIX}
labels:
app: continuous-tests-runner
name: ${NAMEPREFIX}-${RUNID}
name: ${NAMEPREFIX}
runid: ${RUNID}
spec:
priorityClassName: system-node-critical