status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-02-19 19:48:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/tests

Browse Source
This commit is contained in:
Corbo12 2023-06-07 14:14:26 +02:00
commit 21292456aa
62 changed files with 1336 additions and 500 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
.github/workflows/dist-tests.yaml vendored Normal file
View File

@ -0,0 +1,73 @@
name: Dist Tests
on:
# push:
# branches:
# - master
# tags:
# - 'v*.*.*'
# paths-ignore:
# - '**/*.md'
# - '.gitignore'
# - 'docker/**'
# - '!docker/job.yaml'
workflow_dispatch:
inputs:
branch:
description: Branch
required: false
type: string
source:
description: Repository with tests
required: false
type: string
nameprefix:
description: Runner job/pod name prefix
required: false
type: string
namespace:
description: Kubernetes namespace for runner
required: false
type: string
env:
BRANCH: ${{ github.ref_name }}
SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
NAMEPREFIX: cs-codex-dist-tests
NAMESPACE: cs-codex-dist-tests
JOB_MANIFEST: docker/job.yaml
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
KUBE_VERSION: v1.26.1
jobs:
run_tests:
name: Run Tests
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Variables
run: |
[[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
[[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
- name: Kubectl - Install ${{ env.KUBE_VERSION }}
uses: azure/setup-kubectl@v3
with:
version: ${{ env.KUBE_VERSION }}
- name: Kubectl - Kubeconfig
run: |
mkdir -p "${HOME}"/.kube
echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
- name: Kubectl - Create Job
run: |
export RUNID=$(date +%Y%m%d-%H%M%S)
envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

21
DistTestCore/AutoBootstrapDistTest.cs
View File

@ -1,9 +1,9 @@
namespace DistTestCore
using NUnit.Framework;
namespace DistTestCore
{
public class AutoBootstrapDistTest : DistTest
{
private IOnlineCodexNode? bootstrapNode;
public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
{
throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " +
@ -12,19 +12,18 @@
public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
{
var codexSetup = new CodexSetup(numberOfNodes);
var codexSetup = CreateCodexSetup(numberOfNodes);
setup(codexSetup);
codexSetup.WithBootstrapNode(EnsureBootstapNode());
codexSetup.WithBootstrapNode(BootstrapNode);
return BringOnline(codexSetup);
}
private IOnlineCodexNode EnsureBootstapNode()
[SetUp]
public void SetUpBootstrapNode()
{
if (bootstrapNode == null)
{
bootstrapNode = base.SetupCodexBootstrapNode(s => { });
}
return bootstrapNode;
BootstrapNode = BringOnline(CreateCodexSetup(1))[0];
}
protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
}
}

5
DistTestCore/BaseStarter.cs
View File

@ -32,6 +32,11 @@ namespace DistTestCore
lifecycle.Log.Log($"{GetClassName()} {msg}");
}
protected void Debug(string msg)
{
lifecycle.Log.Debug($"{GetClassName()} {msg}", 1);
}
private string GetClassName()
{
return $"({GetType().Name})";

86
DistTestCore/Codex/CodexAccess.cs
View File

@ -1,17 +1,14 @@
using KubernetesWorkflow;
using Logging;
namespace DistTestCore.Codex
{
public class CodexAccess
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
private readonly TestLifecycle lifecycle;
public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer)
{
this.log = log;
this.timeSet = timeSet;
this.lifecycle = lifecycle;
Container = runningContainer;
}
@ -19,7 +16,30 @@ namespace DistTestCore.Codex
public CodexDebugResponse GetDebugInfo()
{
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
var http = Http(timeout);
var str = http.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
{
return new CodexDebugPeerResponse
{
IsPeerFound = false
};
}
var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
result.IsPeerFound = true;
return result;
}
public string UploadFile(FileStream fileStream)
@ -42,6 +62,11 @@ namespace DistTestCore.Codex
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
public void EnsureOnline()
{
try
@ -51,25 +76,20 @@ namespace DistTestCore.Codex
var nodePeerId = debugInfo.id;
var nodeName = Container.Name;
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
}
catch (Exception e)
{
log.Error($"Failed to start codex node: {e}. Test infra failure.");
lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
}
}
private Http Http()
private Http Http(TimeSpan? timeoutOverride = null)
{
var ip = Container.Pod.Cluster.IP;
var port = Container.ServicePorts[0].Number;
return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
var address = lifecycle.Configuration.GetAddress(Container);
return new Http(lifecycle.Log, lifecycle.TimeSet, address, baseUrl: "/api/codex/v1", timeoutOverride);
}
}
@ -82,6 +102,22 @@ namespace DistTestCore.Codex
public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
public CodexDebugVersionResponse codex { get; set; } = new();
public CodexDebugTableResponse table { get; set; } = new();
}
public class CodexDebugTableResponse
{
public CodexDebugTableNodeResponse localNode { get; set; } = new();
public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
}
public class CodexDebugTableNodeResponse
{
public string nodeId { get; set; } = string.Empty;
public string peerId { get; set; } = string.Empty;
public string record { get; set; } = string.Empty;
public string address { get; set; } = string.Empty;
public bool seen { get; set; }
}
public class EnginePeerResponse
@ -110,6 +146,20 @@ namespace DistTestCore.Codex
public string revision { get; set; } = string.Empty;
}
public class CodexDebugPeerResponse
{
public bool IsPeerFound { get; set; }
public string peerId { get; set; } = string.Empty;
public long seqNo { get; set; }
public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
}
public class CodexDebugPeerAddressResponse
{
public string address { get; set; } = string.Empty;
}
public class CodexSalesAvailabilityRequest
{
public string size { get; set; } = string.Empty;

21
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -1,5 +1,4 @@
using System.Runtime.InteropServices;
using DistTestCore.Marketplace;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore.Codex
@ -7,12 +6,13 @@ namespace DistTestCore.Codex
public class CodexContainerRecipe : ContainerRecipeFactory
{
#if Arm64
public const string DockerImage = "emizzle/nim-codex-arm64:sha-c7af585";
public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
#else
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
public const string DockerImage = "thatbenbierens/codexlocal:latest";
//public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
#endif
public const string MetricsPortTag = "metrics_port";
public const string DiscoveryPortTag = "discovery-port";
protected override string Image => DockerImage;
@ -22,7 +22,8 @@ namespace DistTestCore.Codex
AddExposedPortAndVar("API_PORT");
AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}");
AddInternalPortAndVar("DISC_PORT");
AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag);
AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
var listenPort = AddInternalPort();
AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
@ -31,11 +32,6 @@ namespace DistTestCore.Codex
{
AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr);
}
if (config.LogLevel != null)
{
AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
}
if (config.StorageQuota != null)
{
AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
@ -53,12 +49,13 @@ namespace DistTestCore.Codex
var companionNodeAccount = companionNode.Accounts[Index];
Additional(companionNodeAccount);
var ip = companionNode.RunningContainer.Pod.Ip;
var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");
AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
AddEnvVar("PERSISTENCE", "1");
}
}
}

7
DistTestCore/Codex/CodexStartupConfig.cs
View File

@ -5,9 +5,14 @@ namespace DistTestCore.Codex
{
public class CodexStartupConfig
{
public CodexStartupConfig(CodexLogLevel logLevel)
{
LogLevel = logLevel;
}
public string? NameOverride { get; set; }
public Location Location { get; set; }
public CodexLogLevel? LogLevel { get; set; }
public CodexLogLevel LogLevel { get; }
public ByteSize? StorageQuota { get; set; }
public bool MetricsEnabled { get; set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; set; }

2
DistTestCore/CodexNodeGroup.cs
View File

@ -69,7 +69,7 @@ namespace DistTestCore
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{
var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
var access = new CodexAccess(lifecycle, c);
return factory.CreateOnlineCodexNode(access, this);
}
}

12
DistTestCore/CodexSetup.cs
View File

@ -8,7 +8,6 @@ namespace DistTestCore
{
ICodexSetup WithName(string name);
ICodexSetup At(Location location);
ICodexSetup WithLogLevel(CodexLogLevel level);
ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
ICodexSetup WithStorageQuota(ByteSize storageQuota);
ICodexSetup EnableMetrics();
@ -20,7 +19,8 @@ namespace DistTestCore
{
public int NumberOfNodes { get; }
public CodexSetup(int numberOfNodes)
public CodexSetup(int numberOfNodes, CodexLogLevel logLevel)
: base(logLevel)
{
NumberOfNodes = numberOfNodes;
}
@ -43,12 +43,6 @@ namespace DistTestCore
return this;
}
public ICodexSetup WithLogLevel(CodexLogLevel level)
{
LogLevel = level;
return this;
}
public ICodexSetup WithStorageQuota(ByteSize storageQuota)
{
StorageQuota = storageQuota;
@ -80,7 +74,7 @@ namespace DistTestCore
private IEnumerable<string> DescribeArgs()
{
if (LogLevel != null) yield return $"LogLevel={LogLevel}";
yield return $"LogLevel={LogLevel}";
if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
}

6
DistTestCore/CodexStarter.cs
View File

@ -1,6 +1,7 @@
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
using Logging;
namespace DistTestCore
{
@ -27,7 +28,8 @@ namespace DistTestCore
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
var podInfo = group.Containers.RunningPod.PodInfo;
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
LogSeparator();
return group;
}
@ -74,7 +76,7 @@ namespace DistTestCore
{
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
RunningGroups.Add(group);
group.EnsureOnline();
Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
return group;
}

77
DistTestCore/Configuration.cs
View File

@ -1,32 +1,89 @@
using KubernetesWorkflow;
using DistTestCore.Codex;
using KubernetesWorkflow;
namespace DistTestCore
{
public class Configuration
{
private readonly string? kubeConfigFile;
private readonly string logPath;
private readonly bool logDebug;
private readonly string dataFilesPath;
private readonly CodexLogLevel codexLogLevel;
private readonly TestRunnerLocation runnerLocation;
public Configuration()
{
kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
codexLogLevel = ParseEnum<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
runnerLocation = ParseEnum<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
}
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{
return new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: "ct-",
kubeConfigFile: null,
kubeConfigFile: kubeConfigFile,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay(),
locationMap: new[]
{
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
}
retryDelay: timeSet.WaitForK8sServiceDelay()
);
}
public Logging.LogConfig GetLogConfig()
{
return new Logging.LogConfig("CodexTestLogs", debugEnabled: false);
return new Logging.LogConfig(logPath, debugEnabled: logDebug);
}
public string GetFileManagerFolder()
{
return "TestDataFiles";
return dataFilesPath;
}
public CodexLogLevel GetCodexLogLevel()
{
return codexLogLevel;
}
public TestRunnerLocation GetTestRunnerLocation()
{
return runnerLocation;
}
public RunningContainerAddress GetAddress(RunningContainer container)
{
if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
{
return container.ClusterInternalAddress;
}
return container.ClusterExternalAddress;
}
private static string GetEnvVarOrDefault(string varName, string defaultValue)
{
var v = Environment.GetEnvironmentVariable(varName);
if (v == null) return defaultValue;
return v;
}
private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue)
{
var v = Environment.GetEnvironmentVariable(varName);
if (v == null) return defaultValue;
return v;
}
private static T ParseEnum<T>(string value)
{
return (T)Enum.Parse(typeof(T), value, true);
}
}
public enum TestRunnerLocation
{
ExternalToCluster,
InternalToCluster,
}
}

45
DistTestCore/DistTest.cs
View File

@ -1,4 +1,5 @@
using DistTestCore.Codex;
using DistTestCore.Helpers;
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
@ -25,8 +26,14 @@ namespace DistTestCore
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
}
public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
[OneTimeSetUp]
public void GlobalSetup()
{
@ -85,6 +92,17 @@ namespace DistTestCore
return Get().FileManager.GenerateTestFile(size);
}
/// <summary>
/// Any test files generated in 'action' will be deleted after it returns.
/// This helps prevent large tests from filling up discs.
/// </summary>
public void ScopedTestFiles(Action action)
{
Get().FileManager.PushFileSet();
action();
Get().FileManager.PopFileSet();
}
public IOnlineCodexNode SetupCodexBootstrapNode()
{
return SetupCodexBootstrapNode(s => { });
@ -116,7 +134,7 @@ namespace DistTestCore
public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
{
var codexSetup = new CodexSetup(numberOfNodes);
var codexSetup = CreateCodexSetup(numberOfNodes);
setup(codexSetup);
@ -128,16 +146,31 @@ namespace DistTestCore
return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
}
protected void Log(string msg)
public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
{
TestContext.Progress.WriteLine(msg);
Get().Log.Log(msg);
return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
}
protected void Debug(string msg)
public BaseLog GetTestLog()
{
return Get().Log;
}
public void Log(string msg)
{
TestContext.Progress.WriteLine(msg);
Get().Log.Debug(msg);
GetTestLog().Log(msg);
}
public void Debug(string msg)
{
TestContext.Progress.WriteLine(msg);
GetTestLog().Debug(msg);
}
protected CodexSetup CreateCodexSetup(int numberOfNodes)
{
return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
}
private TestLifecycle Get()

25
DistTestCore/FileManager.cs
View File

@ -9,6 +9,8 @@ namespace DistTestCore
TestFile CreateEmptyTestFile();
TestFile GenerateTestFile(ByteSize size);
void DeleteAllTestFiles();
void PushFileSet();
void PopFileSet();
}
public class FileManager : IFileManager
@ -18,6 +20,7 @@ namespace DistTestCore
private readonly Random random = new Random();
private readonly TestLog log;
private readonly string folder;
private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
public FileManager(TestLog log, Configuration configuration)
{
@ -31,6 +34,7 @@ namespace DistTestCore
{
var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
File.Create(result.Filename).Close();
if (fileSetStack.Any()) fileSetStack.Last().Add(result);
return result;
}
@ -47,6 +51,27 @@ namespace DistTestCore
DeleteDirectory();
}
public void PushFileSet()
{
fileSetStack.Add(new List<TestFile>());
}
public void PopFileSet()
{
if (!fileSetStack.Any()) return;
var pop = fileSetStack.Last();
fileSetStack.Remove(pop);
foreach (var file in pop)
{
try
{
File.Delete(file.Filename);
}
catch { }
}
}
private void GenerateFileBytes(TestFile result, ByteSize size)
{
long bytesLeft = size.SizeInBytes;

4
DistTestCore/GethStarter.cs
View File

@ -33,7 +33,7 @@ namespace DistTestCore
private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
{
var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
var interaction = marketplaceNetwork.StartInteraction(lifecycle);
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
@ -52,7 +52,7 @@ namespace DistTestCore
private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
{
return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork);
return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork);
}
private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)

22
DistTestCore/Helpers/AssertHelpers.cs Normal file
View File

@ -0,0 +1,22 @@
using NUnit.Framework.Constraints;
using NUnit.Framework;
using Utils;
namespace DistTestCore.Helpers
{
public static class AssertHelpers
{
public static void RetryAssert<T>(IResolveConstraint constraint, Func<T> actual, string message)
{
try
{
var c = constraint.Resolve();
Time.WaitUntil(() => c.ApplyTo(actual()).IsSuccess);
}
catch (TimeoutException)
{
Assert.That(actual(), constraint, message);
}
}
}
}

239
DistTestCore/Helpers/PeerConnectionTestHelpers.cs Normal file
View File

@ -0,0 +1,239 @@
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
namespace DistTestCore.Helpers
{
public class PeerConnectionTestHelpers
{
private readonly Random random = new Random();
private readonly DistTest test;
public PeerConnectionTestHelpers(DistTest test)
{
this.test = test;
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullyConnected(nodes.ToArray());
}
public void AssertFullyConnected(params IOnlineCodexNode[] nodes)
{
test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);
var pairs = CreatePairs(entries);
RetryWhilePairs(pairs, () =>
{
CheckAndRemoveSuccessful(pairs);
});
if (pairs.Any())
{
test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
}
else
{
test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
}
private static void RetryWhilePairs(List<Pair> pairs, Action action)
{
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10);
while (pairs.Any() && timeout > DateTime.UtcNow)
{
action();
if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5));
}
}
private void CheckAndRemoveSuccessful(List<Pair> pairs)
{
var checkTasks = pairs.Select(p => Task.Run(() =>
{
ApplyRandomDelay();
p.Check();
})).ToArray();
Task.WaitAll(checkTasks);
foreach (var pair in pairs.ToArray())
{
if (pair.Success)
{
test.Log(pair.GetMessage());
pairs.Remove(pair);
}
}
}
private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
if (incorrectDiscoveryEndpoints.Any())
{
Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
}
return entries;
}
private static List<Pair> CreatePairs(Entry[] entries)
{
return CreatePairsIterator(entries).ToList();
}
private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
{
for (var x = 0; x < entries.Length; x++)
{
for (var y = x + 1; y < entries.Length; y++)
{
yield return new Pair(entries[x], entries[y]);
}
}
}
private void ApplyRandomDelay()
{
// Calling all the nodes all at the same time is not exactly nice.
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
}
public class Entry
{
public Entry(IOnlineCodexNode node)
{
Node = node;
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get; }
public CodexDebugResponse Response { get; }
public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
{
foreach (var peer in Response.table.nodes)
{
var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
if (expected != peer.address)
{
yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
}
}
}
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
{
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
if (peer == null) return $"peerId: {node.peerId} is not known.";
var n = (OnlineCodexNode)peer.Node;
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
return $"{ip}:{discPort.Number}";
}
}
public enum PeerConnectionState
{
Unknown,
Connection,
NoConnection,
}
public class Pair
{
private readonly TimeSpan timeout = TimeSpan.FromSeconds(60);
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
public Pair(Entry a, Entry b)
{
A = a;
B = b;
}
public Entry A { get; }
public Entry B { get; }
public PeerConnectionState AKnowsB { get; private set; }
public PeerConnectionState BKnowsA { get; private set; }
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
public void Check()
{
aToBTime = Measure(() => AKnowsB = Knows(A, B));
bToATime = Measure(() => BKnowsA = Knows(B, A));
}
public string GetMessage()
{
return GetResultMessage() + GetTimePostfix();
}
private string GetResultMessage()
{
var aName = A.Response.id;
var bName = B.Response.id;
if (Success)
{
return $"{aName} and {bName} know each other.";
}
return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
}
private string GetTimePostfix()
{
var aName = A.Response.id;
var bName = B.Response.id;
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
}
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;
action();
return DateTime.UtcNow - start;
}
private PeerConnectionState Knows(Entry a, Entry b)
{
lock (a)
{
var peerId = b.Response.id;
try
{
var response = a.Node.GetDebugPeer(peerId, timeout);
if (!response.IsPeerFound)
{
return PeerConnectionState.NoConnection;
}
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
{
return PeerConnectionState.Connection;
}
}
catch
{
}
// Didn't get a conclusive answer. Try again later.
return PeerConnectionState.Unknown;
}
}
}
}
}

52
DistTestCore/Helpers/PeerDownloadTestHelpers.cs Normal file
View File

@ -0,0 +1,52 @@
namespace DistTestCore.Helpers
{
public class PeerDownloadTestHelpers
{
private readonly DistTest test;
public PeerDownloadTestHelpers(DistTest test)
{
this.test = test;
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullDownloadInterconnectivity(nodes, 1.MB());
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
{
test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
foreach (var node in nodes)
{
var uploader = node;
var downloaders = nodes.Where(n => n != uploader).ToArray();
test.ScopedTestFiles(() =>
{
PerformTest(uploader, downloaders, testFileSize);
});
}
test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
{
// 1 test file per downloader.
var files = downloaders.Select(d => test.GenerateTestFile(testFileSize)).ToArray();
// Upload all the test files to the uploader.
var contentIds = files.Select(uploader.UploadFile).ToArray();
// Each downloader should retrieve its own test file.
for (var i = 0; i < downloaders.Length; i++)
{
var expectedFile = files[i];
var downloadedFile = downloaders[i].DownloadContent(contentIds[i]);
expectedFile.AssertIsEqual(downloadedFile);
}
}
}
}

87
DistTestCore/Http.cs
View File

@ -1,6 +1,6 @@
using Logging;
using KubernetesWorkflow;
using Logging;
using Newtonsoft.Json;
using NUnit.Framework;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using Utils;
@ -11,18 +11,17 @@ namespace DistTestCore
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
private readonly string ip;
private readonly int port;
private readonly RunningContainerAddress address;
private readonly string baseUrl;
private readonly TimeSpan? timeoutOverride;
public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
public Http(BaseLog log, ITimeSet timeSet, RunningContainerAddress address, string baseUrl, TimeSpan? timeoutOverride = null)
{
this.log = log;
this.timeSet = timeSet;
this.ip = ip;
this.port = port;
this.address = address;
this.baseUrl = baseUrl;
this.timeoutOverride = timeoutOverride;
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
}
@ -38,7 +37,7 @@ namespace DistTestCore
var str = Time.Wait(result.Content.ReadAsStringAsync());
Log(url, str);
return str; ;
});
}, $"HTTP-GET:{route}");
}
public T HttpGetJson<T>(string route)
@ -62,10 +61,10 @@ namespace DistTestCore
using var content = JsonContent.Create(body);
Log(url, JsonConvert.SerializeObject(body));
var result = Time.Wait(client.PostAsync(url, content));
var str= Time.Wait(result.Content.ReadAsStringAsync());
var str = Time.Wait(result.Content.ReadAsStringAsync());
Log(url, str);
return str;
});
}, $"HTTP-POST-JSON: {route}");
}
public string HttpPostStream(string route, Stream stream)
@ -81,7 +80,7 @@ namespace DistTestCore
var str =Time.Wait(response.Content.ReadAsStringAsync());
Log(url, str);
return str;
});
}, $"HTTP-POST-STREAM: {route}");
}
public Stream HttpGetStream(string route)
@ -92,43 +91,10 @@ namespace DistTestCore
var url = GetUrl() + route;
Log(url, "~ STREAM ~");
return Time.Wait(client.GetStreamAsync(url));
});
}, $"HTTP-GET-STREAM: {route}");
}
private string GetUrl()
{
return $"http://{ip}:{port}{baseUrl}";
}
private void Log(string url, string message)
{
log.Debug($"({url}) = '{message}'", 3);
}
private T Retry<T>(Func<T> operation)
{
var retryCounter = 0;
while (true)
{
try
{
return operation();
}
catch (Exception exception)
{
timeSet.HttpCallRetryDelay();
retryCounter++;
if (retryCounter > timeSet.HttpCallRetryCount())
{
Assert.Fail(exception.ToString());
throw;
}
}
}
}
private static T TryJsonDeserialize<T>(string json)
public T TryJsonDeserialize<T>(string json)
{
try
{
@ -137,15 +103,38 @@ namespace DistTestCore
catch (Exception exception)
{
var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
Assert.Fail(msg);
throw new InvalidOperationException(msg, exception);
}
}
private string GetUrl()
{
return $"{address.Host}:{address.Port}{baseUrl}";
}
private void Log(string url, string message)
{
log.Debug($"({url}) = '{message}'", 3);
}
private T Retry<T>(Func<T> operation, string description)
{
return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
}
private HttpClient GetClient()
{
if (timeoutOverride.HasValue)
{
return GetClient(timeoutOverride.Value);
}
return GetClient(timeSet.HttpCallTimeout());
}
private HttpClient GetClient(TimeSpan timeout)
{
var client = new HttpClient();
client.Timeout = timeSet.HttpCallTimeout();
client.Timeout = timeout;
return client;
}
}

2
DistTestCore/Marketplace/CodexContractsContainerRecipe.cs
View File

@ -7,7 +7,7 @@ namespace DistTestCore.Marketplace
#if Arm64
public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
#else
public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint";
public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint2";
#endif
public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";

31
DistTestCore/Marketplace/CodexContractsStarter.cs
View File

@ -5,7 +5,6 @@ namespace DistTestCore.Marketplace
{
public class CodexContractsStarter : BaseStarter
{
private const string readyString = "Done! Sleeping indefinitely...";
public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
@ -14,7 +13,7 @@ namespace DistTestCore.Marketplace
public MarketplaceInfo Start(GethBootstrapNodeInfo bootstrapNode)
{
LogStart("Deploying Codex contracts...");
LogStart("Deploying Codex Marketplace...");
var workflow = workflowCreator.CreateWorkflow();
var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
@ -25,32 +24,33 @@ namespace DistTestCore.Marketplace
WaitUntil(() =>
{
var logHandler = new ContractsReadyLogHandler(readyString);
var logHandler = new ContractsReadyLogHandler(Debug);
workflow.DownloadContainerLog(container, logHandler);
return logHandler.Found;
});
Log("Contracts deployed. Extracting addresses...");
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
var marketplaceAddress = extractor.ExtractMarketplaceAddress();
var abi = extractor.ExtractMarketplaceAbi();
var interaction = bootstrapNode.StartInteraction(lifecycle.Log);
var interaction = bootstrapNode.StartInteraction(lifecycle);
var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
LogEnd("Contracts deployed.");
LogEnd("Extract completed. Marketplace deployed.");
return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress);
}
private void WaitUntil(Func<bool> predicate)
{
Time.WaitUntil(predicate, TimeSpan.FromMinutes(2), TimeSpan.FromSeconds(1));
Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
}
private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
{
var startupConfig = new StartupConfig();
var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
startupConfig.Add(contractsConfig);
return startupConfig;
}
@ -72,18 +72,27 @@ namespace DistTestCore.Marketplace
public class ContractsReadyLogHandler : LogHandler
{
private readonly string targetString;
// Log should contain 'Compiled 15 Solidity files successfully' at some point.
private const string RequiredCompiledString = "Solidity files successfully";
// When script is done, it prints the ready-string.
private const string ReadyString = "Done! Sleeping indefinitely...";
private readonly Action<string> debug;
public ContractsReadyLogHandler(string targetString)
public ContractsReadyLogHandler(Action<string> debug)
{
this.targetString = targetString;
this.debug = debug;
debug($"Looking for '{RequiredCompiledString}' and '{ReadyString}' in container logs...");
}
public bool SeenCompileString { get; private set; }
public bool Found { get; private set; }
protected override void ProcessLine(string line)
{
if (line.Contains(targetString)) Found = true;
debug(line);
if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
if (SeenCompileString && line.Contains(ReadyString)) Found = true;
}
}
}

40
DistTestCore/Marketplace/ContainerInfoExtractor.cs
View File

@ -56,30 +56,6 @@ namespace DistTestCore.Marketplace
return marketplaceAbi;
}
private string Retry(Func<string> fetch)
{
var result = string.Empty;
Time.WaitUntil(() =>
{
result = Catch(fetch);
return !string.IsNullOrEmpty(result);
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
return result;
}
private string Catch(Func<string> fetch)
{
try
{
return fetch();
}
catch
{
return string.Empty;
}
}
private string FetchAccountsCsv()
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
@ -103,7 +79,7 @@ namespace DistTestCore.Marketplace
private string FetchPubKey()
{
var enodeFinder = new PubKeyFinder();
var enodeFinder = new PubKeyFinder(s => log.Debug(s));
workflow.DownloadContainerLog(container, enodeFinder);
return enodeFinder.GetPubKey();
}
@ -116,21 +92,35 @@ namespace DistTestCore.Marketplace
var privateKey = tokens[1];
return new GethAccount(account, privateKey);
}
private static string Retry(Func<string> fetch)
{
return Time.Retry(fetch, nameof(ContainerInfoExtractor));
}
}
public class PubKeyFinder : LogHandler, ILogHandler
{
private const string openTag = "self=enode://";
private const string openTagQuote = "self=\"enode://";
private readonly Action<string> debug;
private string pubKey = string.Empty;
public PubKeyFinder(Action<string> debug)
{
this.debug = debug;
debug($"Looking for '{openTag}' in container logs...");
}
public string GetPubKey()
{
if (string.IsNullOrEmpty(pubKey)) throw new Exception("Not found yet exception.");
return pubKey;
}
protected override void ProcessLine(string line)
{
debug(line);
if (line.Contains(openTag))
{
ExtractPubKey(openTag, line);

8
DistTestCore/Marketplace/GethBootstrapNodeInfo.cs
View File

@ -1,5 +1,4 @@
using KubernetesWorkflow;
using Logging;
using NethereumWorkflow;
namespace DistTestCore.Marketplace
@ -21,13 +20,12 @@ namespace DistTestCore.Marketplace
public string PubKey { get; }
public Port DiscoveryPort { get; }
public NethereumInteraction StartInteraction(BaseLog log)
public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
{
var ip = RunningContainers.RunningPod.Cluster.IP;
var port = RunningContainers.Containers[0].ServicePorts[0].Number;
var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]);
var account = Account;
var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey);
var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey);
return creator.CreateWorkflow();
}
}

8
DistTestCore/Marketplace/GethCompanionNodeInfo.cs
View File

@ -1,5 +1,4 @@
using KubernetesWorkflow;
using Logging;
using NethereumWorkflow;
namespace DistTestCore.Marketplace
@ -15,13 +14,12 @@ namespace DistTestCore.Marketplace
public RunningContainer RunningContainer { get; }
public GethAccount[] Accounts { get; }
public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
public NethereumInteraction StartInteraction(TestLifecycle lifecycle, GethAccount account)
{
var ip = RunningContainer.Pod.Cluster.IP;
var port = RunningContainer.ServicePorts[0].Number;
var address = lifecycle.Configuration.GetAddress(RunningContainer);
var privateKey = account.PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, privateKey);
var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, privateKey);
return creator.CreateWorkflow();
}
}

2
DistTestCore/Marketplace/GethCompanionNodeStarter.cs
View File

@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
{
Time.WaitUntil(() =>
{
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
var interaction = node.StartInteraction(lifecycle, node.Accounts.First());
return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
}

2
DistTestCore/Marketplace/GethContainerRecipe.cs
View File

@ -58,7 +58,7 @@ namespace DistTestCore.Marketplace
var httpPort = AddExposedPort(tag: HttpPortTag);
var bootPubKey = config.BootstrapNode.PubKey;
var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
var bootPort = config.BootstrapNode.DiscoveryPort.Number;
var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";

14
DistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -1,5 +1,5 @@
using DistTestCore.Codex;
using Logging;
using DistTestCore.Helpers;
using NUnit.Framework;
using NUnit.Framework.Constraints;
using System.Numerics;
@ -17,14 +17,14 @@ namespace DistTestCore.Marketplace
public class MarketplaceAccess : IMarketplaceAccess
{
private readonly TestLog log;
private readonly TestLifecycle lifecycle;
private readonly MarketplaceNetwork marketplaceNetwork;
private readonly GethAccount account;
private readonly CodexAccess codexAccess;
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
public MarketplaceAccess(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
{
this.log = log;
this.lifecycle = lifecycle;
this.marketplaceNetwork = marketplaceNetwork;
this.account = account;
this.codexAccess = codexAccess;
@ -98,12 +98,12 @@ namespace DistTestCore.Marketplace
public void AssertThatBalance(IResolveConstraint constraint, string message = "")
{
Assert.That(GetBalance(), constraint, message);
AssertHelpers.RetryAssert(constraint, GetBalance, message);
}
public TestToken GetBalance()
{
var interaction = marketplaceNetwork.StartInteraction(log);
var interaction = marketplaceNetwork.StartInteraction(lifecycle);
var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
var balance = new TestToken(amount);
@ -114,7 +114,7 @@ namespace DistTestCore.Marketplace
private void Log(string msg)
{
log.Log($"{codexAccess.Container.Name} {msg}");
lifecycle.Log.Log($"{codexAccess.Container.Name} {msg}");
}
}

8
DistTestCore/Marketplace/MarketplaceAccessFactory.cs
View File

@ -18,19 +18,19 @@ namespace DistTestCore.Marketplace
public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
{
private readonly TestLog log;
private readonly TestLifecycle lifecycle;
private readonly MarketplaceNetwork marketplaceNetwork;
public GethMarketplaceAccessFactory(TestLog log, MarketplaceNetwork marketplaceNetwork)
public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork)
{
this.log = log;
this.lifecycle = lifecycle;
this.marketplaceNetwork = marketplaceNetwork;
}
public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
{
var companionNode = GetGethCompanionNode(access);
return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access);
}
private GethAccount GetGethCompanionNode(CodexAccess access)

7
DistTestCore/Marketplace/MarketplaceNetwork.cs
View File

@ -1,5 +1,4 @@
using Logging;
using NethereumWorkflow;
using NethereumWorkflow;
namespace DistTestCore.Marketplace
{
@ -14,9 +13,9 @@ namespace DistTestCore.Marketplace
public GethBootstrapNodeInfo Bootstrap { get; }
public MarketplaceInfo Marketplace { get; }
public NethereumInteraction StartInteraction(BaseLog log)
public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
{
return Bootstrap.StartInteraction(log);
return Bootstrap.StartInteraction(lifecycle);
}
}
}

15
DistTestCore/Metrics/MetricsAccess.cs
View File

@ -1,4 +1,5 @@
using KubernetesWorkflow;
using DistTestCore.Helpers;
using KubernetesWorkflow;
using Logging;
using NUnit.Framework;
using NUnit.Framework.Constraints;
@ -28,12 +29,14 @@ namespace DistTestCore.Metrics
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
{
var metricSet = GetMetricWithTimeout(metricName);
var metricValue = metricSet.Values[0].Value;
AssertHelpers.RetryAssert(constraint, () =>
{
var metricSet = GetMetricWithTimeout(metricName);
var metricValue = metricSet.Values[0].Value;
log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
Assert.That(metricValue, constraint, message);
log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
return metricValue;
}, message);
}
public Metrics? GetAllMetrics()

2
DistTestCore/Metrics/MetricsAccessFactory.cs
View File

@ -28,7 +28,7 @@ namespace DistTestCore.Metrics
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{
var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
var query = new MetricsQuery(lifecycle, prometheusContainer);
return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
}
}

14
DistTestCore/Metrics/MetricsQuery.cs
View File

@ -1,6 +1,5 @@
using DistTestCore.Codex;
using KubernetesWorkflow;
using Logging;
using System.Globalization;
namespace DistTestCore.Metrics
@ -9,15 +8,16 @@ namespace DistTestCore.Metrics
{
private readonly Http http;
public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers)
{
RunningContainers = runningContainers;
var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]);
http = new Http(
log,
timeSet,
runningContainers.RunningPod.Cluster.IP,
runningContainers.Containers[0].ServicePorts[0].Number,
lifecycle.Log,
lifecycle.TimeSet,
address,
"api/v1");
}
@ -119,7 +119,7 @@ namespace DistTestCore.Metrics
private string GetInstanceNameForNode(RunningContainer node)
{
var ip = node.Pod.Ip;
var ip = node.Pod.PodInfo.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
return $"{ip}:{port}";
}

17
DistTestCore/OnlineCodexNode.cs
View File

@ -10,6 +10,8 @@ namespace DistTestCore
{
string GetName();
CodexDebugResponse GetDebugInfo();
CodexDebugPeerResponse GetDebugPeer(string peerId);
CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node);
@ -47,10 +49,21 @@ namespace DistTestCore
public CodexDebugResponse GetDebugInfo()
{
var debugInfo = CodexAccess.GetDebugInfo();
Log($"Got DebugInfo with id: '{debugInfo.id}'.");
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
return debugInfo;
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return CodexAccess.GetDebugPeer(peerId);
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
return CodexAccess.GetDebugPeer(peerId, timeout);
}
public ContentId UploadFile(TestFile file)
{
Log($"Uploading file of size {file.GetFileSize()}...");
@ -111,7 +124,7 @@ namespace DistTestCore
// The peer we want to connect is in a different pod.
// We must replace the default IP with the pod IP in the multiAddress.
return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
}
private void DownloadToFile(string contentId, TestFile file)

2
DistTestCore/PrometheusStarter.cs
View File

@ -44,7 +44,7 @@ namespace DistTestCore
foreach (var node in nodes)
{
var ip = node.Pod.Ip;
var ip = node.Pod.PodInfo.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
config += $" - '{ip}:{port}'\n";
}

2
DistTestCore/TestLifecycle.cs
View File

@ -13,6 +13,7 @@ namespace DistTestCore
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
{
Log = log;
Configuration = configuration;
TimeSet = timeSet;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
@ -24,6 +25,7 @@ namespace DistTestCore
}
public TestLog Log { get; }
public Configuration Configuration { get; }
public ITimeSet TimeSet { get; }
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }

21
DistTestCore/Timing.cs
View File

@ -1,5 +1,4 @@
using NUnit.Framework;
using Utils;
namespace DistTestCore
{
@ -11,8 +10,8 @@ namespace DistTestCore
public interface ITimeSet
{
TimeSpan HttpCallTimeout();
int HttpCallRetryCount();
void HttpCallRetryDelay();
TimeSpan HttpCallRetryTimeout();
TimeSpan HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout();
@ -25,14 +24,14 @@ namespace DistTestCore
return TimeSpan.FromSeconds(10);
}
public int HttpCallRetryCount()
public TimeSpan HttpCallRetryTimeout()
{
return 5;
return TimeSpan.FromMinutes(1);
}
public void HttpCallRetryDelay()
public TimeSpan HttpCallRetryDelay()
{
Time.Sleep(TimeSpan.FromSeconds(3));
return TimeSpan.FromSeconds(3);
}
public TimeSpan WaitForK8sServiceDelay()
@ -58,14 +57,14 @@ namespace DistTestCore
return TimeSpan.FromHours(2);
}
public int HttpCallRetryCount()
public TimeSpan HttpCallRetryTimeout()
{
return 2;
return TimeSpan.FromHours(5);
}
public void HttpCallRetryDelay()
public TimeSpan HttpCallRetryDelay()
{
Time.Sleep(TimeSpan.FromMinutes(5));
return TimeSpan.FromMinutes(5);
}
public TimeSpan WaitForK8sServiceDelay()

40
KubernetesWorkflow/ApplicationLifecycle.cs
View File

@ -1,40 +0,0 @@
using Utils;
namespace KubernetesWorkflow
{
public class ApplicationLifecycle
{
private static object instanceLock = new object();
private static ApplicationLifecycle? instance;
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource namespaceNumberSource = new NumberSource(0);
private ApplicationLifecycle()
{
}
public static ApplicationLifecycle Instance
{
// I know singletons are quite evil. But we need to be sure this object is created only once
// and persists for the entire application lifecycle.
get
{
lock (instanceLock)
{
if (instance == null) instance = new ApplicationLifecycle();
return instance;
}
}
}
public NumberSource GetServiceNumberSource()
{
return servicePortNumberSource;
}
public string GetTestNamespace()
{
return namespaceNumberSource.GetNextNumber().ToString("D5");
}
}
}

2
KubernetesWorkflow/CommandRunner.cs
View File

@ -28,7 +28,7 @@ namespace KubernetesWorkflow
var input = new[] { command }.Concat(arguments).ToArray();
Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
}
public string GetStdOut()

16
KubernetesWorkflow/Configuration.cs
View File

@ -2,31 +2,17 @@
{
public class Configuration
{
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
{
K8sNamespacePrefix = k8sNamespacePrefix;
KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout;
RetryDelay = retryDelay;
LocationMap = locationMap;
}
public string K8sNamespacePrefix { get; }
public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; }
public ConfigurationLocationEntry[] LocationMap { get; }
}
public class ConfigurationLocationEntry
{
public ConfigurationLocationEntry(Location location, string workerName)
{
Location = location;
WorkerName = workerName;
}
public Location Location { get; }
public string WorkerName { get; }
}
}

48
KubernetesWorkflow/K8sCluster.cs
View File

@ -10,19 +10,28 @@ namespace KubernetesWorkflow
}
public Configuration Configuration { get; }
public string IP { get; private set; } = string.Empty;
public string HostAddress { get; private set; } = string.Empty;
public K8sNodeLabel[] AvailableK8sNodes { get; set; } = new K8sNodeLabel[0];
public KubernetesClientConfiguration GetK8sClientConfig()
{
var config = GetConfig();
UpdateIp(config);
UpdateHostAddress(config);
return config;
}
public string GetNodeLabelForLocation(Location location)
public K8sNodeLabel? GetNodeLabelForLocation(Location location)
{
if (location == Location.Unspecified) return string.Empty;
return Configuration.LocationMap.Single(l => l.Location == location).WorkerName;
switch (location)
{
case Location.One:
return K8sNodeIfAvailable(0);
case Location.Two:
return K8sNodeIfAvailable(1);
case Location.Three:
return K8sNodeIfAvailable(2);
}
return null;
}
public TimeSpan K8sOperationTimeout()
@ -47,10 +56,35 @@ namespace KubernetesWorkflow
}
}
private void UpdateIp(KubernetesClientConfiguration config)
private void UpdateHostAddress(KubernetesClientConfiguration config)
{
var host = config.Host.Replace("https://", "");
IP = host.Substring(0, host.IndexOf(':'));
if (host.Contains(":"))
{
HostAddress = "http://" + host.Substring(0, host.IndexOf(':'));
}
else
{
HostAddress = config.Host;
}
}
private K8sNodeLabel? K8sNodeIfAvailable(int index)
{
if (AvailableK8sNodes.Length <= index) return null;
return AvailableK8sNodes[index];
}
}
public class K8sNodeLabel
{
public K8sNodeLabel(string key, string value)
{
Key = key;
Value = value;
}
public string Key { get; }
public string Value { get; }
}
}

196
KubernetesWorkflow/K8sController.cs
View File

@ -22,7 +22,6 @@ namespace KubernetesWorkflow
client = new K8sClient(cluster.GetK8sClientConfig());
K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
log.Debug($"Test namespace: '{K8sTestNamespace}'");
}
public void Dispose()
@ -33,13 +32,14 @@ namespace KubernetesWorkflow
public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location)
{
log.Debug();
DiscoverK8sNodes();
EnsureTestNamespace();
var deploymentName = CreateDeployment(containerRecipes, location);
var (serviceName, servicePortsMap) = CreateService(containerRecipes);
var (podName, podIp) = FetchNewPod();
var podInfo = FetchNewPod();
return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap);
return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
}
public void Stop(RunningPod pod)
@ -48,22 +48,27 @@ namespace KubernetesWorkflow
if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
DeleteDeployment(pod.DeploymentName);
WaitUntilDeploymentOffline(pod.DeploymentName);
WaitUntilPodOffline(pod.Name);
WaitUntilPodOffline(pod.PodInfo.Name);
}
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{
log.Debug();
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
logHandler.Log(stream);
}
public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
{
log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})";
log.Debug(cmdAndArgs);
var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
runner.Run();
return runner.GetStdOut();
var result = runner.GetStdOut();
log.Debug($"{cmdAndArgs} = '{result}'");
return result;
}
public void DeleteAllResources()
@ -102,6 +107,42 @@ namespace KubernetesWorkflow
}
}
#region Discover K8s Nodes
private void DiscoverK8sNodes()
{
if (cluster.AvailableK8sNodes == null || !cluster.AvailableK8sNodes.Any())
{
cluster.AvailableK8sNodes = GetAvailableK8sNodes();
if (cluster.AvailableK8sNodes.Length < 3)
{
log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes.Select(p => $"{p.Key}={p.Value}"))}'.");
}
}
}
private K8sNodeLabel[] GetAvailableK8sNodes()
{
var nodes = client.Run(c => c.ListNode());
var optionals = nodes.Items.Select(i => CreateNodeLabel(i));
return optionals.Where(n => n != null).Select(n => n!).ToArray();
}
private K8sNodeLabel? CreateNodeLabel(V1Node i)
{
var keys = i.Metadata.Labels.Keys;
var hostnameKey = keys.SingleOrDefault(k => k.ToLowerInvariant().Contains("hostname"));
if (hostnameKey != null)
{
var hostnameValue = i.Metadata.Labels[hostnameKey];
return new K8sNodeLabel(hostnameKey, hostnameValue);
}
return null;
}
#endregion
#region Namespace management
private string K8sTestNamespace { get; }
@ -148,10 +189,7 @@ namespace KubernetesWorkflow
},
Spec = new V1NetworkPolicySpec
{
PodSelector = new V1LabelSelector
{
MatchLabels = GetSelector()
},
PodSelector = new V1LabelSelector {},
PolicyTypes = new[]
{
"Ingress",
@ -159,6 +197,16 @@ namespace KubernetesWorkflow
},
Ingress = new List<V1NetworkPolicyIngressRule>
{
new V1NetworkPolicyIngressRule
{
FromProperty = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
PodSelector = new V1LabelSelector {}
}
}
},
new V1NetworkPolicyIngressRule
{
FromProperty = new List<V1NetworkPolicyPeer>
@ -167,7 +215,7 @@ namespace KubernetesWorkflow
{
NamespaceSelector = new V1LabelSelector
{
MatchLabels = GetMyNamespaceSelector()
MatchLabels = GetRunnerNamespaceSelector()
}
}
}
@ -175,6 +223,16 @@ namespace KubernetesWorkflow
},
Egress = new List<V1NetworkPolicyEgressRule>
{
new V1NetworkPolicyEgressRule
{
To = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
PodSelector = new V1LabelSelector {}
}
}
},
new V1NetworkPolicyEgressRule
{
To = new List<V1NetworkPolicyPeer>
@ -183,11 +241,62 @@ namespace KubernetesWorkflow
{
NamespaceSelector = new V1LabelSelector
{
MatchLabels = GetMyNamespaceSelector()
MatchLabels = new Dictionary<string, string> { { "kubernetes.io/metadata.name", "kube-system" } }
}
},
new V1NetworkPolicyPeer
{
PodSelector = new V1LabelSelector
{
MatchLabels = new Dictionary<string, string> { { "k8s-app", "kube-dns" } }
}
}
},
Ports = new List<V1NetworkPolicyPort>
{
new V1NetworkPolicyPort
{
Port = new IntstrIntOrString
{
Value = "53"
},
Protocol = "UDP"
}
}
},
new V1NetworkPolicyEgressRule
{
To = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
IpBlock = new V1IPBlock
{
Cidr = "0.0.0.0/0"
}
}
},
Ports = new List<V1NetworkPolicyPort>
{
new V1NetworkPolicyPort
{
Port = new IntstrIntOrString
{
Value = "80"
},
Protocol = "TCP"
},
new V1NetworkPolicyPort
{
Port = new IntstrIntOrString
{
Value = "443"
},
Protocol = "TCP"
}
}
}
}
}
};
@ -242,11 +351,12 @@ namespace KubernetesWorkflow
private IDictionary<string, string> CreateNodeSelector(Location location)
{
if (location == Location.Unspecified) return new Dictionary<string, string>();
var nodeLabel = cluster.GetNodeLabelForLocation(location);
if (nodeLabel == null) return new Dictionary<string, string>();
return new Dictionary<string, string>
{
{ "codex-test-location", cluster.GetNodeLabelForLocation(location) }
{ nodeLabel.Key, nodeLabel.Value }
};
}
@ -255,9 +365,9 @@ namespace KubernetesWorkflow
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
}
private IDictionary<string, string> GetMyNamespaceSelector()
private IDictionary<string, string> GetRunnerNamespaceSelector()
{
return new Dictionary<string, string> { { "name", "thatisincorrect" } };
return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
}
private V1ObjectMeta CreateDeploymentMetadata()
@ -329,11 +439,11 @@ namespace KubernetesWorkflow
{
var result = new Dictionary<ContainerRecipe, Port[]>();
var ports = CreateServicePorts(result, containerRecipes);
var ports = CreateServicePorts(containerRecipes);
if (!ports.Any())
{
// None of these container-recipes wish to expose anything via a serice port.
// None of these container-recipes wish to expose anything via a service port.
// So, we don't have to create a service.
return (string.Empty, result);
}
@ -352,9 +462,40 @@ namespace KubernetesWorkflow
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
return (serviceSpec.Metadata.Name, result);
}
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
{
// For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
foreach (var r in containerRecipes)
{
if (r.ExposedPorts.Any())
{
var firstExposedPort = r.ExposedPorts.First();
var portName = GetNameForPort(r, firstExposedPort);
var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
if (matchingServicePorts.Any())
{
// These service ports belongs to this recipe.
var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
result.Add(r, ports);
}
}
}
}
private Port? MapNodePortIfAble(V1ServicePort p, string tag)
{
if (p.NodePort == null) return null;
return new Port(p.NodePort.Value, tag);
}
private void DeleteService(string serviceName)
{
client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
@ -369,36 +510,30 @@ namespace KubernetesWorkflow
};
}
private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe[] recipes)
private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes)
{
var result = new List<V1ServicePort>();
foreach (var recipe in recipes)
{
result.AddRange(CreateServicePorts(servicePorts, recipe));
result.AddRange(CreateServicePorts(recipe));
}
return result;
}
private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe recipe)
private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe)
{
var result = new List<V1ServicePort>();
var usedPorts = new List<Port>();
foreach (var port in recipe.ExposedPorts)
{
var servicePort = workflowNumberSource.GetServicePort();
usedPorts.Add(new Port(servicePort, ""));
result.Add(new V1ServicePort
{
Name = GetNameForPort(recipe, port),
Protocol = "TCP",
Port = port.Number,
TargetPort = GetNameForPort(recipe, port),
NodePort = servicePort
});
}
servicePorts.Add(recipe, usedPorts.ToArray());
return result;
}
@ -465,7 +600,7 @@ namespace KubernetesWorkflow
#endregion
private (string, string) FetchNewPod()
private PodInfo FetchNewPod()
{
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
@ -475,12 +610,13 @@ namespace KubernetesWorkflow
var newPod = newPods.Single();
var name = newPod.Name();
var ip = newPod.Status.PodIP;
var k8sNodeName = newPod.Spec.NodeName;
if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
knownPods.Add(name);
return (name, ip);
return new PodInfo(name, ip, k8sNodeName);
}
}
}

5
KubernetesWorkflow/Location.cs
View File

@ -3,7 +3,8 @@
public enum Location
{
Unspecified,
BensLaptop,
BensOldGamingMachine
One,
Two,
Three,
}
}

18
KubernetesWorkflow/RunningContainers.cs
View File

@ -21,18 +21,22 @@
public class RunningContainer
{
public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig)
public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, RunningContainerAddress clusterExternalAddress, RunningContainerAddress clusterInternalAddress)
{
Pod = pod;
Recipe = recipe;
ServicePorts = servicePorts;
Name = GetContainerName(recipe, startupConfig);
ClusterExternalAddress = clusterExternalAddress;
ClusterInternalAddress = clusterInternalAddress;
}
public string Name { get; }
public RunningPod Pod { get; }
public ContainerRecipe Recipe { get; }
public Port[] ServicePorts { get; }
public RunningContainerAddress ClusterExternalAddress { get; }
public RunningContainerAddress ClusterInternalAddress { get; }
private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
{
@ -46,4 +50,16 @@
}
}
}
public class RunningContainerAddress
{
public RunningContainerAddress(string host, int port)
{
Host = host;
Port = port;
}
public string Host { get; }
public int Port { get; }
}
}

23
KubernetesWorkflow/RunningPod.cs
View File

@ -4,25 +4,38 @@
{
private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
{
Cluster = cluster;
Name = name;
Ip = ip;
PodInfo = podInfo;
DeploymentName = deploymentName;
ServiceName = serviceName;
this.servicePortMap = servicePortMap;
}
public K8sCluster Cluster { get; }
public string Name { get; }
public string Ip { get; }
public PodInfo PodInfo { get; }
internal string DeploymentName { get; }
internal string ServiceName { get; }
public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
{
if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
return servicePortMap[containerRecipe];
}
}
public class PodInfo
{
public PodInfo(string podName, string podIp, string k8sNodeName)
{
Name = podName;
Ip = podIp;
K8SNodeName = k8sNodeName;
}
public string Name { get; }
public string Ip { get; }
public string K8SNodeName { get; }
}
}

35
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -80,10 +80,43 @@ namespace KubernetesWorkflow
var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
return new RunningContainer(runningPod, r, servicePorts, startupConfig);
return new RunningContainer(runningPod, r, servicePorts, startupConfig,
GetContainerExternalAddress(runningPod, servicePorts),
GetContainerInternalAddress(r));
}).ToArray();
}
private RunningContainerAddress GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
{
return new RunningContainerAddress(
pod.Cluster.HostAddress,
GetServicePort(servicePorts));
}
private RunningContainerAddress GetContainerInternalAddress(ContainerRecipe recipe)
{
var serviceName = "service-" + numberSource.WorkflowNumber;
var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace;
var port = GetInternalPort(recipe);
return new RunningContainerAddress(
$"http://{serviceName}.{namespaceName}.svc.cluster.local",
port);
}
private static int GetServicePort(Port[] servicePorts)
{
if (servicePorts.Any()) return servicePorts.First().Number;
return 0;
}
private static int GetInternalPort(ContainerRecipe recipe)
{
if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number;
return 0;
}
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
{
log.Debug();

3
KubernetesWorkflow/WorkflowCreator.cs
View File

@ -16,13 +16,12 @@ namespace KubernetesWorkflow
{
cluster = new K8sCluster(configuration);
this.log = log;
testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
testNamespace = Guid.NewGuid().ToString().ToLowerInvariant();
}
public StartupWorkflow CreateWorkflow()
{
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
ApplicationLifecycle.Instance.GetServiceNumberSource(),
containerNumberSource);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);

9
KubernetesWorkflow/WorkflowNumberSource.cs
View File

@ -4,13 +4,11 @@ namespace KubernetesWorkflow
{
public class WorkflowNumberSource
{
private readonly NumberSource servicePortNumberSource;
private readonly NumberSource containerNumberSource;
public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource)
public WorkflowNumberSource(int workflowNumber, NumberSource containerNumberSource)
{
WorkflowNumber = workflowNumber;
this.servicePortNumberSource = servicePortNumberSource;
this.containerNumberSource = containerNumberSource;
}
@ -20,10 +18,5 @@ namespace KubernetesWorkflow
{
return containerNumberSource.GetNextNumber();
}
public int GetServicePort()
{
return servicePortNumberSource.GetNextNumber();
}
}
}

1
Logging/BaseLog.cs
View File

@ -54,6 +54,7 @@ namespace Logging
public void AddStringReplace(string from, string to)
{
if (string.IsNullOrWhiteSpace(from)) return;
replacements.Add(new BaseLogStringReplacement(from, to));
}

4
Tests/BasicTests/DownloadTests.cs → LongTests/BasicTests/DownloadTests.cs
View File

@ -1,7 +1,7 @@
using DistTestCore;
using NUnit.Framework;
namespace Tests.ParallelTests
namespace TestsLong.BasicTests
{
[TestFixture]
public class DownloadTests : DistTest
@ -23,7 +23,7 @@ namespace Tests.ParallelTests
var testFile = GenerateTestFile(filesizeMb.MB());
var contentId = host.UploadFile(testFile);
var list = new List<Task<TestFile?>>();
foreach (var node in group)
{
list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));

2
LongTests/BasicTests/LargeFileTests.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace TestsLong.BasicTests
@ -11,7 +10,6 @@ namespace TestsLong.BasicTests
public void OneClientLargeFileTest()
{
var primary = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Warn)
.WithStorageQuota(20.GB()));
var testFile = GenerateTestFile(10.GB());

2
LongTests/BasicTests/TestInfraTests.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace TestsLong.BasicTests
@ -32,7 +31,6 @@ namespace TestsLong.BasicTests
public void DownloadConsistencyTest()
{
var primary = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithStorageQuota(2.MB()));
var testFile = GenerateTestFile(1.MB());

2
Tests/BasicTests/UploadTests.cs → LongTests/BasicTests/UploadTests.cs
View File

@ -1,7 +1,7 @@
using DistTestCore;
using NUnit.Framework;
namespace Tests.ParallelTests
namespace TestsLong.BasicTests
{
[TestFixture]
public class UploadTests : DistTest

4
Nethereum/NethereumInteraction.cs
View File

@ -131,7 +131,7 @@ namespace NethereumWorkflow
public class MintTokensFunction : FunctionMessage
{
[Parameter("address", "holder", 1)]
public string Holder { get; set; }
public string Holder { get; set; } = string.Empty;
[Parameter("uint256", "amount", 2)]
public BigInteger Amount { get; set; }
@ -141,6 +141,6 @@ namespace NethereumWorkflow
public class GetTokenBalanceFunction : FunctionMessage
{
[Parameter("address", "owner", 1)]
public string Owner { get; set; }
public string Owner { get; set; } = string.Empty;
}
}

2
Nethereum/NethereumInteractionCreator.cs
View File

@ -26,7 +26,7 @@ namespace NethereumWorkflow
private Web3 CreateWeb3()
{
var account = new Nethereum.Web3.Accounts.Account(privateKey);
return new Web3(account, $"http://{ip}:{port}");
return new Web3(account, $"{ip}:{port}");
}
}
}

23
README.md
View File

@ -1,11 +1,13 @@
# Distributed System Tests for Nim-Codex
Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducable environment.
Nim-Codex: https://github.com/status-im/nim-codex
Dotnet: v6.0
Kubernetes: v1.25.4
Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp
Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
Nim-Codex: https://github.com/codex-storage/nim-codex
Dotnet: v6.0
Kubernetes: v1.25.4
Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp
Nethereum: v4.14.0
## Tests
@ -15,6 +17,17 @@ Tests are devided into two assemblies: `/Tests` and `/LongTests`.
TODO: All tests will eventually be running as part of a dedicated CI pipeline and kubernetes cluster. Currently, we're developing these tests and the infra-code to support it by running the whole thing locally.
## Configuration
Test executing can be configured using the following environment variables.
| Variable | Description | Default |
|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
| KUBECONFIG | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available. | (null) |
| LOGPATH | Path (abs or rel) where log files will be saved. | "CodexTestLogs" |
| LOGDEBUG | When "true", enables additional test-runner debug log output. | "false" |
| DATAFILEPATH | Path (abs or rel) where temporary test data files will be saved. | "TestDataFiles" |
| LOGLEVEL | Codex log-level. (case-insensitive) | "Trace" |
| RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
## Test logs
Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.

5
Tests/BasicTests/ExampleTests.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
@ -11,7 +10,7 @@ namespace Tests.BasicTests
[Test]
public void CodexLogExample()
{
var primary = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace));
var primary = SetupCodexNode();
primary.UploadFile(GenerateTestFile(5.MB()));
@ -47,7 +46,6 @@ namespace Tests.BasicTests
var buyerInitialBalance = 1000.TestTokens();
var seller = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithStorageQuota(11.GB())
.EnableMarketplace(sellerInitialBalance));
@ -61,7 +59,6 @@ namespace Tests.BasicTests
var testFile = GenerateTestFile(10.MB());
var buyer = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithBootstrapNode(seller)
.EnableMarketplace(buyerInitialBalance));

85
Tests/BasicTests/PeerTests.cs
View File

@ -1,85 +0,0 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace Tests.BasicTests
{
[TestFixture]
public class PeerTests : DistTest
{
[Test]
public void TwoNodes()
{
var primary = SetupCodexBootstrapNode();
var secondary = SetupCodexNode(s => s.WithBootstrapNode(primary));
primary.ConnectToPeer(secondary); // TODO REMOVE THIS: This is required for the switchPeers to show up.
// This is required for the enginePeers to show up.
//var file = GenerateTestFile(10.MB());
//var contentId = primary.UploadFile(file);
//var file2 = secondary.DownloadContent(contentId);
//file.AssertIsEqual(file2);
AssertKnowEachother(primary, secondary);
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
public void VariableNodes(int number)
{
var bootstrap = SetupCodexBootstrapNode();
var nodes = SetupCodexNodes(number, s => s.WithBootstrapNode(bootstrap));
var file = GenerateTestFile(10.MB());
var contentId = nodes.First().UploadFile(file);
var file2 = nodes.Last().DownloadContent(contentId);
file.AssertIsEqual(file2);
// <TODO REMOVE THIS>
foreach (var node in nodes) bootstrap.ConnectToPeer(node);
for (var x = 0; x < number; x++)
{
for (var y = x + 1; y < number; y++)
{
nodes[x].ConnectToPeer(nodes[y]);
}
}
// </TODO REMOVE THIS>
foreach (var node in nodes) AssertKnowEachother(node, bootstrap);
for (var x = 0; x < number; x++)
{
for (var y = x + 1; y < number; y++)
{
AssertKnowEachother(nodes[x], nodes[y]);
}
}
}
private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b)
{
AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo());
}
private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b)
{
AssertKnows(a, b);
AssertKnows(b, a);
}
private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
{
var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
}
}
}

5
Tests/BasicTests/TwoClientTests.cs
View File

@ -28,11 +28,10 @@ namespace Tests.BasicTests
}
[Test]
[Ignore("Requires Location map to be configured for k8s cluster.")]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNode(s => s.At(Location.BensLaptop));
var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine));
var primary = SetupCodexNode(s => s.At(Location.One));
var secondary = SetupCodexNode(s => s.At(Location.Two));
PerformTwoClientTest(primary, secondary);
}

66
Tests/DurabilityTests/DurabilityTests.cs
View File

@ -1,66 +0,0 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
namespace Tests.DurabilityTests
{
[TestFixture]
public class DurabilityTests : DistTest
{
[Test]
public void BootstrapNodeDisappearsTest()
{
var bootstrapNode = SetupCodexBootstrapNode();
var group = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode));
var primary = group[0];
var secondary = group[1];
// There is 1 minute of time f or the nodes to connect to each other.
// (Should be easy, they're in the same pod.)
Time.Sleep(TimeSpan.FromMinutes(6));
bootstrapNode.BringOffline();
var file = GenerateTestFile(10.MB());
var contentId = primary.UploadFile(file);
var downloadedFile = secondary.DownloadContent(contentId);
file.AssertIsEqual(downloadedFile);
}
[Test]
public void DataRetentionTest()
{
var bootstrapNode = SetupCodexBootstrapNode(s => s.WithLogLevel(CodexLogLevel.Trace));
var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
var file = GenerateTestFile(10.MB());
// Both nodes in the start group have the file.
var content = startGroup[0].UploadFile(file);
DownloadAndAssert(content, file, startGroup[1]);
// Three nodes of the finish group have the file.
DownloadAndAssert(content, file, finishGroup[0]);
DownloadAndAssert(content, file, finishGroup[1]);
DownloadAndAssert(content, file, finishGroup[2]);
// The start group goes away.
startGroup.BringOffline();
// All nodes in the finish group can access the file.
foreach (var node in finishGroup)
{
DownloadAndAssert(content, file, node);
}
}
private void DownloadAndAssert(ContentId content, TestFile file, IOnlineCodexNode onlineCodexNode)
{
var downloaded = onlineCodexNode.DownloadContent(content);
file.AssertIsEqual(downloaded);
}
}
}

62
Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs Normal file
View File

@ -0,0 +1,62 @@
using DistTestCore;
using DistTestCore.Helpers;
using NUnit.Framework;
using Utils;
namespace Tests.PeerDiscoveryTests
{
[TestFixture]
public class LayeredDiscoveryTests : DistTest
{
[Test]
public void TwoLayersTest()
{
var root = SetupCodexNode();
var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root));
var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root));
var l2Target = SetupCodexNode(s => s.WithBootstrapNode(l1Node));
AssertAllNodesConnected();
}
[Test]
public void ThreeLayersTest()
{
var root = SetupCodexNode();
var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root));
var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root));
var l2Node = SetupCodexNode(s => s.WithBootstrapNode(l1Node));
var l3Target = SetupCodexNode(s => s.WithBootstrapNode(l2Node));
AssertAllNodesConnected();
}
[TestCase(3)]
[TestCase(5)]
[TestCase(10)]
[TestCase(20)]
[TestCase(50)]
public void NodeChainTest(int chainLength)
{
var node = SetupCodexNode();
for (var i = 1; i < chainLength; i++)
{
node = SetupCodexNode(s => s.WithBootstrapNode(node));
}
AssertAllNodesConnected();
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertAllNodesConnected();
}
}
private void AssertAllNodesConnected()
{
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
//PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
}
}
}

73
Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs Normal file
View File

@ -0,0 +1,73 @@
using DistTestCore;
using DistTestCore.Helpers;
using NUnit.Framework;
using Utils;
namespace Tests.PeerDiscoveryTests
{
[TestFixture]
public class PeerDiscoveryTests : AutoBootstrapDistTest
{
[Test]
public void CanReportUnknownPeerId()
{
var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK";
var node = SetupCodexNode();
var result = node.GetDebugPeer(unknownId);
Assert.That(result.IsPeerFound, Is.False);
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
public void VariableNodes(int number)
{
SetupCodexNodes(number);
AssertAllNodesConnected();
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
[TestCase(20)]
public void VariableNodesInPods(int number)
{
for (var i = 0; i < number; i++)
{
SetupCodexNode();
}
AssertAllNodesConnected();
}
[TestCase(3, 3)]
[TestCase(3, 5)]
[TestCase(3, 10)]
[TestCase(5, 10)]
[TestCase(3, 20)]
[TestCase(5, 20)]
public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
{
for (var i = 0; i < numberOfStages; i++)
{
SetupCodexNodes(numberOfNodes);
AssertAllNodesConnected();
}
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertAllNodesConnected();
}
}
private void AssertAllNodesConnected()
{
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
//PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
}
}
}

74
Utils/Time.cs
View File

@ -22,6 +22,11 @@
result += $"{d.Seconds} secs";
return result;
}
public static void WaitUntil(Func<bool> predicate)
{
WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
}
public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryTime)
{
@ -38,5 +43,74 @@
state = predicate();
}
}
public static void Retry(Action action, string description)
{
Retry(action, TimeSpan.FromMinutes(1), description);
}
public static T Retry<T>(Func<T> action, string description)
{
return Retry(action, TimeSpan.FromMinutes(1), description);
}
public static void Retry(Action action, TimeSpan timeout, string description)
{
Retry(action, timeout, TimeSpan.FromSeconds(1), description);
}
public static T Retry<T>(Func<T> action, TimeSpan timeout, string description)
{
return Retry(action, timeout, TimeSpan.FromSeconds(1), description);
}
public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;
var exceptions = new List<Exception>();
while (true)
{
if (DateTime.UtcNow - start > timeout)
{
throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
}
try
{
action();
return;
}
catch (Exception ex)
{
exceptions.Add(ex);
}
Sleep(retryTime);
}
}
public static T Retry<T>(Func<T> action, TimeSpan timeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;
var exceptions = new List<Exception>();
while (true)
{
if (DateTime.UtcNow - start > timeout)
{
throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
}
try
{
return action();
}
catch (Exception ex)
{
exceptions.Add(ex);
}
Sleep(retryTime);
}
}
}
}

4
codexnode-manifest.yml
View File

@ -24,7 +24,7 @@ spec:
spec:
containers:
- name: codex-node1
image: thatbenbierens/nim-codex:sha-b204837
image: codexstorage/nim-codex:sha-7b88ea0
ports:
- containerPort: 8080
name: api-1
@ -38,7 +38,7 @@ spec:
- name: LISTEN_ADDRS
value: "/ip4/0.0.0.0/tcp/8082"
- name: codex-node2
image: thatbenbierens/nim-codex:sha-b204837
image: codexstorage/nim-codex:sha-7b88ea0
ports:
- containerPort: 8083
name: api-2

48
docker/job.yaml Normal file
View File

@ -0,0 +1,48 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ${NAMEPREFIX}-${RUNID}
namespace: ${NAMESPACE}
labels:
name: ${NAMEPREFIX}-${RUNID}
run-id: ${RUNID}
spec:
backoffLimit: 0
template:
metadata:
name: ${NAMEPREFIX}
spec:
containers:
- name: ${NAMEPREFIX}-runner
image: codexstorage/cs-codex-dist-tests:sha-300b91e
env:
- name: RUNNERLOCATION
value: InternalToCluster
- name: KUBECONFIG
value: /opt/kubeconfig.yaml
- name: LOGPATH
value: /var/log/cs-codex-dist-tests
- name: NAMESPACE
value: ${NAMESPACE}
- name: BRANCH
value: ${BRANCH}
- name: SOURCE
value: ${SOURCE}
volumeMounts:
- name: kubeconfig
mountPath: /opt/kubeconfig.yaml
subPath: kubeconfig.yaml
- name: logs
mountPath: /var/log/cs-codex-dist-tests
# command:
# - "dotnet"
# - "test"
# - "Tests"
restartPolicy: Never
volumes:
- name: kubeconfig
secret:
secretName: cs-codex-dist-tests-app-kubeconfig
- name: logs
hostPath:
path: /var/log/cs-codex-dist-tests