Merge branch 'feature/parallel-tests'
This commit is contained in:
commit
3aba6d5082
|
@ -6,10 +6,12 @@ namespace DistTestCore.Codex
|
||||||
public class CodexAccess
|
public class CodexAccess
|
||||||
{
|
{
|
||||||
private readonly BaseLog log;
|
private readonly BaseLog log;
|
||||||
|
private readonly ITimeSet timeSet;
|
||||||
|
|
||||||
public CodexAccess(BaseLog log, RunningContainer runningContainer)
|
public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
|
this.timeSet = timeSet;
|
||||||
Container = runningContainer;
|
Container = runningContainer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -40,11 +42,29 @@ namespace DistTestCore.Codex
|
||||||
return Http().HttpPostJson($"storage/request/{contentId}", request);
|
return Http().HttpPostJson($"storage/request/{contentId}", request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void EnsureOnline()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var debugInfo = GetDebugInfo();
|
||||||
|
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
|
||||||
|
|
||||||
|
var nodePeerId = debugInfo.id;
|
||||||
|
var nodeName = Container.Name;
|
||||||
|
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
log.Error($"Failed to start codex node: {e}. Test infra failure.");
|
||||||
|
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private Http Http()
|
private Http Http()
|
||||||
{
|
{
|
||||||
var ip = Container.Pod.Cluster.IP;
|
var ip = Container.Pod.Cluster.IP;
|
||||||
var port = Container.ServicePorts[0].Number;
|
var port = Container.ServicePorts[0].Number;
|
||||||
return new Http(log, ip, port, baseUrl: "/api/codex/v1");
|
return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
|
||||||
}
|
}
|
||||||
|
|
||||||
public string ConnectToPeer(string peerId, string peerMultiAddress)
|
public string ConnectToPeer(string peerId, string peerMultiAddress)
|
||||||
|
|
|
@ -1,12 +1,17 @@
|
||||||
using DistTestCore.Marketplace;
|
using System.Runtime.InteropServices;
|
||||||
|
using DistTestCore.Marketplace;
|
||||||
using KubernetesWorkflow;
|
using KubernetesWorkflow;
|
||||||
|
|
||||||
namespace DistTestCore.Codex
|
namespace DistTestCore.Codex
|
||||||
{
|
{
|
||||||
public class CodexContainerRecipe : ContainerRecipeFactory
|
public class CodexContainerRecipe : ContainerRecipeFactory
|
||||||
{
|
{
|
||||||
|
#if Arm64
|
||||||
|
public const string DockerImage = "emizzle/nim-codex-arm64:sha-c7af585";
|
||||||
|
#else
|
||||||
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
|
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
|
||||||
public const string DockerImage = "thatbenbierens/codexlocal:latest";
|
public const string DockerImage = "thatbenbierens/codexlocal:latest";
|
||||||
|
#endif
|
||||||
public const string MetricsPortTag = "metrics_port";
|
public const string MetricsPortTag = "metrics_port";
|
||||||
|
|
||||||
protected override string Image => DockerImage;
|
protected override string Image => DockerImage;
|
||||||
|
|
|
@ -62,29 +62,15 @@ namespace DistTestCore
|
||||||
return $"group:[{Containers.Describe()}]";
|
return $"group:[{Containers.Describe()}]";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void EnsureOnline()
|
||||||
|
{
|
||||||
|
foreach (var node in Nodes) node.CodexAccess.EnsureOnline();
|
||||||
|
}
|
||||||
|
|
||||||
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
|
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
|
||||||
{
|
{
|
||||||
var access = new CodexAccess(lifecycle.Log, c);
|
var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
|
||||||
EnsureOnline(access);
|
|
||||||
return factory.CreateOnlineCodexNode(access, this);
|
return factory.CreateOnlineCodexNode(access, this);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void EnsureOnline(CodexAccess access)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var debugInfo = access.GetDebugInfo();
|
|
||||||
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
|
|
||||||
|
|
||||||
var nodePeerId = debugInfo.id;
|
|
||||||
var nodeName = access.Container.Name;
|
|
||||||
lifecycle.Log.AddStringReplace(nodePeerId, $"___{nodeName}___");
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
|
|
||||||
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -44,7 +44,7 @@ namespace DistTestCore
|
||||||
public void DeleteAllResources()
|
public void DeleteAllResources()
|
||||||
{
|
{
|
||||||
var workflow = CreateWorkflow();
|
var workflow = CreateWorkflow();
|
||||||
workflow.DeleteAllResources();
|
workflow.DeleteTestResources();
|
||||||
|
|
||||||
RunningGroups.Clear();
|
RunningGroups.Clear();
|
||||||
}
|
}
|
||||||
|
@ -74,6 +74,7 @@ namespace DistTestCore
|
||||||
{
|
{
|
||||||
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
|
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
|
||||||
RunningGroups.Add(group);
|
RunningGroups.Add(group);
|
||||||
|
group.EnsureOnline();
|
||||||
return group;
|
return group;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,13 +4,13 @@ namespace DistTestCore
|
||||||
{
|
{
|
||||||
public class Configuration
|
public class Configuration
|
||||||
{
|
{
|
||||||
public KubernetesWorkflow.Configuration GetK8sConfiguration()
|
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
|
||||||
{
|
{
|
||||||
return new KubernetesWorkflow.Configuration(
|
return new KubernetesWorkflow.Configuration(
|
||||||
k8sNamespace: "codex-test-ns",
|
k8sNamespacePrefix: "ct-",
|
||||||
kubeConfigFile: null,
|
kubeConfigFile: null,
|
||||||
operationTimeout: Timing.K8sOperationTimeout(),
|
operationTimeout: timeSet.K8sOperationTimeout(),
|
||||||
retryDelay: Timing.K8sServiceDelay(),
|
retryDelay: timeSet.WaitForK8sServiceDelay(),
|
||||||
locationMap: new[]
|
locationMap: new[]
|
||||||
{
|
{
|
||||||
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
|
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
|
||||||
|
|
|
@ -6,23 +6,25 @@ using KubernetesWorkflow;
|
||||||
using Logging;
|
using Logging;
|
||||||
using NUnit.Framework;
|
using NUnit.Framework;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using Utils;
|
|
||||||
|
|
||||||
namespace DistTestCore
|
namespace DistTestCore
|
||||||
{
|
{
|
||||||
[SetUpFixture]
|
[SetUpFixture]
|
||||||
|
[Parallelizable(ParallelScope.All)]
|
||||||
public abstract class DistTest
|
public abstract class DistTest
|
||||||
{
|
{
|
||||||
private readonly Configuration configuration = new Configuration();
|
private readonly Configuration configuration = new Configuration();
|
||||||
private readonly Assembly[] testAssemblies;
|
private readonly Assembly[] testAssemblies;
|
||||||
private FixtureLog fixtureLog = null!;
|
private readonly FixtureLog fixtureLog;
|
||||||
private TestLifecycle lifecycle = null!;
|
private readonly object lifecycleLock = new object();
|
||||||
private DateTime testStart = DateTime.MinValue;
|
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
|
||||||
|
|
||||||
public DistTest()
|
public DistTest()
|
||||||
{
|
{
|
||||||
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
|
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
|
||||||
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
|
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
|
||||||
|
|
||||||
|
fixtureLog = new FixtureLog(configuration.GetLogConfig());
|
||||||
}
|
}
|
||||||
|
|
||||||
[OneTimeSetUp]
|
[OneTimeSetUp]
|
||||||
|
@ -30,14 +32,11 @@ namespace DistTestCore
|
||||||
{
|
{
|
||||||
// Previous test run may have been interrupted.
|
// Previous test run may have been interrupted.
|
||||||
// Begin by cleaning everything up.
|
// Begin by cleaning everything up.
|
||||||
Timing.UseLongTimeouts = false;
|
|
||||||
fixtureLog = new FixtureLog(configuration.GetLogConfig());
|
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
Stopwatch.Measure(fixtureLog, "Global setup", () =>
|
Stopwatch.Measure(fixtureLog, "Global setup", () =>
|
||||||
{
|
{
|
||||||
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration());
|
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
|
||||||
wc.CreateWorkflow().DeleteAllResources();
|
wc.CreateWorkflow().DeleteAllResources();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -57,8 +56,6 @@ namespace DistTestCore
|
||||||
[SetUp]
|
[SetUp]
|
||||||
public void SetUpDistTest()
|
public void SetUpDistTest()
|
||||||
{
|
{
|
||||||
Timing.UseLongTimeouts = ShouldUseLongTimeouts();
|
|
||||||
|
|
||||||
if (GlobalTestFailure.HasFailed)
|
if (GlobalTestFailure.HasFailed)
|
||||||
{
|
{
|
||||||
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
|
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
|
||||||
|
@ -85,7 +82,7 @@ namespace DistTestCore
|
||||||
|
|
||||||
public TestFile GenerateTestFile(ByteSize size)
|
public TestFile GenerateTestFile(ByteSize size)
|
||||||
{
|
{
|
||||||
return lifecycle.FileManager.GenerateTestFile(size);
|
return Get().FileManager.GenerateTestFile(size);
|
||||||
}
|
}
|
||||||
|
|
||||||
public IOnlineCodexNode SetupCodexBootstrapNode()
|
public IOnlineCodexNode SetupCodexBootstrapNode()
|
||||||
|
@ -128,12 +125,58 @@ namespace DistTestCore
|
||||||
|
|
||||||
public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
|
public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
|
||||||
{
|
{
|
||||||
return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup);
|
return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected BaseLog Log
|
protected void Log(string msg)
|
||||||
{
|
{
|
||||||
get { return lifecycle.Log; }
|
TestContext.Progress.WriteLine(msg);
|
||||||
|
Get().Log.Log(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void Debug(string msg)
|
||||||
|
{
|
||||||
|
TestContext.Progress.WriteLine(msg);
|
||||||
|
Get().Log.Debug(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TestLifecycle Get()
|
||||||
|
{
|
||||||
|
lock (lifecycleLock)
|
||||||
|
{
|
||||||
|
return lifecycles[GetCurrentTestName()];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void CreateNewTestLifecycle()
|
||||||
|
{
|
||||||
|
var testName = GetCurrentTestName();
|
||||||
|
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
|
||||||
|
{
|
||||||
|
lock (lifecycleLock)
|
||||||
|
{
|
||||||
|
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private void DisposeTestLifecycle()
|
||||||
|
{
|
||||||
|
var lifecycle = Get();
|
||||||
|
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
|
||||||
|
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
|
||||||
|
{
|
||||||
|
lifecycle.Log.EndTest();
|
||||||
|
IncludeLogsAndMetricsOnTestFailure(lifecycle);
|
||||||
|
lifecycle.DeleteAllResources();
|
||||||
|
lifecycle = null!;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private ITimeSet GetTimeSet()
|
||||||
|
{
|
||||||
|
if (ShouldUseLongTimeouts()) return new LongTimeSet();
|
||||||
|
return new DefaultTimeSet();
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool ShouldUseLongTimeouts()
|
private bool ShouldUseLongTimeouts()
|
||||||
|
@ -151,28 +194,7 @@ namespace DistTestCore
|
||||||
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
|
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void CreateNewTestLifecycle()
|
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
|
||||||
{
|
|
||||||
Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
|
|
||||||
{
|
|
||||||
lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
|
|
||||||
testStart = DateTime.UtcNow;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DisposeTestLifecycle()
|
|
||||||
{
|
|
||||||
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
|
|
||||||
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
|
|
||||||
{
|
|
||||||
lifecycle.Log.EndTest();
|
|
||||||
IncludeLogsAndMetricsOnTestFailure();
|
|
||||||
lifecycle.DeleteAllResources();
|
|
||||||
lifecycle = null!;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private void IncludeLogsAndMetricsOnTestFailure()
|
|
||||||
{
|
{
|
||||||
var result = TestContext.CurrentContext.Result;
|
var result = TestContext.CurrentContext.Result;
|
||||||
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
|
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
|
||||||
|
@ -182,8 +204,8 @@ namespace DistTestCore
|
||||||
if (IsDownloadingLogsAndMetricsEnabled())
|
if (IsDownloadingLogsAndMetricsEnabled())
|
||||||
{
|
{
|
||||||
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
|
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
|
||||||
DownloadAllLogs();
|
DownloadAllLogs(lifecycle);
|
||||||
DownloadAllMetrics();
|
DownloadAllMetrics(lifecycle);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -192,25 +214,19 @@ namespace DistTestCore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private string GetTestDuration()
|
private void DownloadAllLogs(TestLifecycle lifecycle)
|
||||||
{
|
{
|
||||||
var testDuration = DateTime.UtcNow - testStart;
|
OnEachCodexNode(lifecycle, node =>
|
||||||
return Time.FormatDuration(testDuration);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void DownloadAllLogs()
|
|
||||||
{
|
|
||||||
OnEachCodexNode(node =>
|
|
||||||
{
|
{
|
||||||
lifecycle.DownloadLog(node);
|
lifecycle.DownloadLog(node);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private void DownloadAllMetrics()
|
private void DownloadAllMetrics(TestLifecycle lifecycle)
|
||||||
{
|
{
|
||||||
var metricsDownloader = new MetricsDownloader(lifecycle.Log);
|
var metricsDownloader = new MetricsDownloader(lifecycle.Log);
|
||||||
|
|
||||||
OnEachCodexNode(node =>
|
OnEachCodexNode(lifecycle, node =>
|
||||||
{
|
{
|
||||||
var m = node.Metrics as MetricsAccess;
|
var m = node.Metrics as MetricsAccess;
|
||||||
if (m != null)
|
if (m != null)
|
||||||
|
@ -220,7 +236,7 @@ namespace DistTestCore
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private void OnEachCodexNode(Action<OnlineCodexNode> action)
|
private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
|
||||||
{
|
{
|
||||||
var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
|
var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
|
||||||
foreach (var node in allNodes)
|
foreach (var node in allNodes)
|
||||||
|
|
|
@ -1,10 +1,14 @@
|
||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<RootNamespace>DistTestCore</RootNamespace>
|
<RootNamespace>DistTestCore</RootNamespace>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
<IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup Condition="'$(IsArm64)'=='true'">
|
||||||
|
<DefineConstants>Arm64</DefineConstants>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
using Logging;
|
using Logging;
|
||||||
using NUnit.Framework;
|
using NUnit.Framework;
|
||||||
|
using Utils;
|
||||||
|
|
||||||
namespace DistTestCore
|
namespace DistTestCore
|
||||||
{
|
{
|
||||||
|
@ -13,13 +14,14 @@ namespace DistTestCore
|
||||||
public class FileManager : IFileManager
|
public class FileManager : IFileManager
|
||||||
{
|
{
|
||||||
public const int ChunkSize = 1024 * 1024;
|
public const int ChunkSize = 1024 * 1024;
|
||||||
|
private static NumberSource folderNumberSource = new NumberSource(0);
|
||||||
private readonly Random random = new Random();
|
private readonly Random random = new Random();
|
||||||
private readonly TestLog log;
|
private readonly TestLog log;
|
||||||
private readonly string folder;
|
private readonly string folder;
|
||||||
|
|
||||||
public FileManager(TestLog log, Configuration configuration)
|
public FileManager(TestLog log, Configuration configuration)
|
||||||
{
|
{
|
||||||
folder = configuration.GetFileManagerFolder();
|
folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
|
||||||
|
|
||||||
EnsureDirectory();
|
EnsureDirectory();
|
||||||
this.log = log;
|
this.log = log;
|
||||||
|
|
|
@ -36,13 +36,8 @@ namespace DistTestCore
|
||||||
var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
|
var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
|
||||||
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
|
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
|
||||||
|
|
||||||
foreach (var account in companionNode.Accounts)
|
var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
|
||||||
{
|
interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
|
||||||
interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei);
|
|
||||||
interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
|
|
||||||
}
|
|
||||||
|
|
||||||
interaction.WaitForAllTransactions();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
|
private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
|
||||||
|
|
|
@ -10,13 +10,15 @@ namespace DistTestCore
|
||||||
public class Http
|
public class Http
|
||||||
{
|
{
|
||||||
private readonly BaseLog log;
|
private readonly BaseLog log;
|
||||||
|
private readonly ITimeSet timeSet;
|
||||||
private readonly string ip;
|
private readonly string ip;
|
||||||
private readonly int port;
|
private readonly int port;
|
||||||
private readonly string baseUrl;
|
private readonly string baseUrl;
|
||||||
|
|
||||||
public Http(BaseLog log, string ip, int port, string baseUrl)
|
public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
|
this.timeSet = timeSet;
|
||||||
this.ip = ip;
|
this.ip = ip;
|
||||||
this.port = port;
|
this.port = port;
|
||||||
this.baseUrl = baseUrl;
|
this.baseUrl = baseUrl;
|
||||||
|
@ -103,7 +105,7 @@ namespace DistTestCore
|
||||||
log.Debug($"({url}) = '{message}'", 3);
|
log.Debug($"({url}) = '{message}'", 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static T Retry<T>(Func<T> operation)
|
private T Retry<T>(Func<T> operation)
|
||||||
{
|
{
|
||||||
var retryCounter = 0;
|
var retryCounter = 0;
|
||||||
|
|
||||||
|
@ -115,9 +117,9 @@ namespace DistTestCore
|
||||||
}
|
}
|
||||||
catch (Exception exception)
|
catch (Exception exception)
|
||||||
{
|
{
|
||||||
Timing.HttpCallRetryDelay();
|
timeSet.HttpCallRetryDelay();
|
||||||
retryCounter++;
|
retryCounter++;
|
||||||
if (retryCounter > Timing.HttpCallRetryCount())
|
if (retryCounter > timeSet.HttpCallRetryCount())
|
||||||
{
|
{
|
||||||
Assert.Fail(exception.ToString());
|
Assert.Fail(exception.ToString());
|
||||||
throw;
|
throw;
|
||||||
|
@ -140,10 +142,10 @@ namespace DistTestCore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static HttpClient GetClient()
|
private HttpClient GetClient()
|
||||||
{
|
{
|
||||||
var client = new HttpClient();
|
var client = new HttpClient();
|
||||||
client.Timeout = Timing.HttpCallTimeout();
|
client.Timeout = timeSet.HttpCallTimeout();
|
||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,11 @@ namespace DistTestCore.Marketplace
|
||||||
{
|
{
|
||||||
public class CodexContractsContainerRecipe : ContainerRecipeFactory
|
public class CodexContractsContainerRecipe : ContainerRecipeFactory
|
||||||
{
|
{
|
||||||
public const string DockerImage = "thatbenbierens/codex-contracts-deployment";
|
#if Arm64
|
||||||
|
public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
|
||||||
|
#else
|
||||||
|
public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint";
|
||||||
|
#endif
|
||||||
public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
|
public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
|
||||||
public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";
|
public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";
|
||||||
|
|
||||||
|
|
|
@ -19,13 +19,14 @@ namespace DistTestCore.Marketplace
|
||||||
this.container = container;
|
this.container = container;
|
||||||
}
|
}
|
||||||
|
|
||||||
public string ExtractAccount(int? orderNumber)
|
public AllGethAccounts ExtractAccounts()
|
||||||
{
|
{
|
||||||
log.Debug();
|
log.Debug();
|
||||||
var account = Retry(() => FetchAccount(orderNumber));
|
var accountsCsv = Retry(() => FetchAccountsCsv());
|
||||||
if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
|
if (string.IsNullOrEmpty(accountsCsv)) throw new InvalidOperationException("Unable to fetch accounts.csv for geth node. Test infra failure.");
|
||||||
|
|
||||||
return account;
|
var lines = accountsCsv.Split('\n');
|
||||||
|
return new AllGethAccounts(lines.Select(ParseLineToAccount).ToArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
public string ExtractPubKey()
|
public string ExtractPubKey()
|
||||||
|
@ -37,15 +38,6 @@ namespace DistTestCore.Marketplace
|
||||||
return pubKey;
|
return pubKey;
|
||||||
}
|
}
|
||||||
|
|
||||||
public string ExtractPrivateKey(int? orderNumber)
|
|
||||||
{
|
|
||||||
log.Debug();
|
|
||||||
var privKey = Retry(() => FetchPrivateKey(orderNumber));
|
|
||||||
if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
|
|
||||||
|
|
||||||
return privKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
public string ExtractMarketplaceAddress()
|
public string ExtractMarketplaceAddress()
|
||||||
{
|
{
|
||||||
log.Debug();
|
log.Debug();
|
||||||
|
@ -88,14 +80,9 @@ namespace DistTestCore.Marketplace
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private string FetchAccount(int? orderNumber)
|
private string FetchAccountsCsv()
|
||||||
{
|
{
|
||||||
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber));
|
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
|
||||||
}
|
|
||||||
|
|
||||||
private string FetchPrivateKey(int? orderNumber)
|
|
||||||
{
|
|
||||||
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private string FetchMarketplaceAddress()
|
private string FetchMarketplaceAddress()
|
||||||
|
@ -120,6 +107,15 @@ namespace DistTestCore.Marketplace
|
||||||
workflow.DownloadContainerLog(container, enodeFinder);
|
workflow.DownloadContainerLog(container, enodeFinder);
|
||||||
return enodeFinder.GetPubKey();
|
return enodeFinder.GetPubKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private GethAccount ParseLineToAccount(string l)
|
||||||
|
{
|
||||||
|
var tokens = l.Replace("\r", "").Split(',');
|
||||||
|
if (tokens.Length != 2) throw new InvalidOperationException();
|
||||||
|
var account = tokens[0];
|
||||||
|
var privateKey = tokens[1];
|
||||||
|
return new GethAccount(account, privateKey);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public class PubKeyFinder : LogHandler, ILogHandler
|
public class PubKeyFinder : LogHandler, ILogHandler
|
||||||
|
|
|
@ -6,19 +6,19 @@ namespace DistTestCore.Marketplace
|
||||||
{
|
{
|
||||||
public class GethBootstrapNodeInfo
|
public class GethBootstrapNodeInfo
|
||||||
{
|
{
|
||||||
public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string pubKey, string privateKey, Port discoveryPort)
|
public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
|
||||||
{
|
{
|
||||||
RunningContainers = runningContainers;
|
RunningContainers = runningContainers;
|
||||||
Account = account;
|
AllAccounts = allAccounts;
|
||||||
|
Account = allAccounts.Accounts[0];
|
||||||
PubKey = pubKey;
|
PubKey = pubKey;
|
||||||
PrivateKey = privateKey;
|
|
||||||
DiscoveryPort = discoveryPort;
|
DiscoveryPort = discoveryPort;
|
||||||
}
|
}
|
||||||
|
|
||||||
public RunningContainers RunningContainers { get; }
|
public RunningContainers RunningContainers { get; }
|
||||||
public string Account { get; }
|
public AllGethAccounts AllAccounts { get; }
|
||||||
|
public GethAccount Account { get; }
|
||||||
public string PubKey { get; }
|
public string PubKey { get; }
|
||||||
public string PrivateKey { get; }
|
|
||||||
public Port DiscoveryPort { get; }
|
public Port DiscoveryPort { get; }
|
||||||
|
|
||||||
public NethereumInteraction StartInteraction(BaseLog log)
|
public NethereumInteraction StartInteraction(BaseLog log)
|
||||||
|
@ -26,10 +26,19 @@ namespace DistTestCore.Marketplace
|
||||||
var ip = RunningContainers.RunningPod.Cluster.IP;
|
var ip = RunningContainers.RunningPod.Cluster.IP;
|
||||||
var port = RunningContainers.Containers[0].ServicePorts[0].Number;
|
var port = RunningContainers.Containers[0].ServicePorts[0].Number;
|
||||||
var account = Account;
|
var account = Account;
|
||||||
var privateKey = PrivateKey;
|
|
||||||
|
|
||||||
var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey);
|
var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey);
|
||||||
return creator.CreateWorkflow();
|
return creator.CreateWorkflow();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public class AllGethAccounts
|
||||||
|
{
|
||||||
|
public GethAccount[] Accounts { get; }
|
||||||
|
|
||||||
|
public AllGethAccounts(GethAccount[] accounts)
|
||||||
|
{
|
||||||
|
Accounts = accounts;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,20 +20,20 @@ namespace DistTestCore.Marketplace
|
||||||
var bootstrapContainer = containers.Containers[0];
|
var bootstrapContainer = containers.Containers[0];
|
||||||
|
|
||||||
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
|
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
|
||||||
var account = extractor.ExtractAccount(null);
|
var accounts = extractor.ExtractAccounts();
|
||||||
var pubKey = extractor.ExtractPubKey();
|
var pubKey = extractor.ExtractPubKey();
|
||||||
var privateKey = extractor.ExtractPrivateKey(null);
|
|
||||||
var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
|
var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
|
||||||
|
var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
|
||||||
|
|
||||||
LogEnd($"Geth bootstrap node started with account '{account}'");
|
LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
|
||||||
|
|
||||||
return new GethBootstrapNodeInfo(containers, account, pubKey, privateKey, discoveryPort);
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private StartupConfig CreateBootstrapStartupConfig()
|
private StartupConfig CreateBootstrapStartupConfig()
|
||||||
{
|
{
|
||||||
var config = new StartupConfig();
|
var config = new StartupConfig();
|
||||||
config.Add(new GethStartupConfig(true, null!, 0));
|
config.Add(new GethStartupConfig(true, null!, 0, 0));
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,30 +6,29 @@ namespace DistTestCore.Marketplace
|
||||||
{
|
{
|
||||||
public class GethCompanionNodeInfo
|
public class GethCompanionNodeInfo
|
||||||
{
|
{
|
||||||
public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts)
|
public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
|
||||||
{
|
{
|
||||||
RunningContainer = runningContainer;
|
RunningContainer = runningContainer;
|
||||||
Accounts = accounts;
|
Accounts = accounts;
|
||||||
}
|
}
|
||||||
|
|
||||||
public RunningContainer RunningContainer { get; }
|
public RunningContainer RunningContainer { get; }
|
||||||
public GethCompanionAccount[] Accounts { get; }
|
public GethAccount[] Accounts { get; }
|
||||||
|
|
||||||
public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account)
|
public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
|
||||||
{
|
{
|
||||||
var ip = RunningContainer.Pod.Cluster.IP;
|
var ip = RunningContainer.Pod.Cluster.IP;
|
||||||
var port = RunningContainer.ServicePorts[0].Number;
|
var port = RunningContainer.ServicePorts[0].Number;
|
||||||
var accountStr = account.Account;
|
|
||||||
var privateKey = account.PrivateKey;
|
var privateKey = account.PrivateKey;
|
||||||
|
|
||||||
var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey);
|
var creator = new NethereumInteractionCreator(log, ip, port, privateKey);
|
||||||
return creator.CreateWorkflow();
|
return creator.CreateWorkflow();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public class GethCompanionAccount
|
public class GethAccount
|
||||||
{
|
{
|
||||||
public GethCompanionAccount(string account, string privateKey)
|
public GethAccount(string account, string privateKey)
|
||||||
{
|
{
|
||||||
Account = account;
|
Account = account;
|
||||||
PrivateKey = privateKey;
|
PrivateKey = privateKey;
|
||||||
|
|
|
@ -5,6 +5,8 @@ namespace DistTestCore.Marketplace
|
||||||
{
|
{
|
||||||
public class GethCompanionNodeStarter : BaseStarter
|
public class GethCompanionNodeStarter : BaseStarter
|
||||||
{
|
{
|
||||||
|
private int companionAccountIndex = 0;
|
||||||
|
|
||||||
public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
|
public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
|
||||||
: base(lifecycle, workflowCreator)
|
: base(lifecycle, workflowCreator)
|
||||||
{
|
{
|
||||||
|
@ -14,53 +16,43 @@ namespace DistTestCore.Marketplace
|
||||||
{
|
{
|
||||||
LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
|
LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
|
||||||
|
|
||||||
var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
|
var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
|
||||||
|
|
||||||
var workflow = workflowCreator.CreateWorkflow();
|
var workflow = workflowCreator.CreateWorkflow();
|
||||||
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
|
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
|
||||||
WaitForAccountCreation(codexSetup.NumberOfNodes);
|
|
||||||
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
|
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
|
||||||
var container = containers.Containers[0];
|
var container = containers.Containers[0];
|
||||||
|
|
||||||
var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes);
|
var node = CreateCompanionInfo(container, marketplace, config);
|
||||||
EnsureCompanionNodeIsSynced(node, marketplace);
|
EnsureCompanionNodeIsSynced(node, marketplace);
|
||||||
|
|
||||||
LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
|
LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
|
||||||
return node;
|
return node;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void WaitForAccountCreation(int numberOfNodes)
|
private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
|
||||||
{
|
{
|
||||||
// We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files
|
var accounts = ExtractAccounts(marketplace, config);
|
||||||
// we will be trying to read in 'ExtractAccount', later on in the start-up process.
|
|
||||||
Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes));
|
|
||||||
}
|
|
||||||
|
|
||||||
private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts)
|
|
||||||
{
|
|
||||||
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
|
|
||||||
var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray();
|
|
||||||
return new GethCompanionNodeInfo(container, accounts);
|
return new GethCompanionNodeInfo(container, accounts);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IEnumerable<GethCompanionAccount> ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts)
|
private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
|
||||||
{
|
{
|
||||||
for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1);
|
return marketplace.Bootstrap.AllAccounts.Accounts
|
||||||
}
|
.Skip(1 + config.CompanionAccountStartIndex)
|
||||||
|
.Take(config.NumberOfCompanionAccounts)
|
||||||
private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber)
|
.ToArray();
|
||||||
{
|
|
||||||
var account = extractor.ExtractAccount(orderNumber);
|
|
||||||
var privKey = extractor.ExtractPrivateKey(orderNumber);
|
|
||||||
return new GethCompanionAccount(account, privKey);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
|
private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
|
{
|
||||||
|
Time.WaitUntil(() =>
|
||||||
{
|
{
|
||||||
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
|
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
|
||||||
interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
|
return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
|
||||||
|
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
|
@ -68,10 +60,17 @@ namespace DistTestCore.Marketplace
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
|
private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
|
||||||
|
{
|
||||||
|
var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
|
||||||
|
companionAccountIndex += numberOfAccounts;
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
|
||||||
{
|
{
|
||||||
var config = new StartupConfig();
|
var config = new StartupConfig();
|
||||||
config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts));
|
config.Add(gethConfig);
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,22 +4,17 @@ namespace DistTestCore.Marketplace
|
||||||
{
|
{
|
||||||
public class GethContainerRecipe : ContainerRecipeFactory
|
public class GethContainerRecipe : ContainerRecipeFactory
|
||||||
{
|
{
|
||||||
public const string DockerImage = "thatbenbierens/geth-confenv:latest";
|
#if Arm64
|
||||||
|
public const string DockerImage = "emizzle/geth-confenv:latest";
|
||||||
|
#else
|
||||||
|
public const string DockerImage = "thatbenbierens/geth-confenv:onethousand";
|
||||||
|
#endif
|
||||||
|
|
||||||
public const string HttpPortTag = "http_port";
|
public const string HttpPortTag = "http_port";
|
||||||
public const string DiscoveryPortTag = "disc_port";
|
public const string DiscoveryPortTag = "disc_port";
|
||||||
private const string defaultArgs = "--ipcdisable --syncmode full";
|
private const string defaultArgs = "--ipcdisable --syncmode full";
|
||||||
|
|
||||||
public static string GetAccountFilename(int? orderNumber)
|
public const string AccountsFilename = "accounts.csv";
|
||||||
{
|
|
||||||
if (orderNumber == null) return "account_string.txt";
|
|
||||||
return $"account_string_{orderNumber.Value}.txt";
|
|
||||||
}
|
|
||||||
|
|
||||||
public static string GetPrivateKeyFilename(int? orderNumber)
|
|
||||||
{
|
|
||||||
if (orderNumber == null) return "private.key";
|
|
||||||
return $"private_{orderNumber.Value}.key";
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override string Image => DockerImage;
|
protected override string Image => DockerImage;
|
||||||
|
|
||||||
|
@ -46,14 +41,17 @@ namespace DistTestCore.Marketplace
|
||||||
|
|
||||||
private string CreateBootstapArgs(Port discovery)
|
private string CreateBootstapArgs(Port discovery)
|
||||||
{
|
{
|
||||||
AddEnvVar("IS_BOOTSTRAP", "1");
|
AddEnvVar("ENABLE_MINER", "1");
|
||||||
|
UnlockAccounts(0, 1);
|
||||||
var exposedPort = AddExposedPort(tag: HttpPortTag);
|
var exposedPort = AddExposedPort(tag: HttpPortTag);
|
||||||
return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
|
return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
|
||||||
}
|
}
|
||||||
|
|
||||||
private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
|
private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
|
||||||
{
|
{
|
||||||
AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString());
|
UnlockAccounts(
|
||||||
|
config.CompanionAccountStartIndex + 1,
|
||||||
|
config.NumberOfCompanionAccounts);
|
||||||
|
|
||||||
var port = AddInternalPort();
|
var port = AddInternalPort();
|
||||||
var authRpc = AddInternalPort();
|
var authRpc = AddInternalPort();
|
||||||
|
@ -66,5 +64,15 @@ namespace DistTestCore.Marketplace
|
||||||
|
|
||||||
return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
|
return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void UnlockAccounts(int startIndex, int numberOfAccounts)
|
||||||
|
{
|
||||||
|
if (startIndex < 0) throw new ArgumentException();
|
||||||
|
if (numberOfAccounts < 1) throw new ArgumentException();
|
||||||
|
if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
|
||||||
|
|
||||||
|
AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
|
||||||
|
AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,15 +2,17 @@
|
||||||
{
|
{
|
||||||
public class GethStartupConfig
|
public class GethStartupConfig
|
||||||
{
|
{
|
||||||
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts)
|
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
|
||||||
{
|
{
|
||||||
IsBootstrapNode = isBootstrapNode;
|
IsBootstrapNode = isBootstrapNode;
|
||||||
BootstrapNode = bootstrapNode;
|
BootstrapNode = bootstrapNode;
|
||||||
|
CompanionAccountStartIndex = companionAccountStartIndex;
|
||||||
NumberOfCompanionAccounts = numberOfCompanionAccounts;
|
NumberOfCompanionAccounts = numberOfCompanionAccounts;
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool IsBootstrapNode { get; }
|
public bool IsBootstrapNode { get; }
|
||||||
public GethBootstrapNodeInfo BootstrapNode { get; }
|
public GethBootstrapNodeInfo BootstrapNode { get; }
|
||||||
|
public int CompanionAccountStartIndex { get; }
|
||||||
public int NumberOfCompanionAccounts { get; }
|
public int NumberOfCompanionAccounts { get; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace
|
||||||
{
|
{
|
||||||
private readonly TestLog log;
|
private readonly TestLog log;
|
||||||
private readonly MarketplaceNetwork marketplaceNetwork;
|
private readonly MarketplaceNetwork marketplaceNetwork;
|
||||||
private readonly GethCompanionAccount account;
|
private readonly GethAccount account;
|
||||||
private readonly CodexAccess codexAccess;
|
private readonly CodexAccess codexAccess;
|
||||||
|
|
||||||
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess)
|
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
this.marketplaceNetwork = marketplaceNetwork;
|
this.marketplaceNetwork = marketplaceNetwork;
|
||||||
|
|
|
@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace
|
||||||
return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
|
return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
|
||||||
}
|
}
|
||||||
|
|
||||||
private GethCompanionAccount GetGethCompanionNode(CodexAccess access)
|
private GethAccount GetGethCompanionNode(CodexAccess access)
|
||||||
{
|
{
|
||||||
var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount);
|
var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
|
||||||
return (GethCompanionAccount)account;
|
return (GethAccount)account;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,12 +14,14 @@ namespace DistTestCore.Metrics
|
||||||
public class MetricsAccess : IMetricsAccess
|
public class MetricsAccess : IMetricsAccess
|
||||||
{
|
{
|
||||||
private readonly TestLog log;
|
private readonly TestLog log;
|
||||||
|
private readonly ITimeSet timeSet;
|
||||||
private readonly MetricsQuery query;
|
private readonly MetricsQuery query;
|
||||||
private readonly RunningContainer node;
|
private readonly RunningContainer node;
|
||||||
|
|
||||||
public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node)
|
public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
|
this.timeSet = timeSet;
|
||||||
this.query = query;
|
this.query = query;
|
||||||
this.node = node;
|
this.node = node;
|
||||||
}
|
}
|
||||||
|
@ -47,7 +49,7 @@ namespace DistTestCore.Metrics
|
||||||
{
|
{
|
||||||
var mostRecent = GetMostRecent(metricName);
|
var mostRecent = GetMostRecent(metricName);
|
||||||
if (mostRecent != null) return mostRecent;
|
if (mostRecent != null) return mostRecent;
|
||||||
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
|
if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
|
||||||
{
|
{
|
||||||
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
|
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
|
||||||
throw new TimeoutException();
|
throw new TimeoutException();
|
||||||
|
|
|
@ -28,8 +28,8 @@ namespace DistTestCore.Metrics
|
||||||
|
|
||||||
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
|
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
|
||||||
{
|
{
|
||||||
var query = new MetricsQuery(lifecycle.Log, prometheusContainer);
|
var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
|
||||||
return new MetricsAccess(lifecycle.Log, query, codexContainer);
|
return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,12 +9,13 @@ namespace DistTestCore.Metrics
|
||||||
{
|
{
|
||||||
private readonly Http http;
|
private readonly Http http;
|
||||||
|
|
||||||
public MetricsQuery(BaseLog log, RunningContainers runningContainers)
|
public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
|
||||||
{
|
{
|
||||||
RunningContainers = runningContainers;
|
RunningContainers = runningContainers;
|
||||||
|
|
||||||
http = new Http(
|
http = new Http(
|
||||||
log,
|
log,
|
||||||
|
timeSet,
|
||||||
runningContainers.RunningPod.Cluster.IP,
|
runningContainers.RunningPod.Cluster.IP,
|
||||||
runningContainers.Containers[0].ServicePorts[0].Number,
|
runningContainers.Containers[0].ServicePorts[0].Number,
|
||||||
"api/v1");
|
"api/v1");
|
||||||
|
|
|
@ -92,6 +92,10 @@ namespace DistTestCore
|
||||||
|
|
||||||
public ICodexSetup BringOffline()
|
public ICodexSetup BringOffline()
|
||||||
{
|
{
|
||||||
|
if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
|
||||||
|
"individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
|
||||||
|
"available for codex-nodes in groups of 1.");
|
||||||
|
|
||||||
return Group.BringOffline();
|
return Group.BringOffline();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,25 +1,30 @@
|
||||||
using DistTestCore.Logs;
|
using DistTestCore.Logs;
|
||||||
using KubernetesWorkflow;
|
using KubernetesWorkflow;
|
||||||
using Logging;
|
using Logging;
|
||||||
|
using Utils;
|
||||||
|
|
||||||
namespace DistTestCore
|
namespace DistTestCore
|
||||||
{
|
{
|
||||||
public class TestLifecycle
|
public class TestLifecycle
|
||||||
{
|
{
|
||||||
private readonly WorkflowCreator workflowCreator;
|
private readonly WorkflowCreator workflowCreator;
|
||||||
|
private DateTime testStart = DateTime.MinValue;
|
||||||
|
|
||||||
public TestLifecycle(TestLog log, Configuration configuration)
|
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
|
||||||
{
|
{
|
||||||
Log = log;
|
Log = log;
|
||||||
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration());
|
TimeSet = timeSet;
|
||||||
|
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
|
||||||
|
|
||||||
FileManager = new FileManager(Log, configuration);
|
FileManager = new FileManager(Log, configuration);
|
||||||
CodexStarter = new CodexStarter(this, workflowCreator);
|
CodexStarter = new CodexStarter(this, workflowCreator);
|
||||||
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
|
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
|
||||||
GethStarter = new GethStarter(this, workflowCreator);
|
GethStarter = new GethStarter(this, workflowCreator);
|
||||||
|
testStart = DateTime.UtcNow;
|
||||||
}
|
}
|
||||||
|
|
||||||
public TestLog Log { get; }
|
public TestLog Log { get; }
|
||||||
|
public ITimeSet TimeSet { get; }
|
||||||
public FileManager FileManager { get; }
|
public FileManager FileManager { get; }
|
||||||
public CodexStarter CodexStarter { get; }
|
public CodexStarter CodexStarter { get; }
|
||||||
public PrometheusStarter PrometheusStarter { get; }
|
public PrometheusStarter PrometheusStarter { get; }
|
||||||
|
@ -42,5 +47,11 @@ namespace DistTestCore
|
||||||
|
|
||||||
return new CodexNodeLog(subFile, node);
|
return new CodexNodeLog(subFile, node);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public string GetTestDuration()
|
||||||
|
{
|
||||||
|
var testDuration = DateTime.UtcNow - testStart;
|
||||||
|
return Time.FormatDuration(testDuration);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,53 +8,11 @@ namespace DistTestCore
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class Timing
|
|
||||||
{
|
|
||||||
public static bool UseLongTimeouts { get; set; }
|
|
||||||
|
|
||||||
|
|
||||||
public static TimeSpan HttpCallTimeout()
|
|
||||||
{
|
|
||||||
return GetTimes().HttpCallTimeout();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int HttpCallRetryCount()
|
|
||||||
{
|
|
||||||
return GetTimes().HttpCallRetryCount();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void HttpCallRetryDelay()
|
|
||||||
{
|
|
||||||
Time.Sleep(GetTimes().HttpCallRetryDelay());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static TimeSpan K8sServiceDelay()
|
|
||||||
{
|
|
||||||
return GetTimes().WaitForK8sServiceDelay();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static TimeSpan K8sOperationTimeout()
|
|
||||||
{
|
|
||||||
return GetTimes().K8sOperationTimeout();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static TimeSpan WaitForMetricTimeout()
|
|
||||||
{
|
|
||||||
return GetTimes().WaitForMetricTimeout();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ITimeSet GetTimes()
|
|
||||||
{
|
|
||||||
if (UseLongTimeouts) return new LongTimeSet();
|
|
||||||
return new DefaultTimeSet();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public interface ITimeSet
|
public interface ITimeSet
|
||||||
{
|
{
|
||||||
TimeSpan HttpCallTimeout();
|
TimeSpan HttpCallTimeout();
|
||||||
int HttpCallRetryCount();
|
int HttpCallRetryCount();
|
||||||
TimeSpan HttpCallRetryDelay();
|
void HttpCallRetryDelay();
|
||||||
TimeSpan WaitForK8sServiceDelay();
|
TimeSpan WaitForK8sServiceDelay();
|
||||||
TimeSpan K8sOperationTimeout();
|
TimeSpan K8sOperationTimeout();
|
||||||
TimeSpan WaitForMetricTimeout();
|
TimeSpan WaitForMetricTimeout();
|
||||||
|
@ -72,9 +30,9 @@ namespace DistTestCore
|
||||||
return 5;
|
return 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryDelay()
|
public void HttpCallRetryDelay()
|
||||||
{
|
{
|
||||||
return TimeSpan.FromSeconds(3);
|
Time.Sleep(TimeSpan.FromSeconds(3));
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan WaitForK8sServiceDelay()
|
public TimeSpan WaitForK8sServiceDelay()
|
||||||
|
@ -105,9 +63,9 @@ namespace DistTestCore
|
||||||
return 2;
|
return 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryDelay()
|
public void HttpCallRetryDelay()
|
||||||
{
|
{
|
||||||
return TimeSpan.FromMinutes(5);
|
Time.Sleep(TimeSpan.FromMinutes(5));
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan WaitForK8sServiceDelay()
|
public TimeSpan WaitForK8sServiceDelay()
|
||||||
|
|
|
@ -0,0 +1,40 @@
|
||||||
|
using Utils;
|
||||||
|
|
||||||
|
namespace KubernetesWorkflow
|
||||||
|
{
|
||||||
|
public class ApplicationLifecycle
|
||||||
|
{
|
||||||
|
private static object instanceLock = new object();
|
||||||
|
private static ApplicationLifecycle? instance;
|
||||||
|
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
|
||||||
|
private readonly NumberSource namespaceNumberSource = new NumberSource(0);
|
||||||
|
|
||||||
|
private ApplicationLifecycle()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ApplicationLifecycle Instance
|
||||||
|
{
|
||||||
|
// I know singletons are quite evil. But we need to be sure this object is created only once
|
||||||
|
// and persists for the entire application lifecycle.
|
||||||
|
get
|
||||||
|
{
|
||||||
|
lock (instanceLock)
|
||||||
|
{
|
||||||
|
if (instance == null) instance = new ApplicationLifecycle();
|
||||||
|
return instance;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public NumberSource GetServiceNumberSource()
|
||||||
|
{
|
||||||
|
return servicePortNumberSource;
|
||||||
|
}
|
||||||
|
|
||||||
|
public string GetTestNamespace()
|
||||||
|
{
|
||||||
|
return namespaceNumberSource.GetNextNumber().ToString("D5");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -5,7 +5,7 @@ namespace KubernetesWorkflow
|
||||||
{
|
{
|
||||||
public class CommandRunner
|
public class CommandRunner
|
||||||
{
|
{
|
||||||
private readonly Kubernetes client;
|
private readonly K8sClient client;
|
||||||
private readonly string k8sNamespace;
|
private readonly string k8sNamespace;
|
||||||
private readonly RunningPod pod;
|
private readonly RunningPod pod;
|
||||||
private readonly string containerName;
|
private readonly string containerName;
|
||||||
|
@ -13,7 +13,7 @@ namespace KubernetesWorkflow
|
||||||
private readonly string[] arguments;
|
private readonly string[] arguments;
|
||||||
private readonly List<string> lines = new List<string>();
|
private readonly List<string> lines = new List<string>();
|
||||||
|
|
||||||
public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
|
public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
|
||||||
{
|
{
|
||||||
this.client = client;
|
this.client = client;
|
||||||
this.k8sNamespace = k8sNamespace;
|
this.k8sNamespace = k8sNamespace;
|
||||||
|
@ -27,8 +27,8 @@ namespace KubernetesWorkflow
|
||||||
{
|
{
|
||||||
var input = new[] { command }.Concat(arguments).ToArray();
|
var input = new[] { command }.Concat(arguments).ToArray();
|
||||||
|
|
||||||
Time.Wait(client.NamespacedPodExecAsync(
|
Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
|
||||||
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
|
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
|
||||||
}
|
}
|
||||||
|
|
||||||
public string GetStdOut()
|
public string GetStdOut()
|
||||||
|
|
|
@ -2,16 +2,16 @@
|
||||||
{
|
{
|
||||||
public class Configuration
|
public class Configuration
|
||||||
{
|
{
|
||||||
public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
|
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
|
||||||
{
|
{
|
||||||
K8sNamespace = k8sNamespace;
|
K8sNamespacePrefix = k8sNamespacePrefix;
|
||||||
KubeConfigFile = kubeConfigFile;
|
KubeConfigFile = kubeConfigFile;
|
||||||
OperationTimeout = operationTimeout;
|
OperationTimeout = operationTimeout;
|
||||||
RetryDelay = retryDelay;
|
RetryDelay = retryDelay;
|
||||||
LocationMap = locationMap;
|
LocationMap = locationMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
public string K8sNamespace { get; }
|
public string K8sNamespacePrefix { get; }
|
||||||
public string? KubeConfigFile { get; }
|
public string? KubeConfigFile { get; }
|
||||||
public TimeSpan OperationTimeout { get; }
|
public TimeSpan OperationTimeout { get; }
|
||||||
public TimeSpan RetryDelay { get; }
|
public TimeSpan RetryDelay { get; }
|
||||||
|
|
|
@ -24,6 +24,14 @@
|
||||||
{
|
{
|
||||||
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
|
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
return $"(container-recipe: {Name}, image: {Image}, " +
|
||||||
|
$"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
|
||||||
|
$"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
|
||||||
|
$"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, ";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public class Port
|
public class Port
|
||||||
|
|
|
@ -0,0 +1,36 @@
|
||||||
|
using k8s;
|
||||||
|
|
||||||
|
namespace KubernetesWorkflow
|
||||||
|
{
|
||||||
|
public class K8sClient
|
||||||
|
{
|
||||||
|
private readonly Kubernetes client;
|
||||||
|
private static readonly object clientLock = new object();
|
||||||
|
|
||||||
|
public K8sClient(KubernetesClientConfiguration config)
|
||||||
|
{
|
||||||
|
client = new Kubernetes(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Run(Action<Kubernetes> action)
|
||||||
|
{
|
||||||
|
lock (clientLock)
|
||||||
|
{
|
||||||
|
action(client);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public T Run<T>(Func<Kubernetes, T> action)
|
||||||
|
{
|
||||||
|
lock (clientLock)
|
||||||
|
{
|
||||||
|
return action(client);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
client.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -11,16 +11,18 @@ namespace KubernetesWorkflow
|
||||||
private readonly K8sCluster cluster;
|
private readonly K8sCluster cluster;
|
||||||
private readonly KnownK8sPods knownPods;
|
private readonly KnownK8sPods knownPods;
|
||||||
private readonly WorkflowNumberSource workflowNumberSource;
|
private readonly WorkflowNumberSource workflowNumberSource;
|
||||||
private readonly Kubernetes client;
|
private readonly K8sClient client;
|
||||||
|
|
||||||
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
|
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
this.cluster = cluster;
|
this.cluster = cluster;
|
||||||
this.knownPods = knownPods;
|
this.knownPods = knownPods;
|
||||||
this.workflowNumberSource = workflowNumberSource;
|
this.workflowNumberSource = workflowNumberSource;
|
||||||
|
client = new K8sClient(cluster.GetK8sClientConfig());
|
||||||
|
|
||||||
client = new Kubernetes(cluster.GetK8sClientConfig());
|
K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
|
||||||
|
log.Debug($"Test namespace: '{K8sTestNamespace}'");
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
|
@ -52,14 +54,14 @@ namespace KubernetesWorkflow
|
||||||
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
|
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
|
||||||
{
|
{
|
||||||
log.Debug();
|
log.Debug();
|
||||||
using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name);
|
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
|
||||||
logHandler.Log(stream);
|
logHandler.Log(stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
|
public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
|
||||||
{
|
{
|
||||||
log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
|
log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
|
||||||
var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
|
var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
|
||||||
runner.Run();
|
runner.Run();
|
||||||
return runner.GetStdOut();
|
return runner.GetStdOut();
|
||||||
}
|
}
|
||||||
|
@ -67,13 +69,43 @@ namespace KubernetesWorkflow
|
||||||
public void DeleteAllResources()
|
public void DeleteAllResources()
|
||||||
{
|
{
|
||||||
log.Debug();
|
log.Debug();
|
||||||
DeleteNamespace();
|
|
||||||
|
|
||||||
|
var all = client.Run(c => c.ListNamespace().Items);
|
||||||
|
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
|
||||||
|
|
||||||
|
foreach (var ns in namespaces)
|
||||||
|
{
|
||||||
|
DeleteNamespace(ns);
|
||||||
|
}
|
||||||
|
foreach (var ns in namespaces)
|
||||||
|
{
|
||||||
|
WaitUntilNamespaceDeleted(ns);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void DeleteTestNamespace()
|
||||||
|
{
|
||||||
|
log.Debug();
|
||||||
|
if (IsTestNamespaceOnline())
|
||||||
|
{
|
||||||
|
client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0));
|
||||||
|
}
|
||||||
WaitUntilNamespaceDeleted();
|
WaitUntilNamespaceDeleted();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void DeleteNamespace(string ns)
|
||||||
|
{
|
||||||
|
log.Debug();
|
||||||
|
if (IsNamespaceOnline(ns))
|
||||||
|
{
|
||||||
|
client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#region Namespace management
|
#region Namespace management
|
||||||
|
|
||||||
|
private string K8sTestNamespace { get; }
|
||||||
|
|
||||||
private void EnsureTestNamespace()
|
private void EnsureTestNamespace()
|
||||||
{
|
{
|
||||||
if (IsTestNamespaceOnline()) return;
|
if (IsTestNamespaceOnline()) return;
|
||||||
|
@ -83,30 +115,85 @@ namespace KubernetesWorkflow
|
||||||
ApiVersion = "v1",
|
ApiVersion = "v1",
|
||||||
Metadata = new V1ObjectMeta
|
Metadata = new V1ObjectMeta
|
||||||
{
|
{
|
||||||
Name = K8sNamespace,
|
Name = K8sTestNamespace,
|
||||||
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
|
Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
client.CreateNamespace(namespaceSpec);
|
client.Run(c => c.CreateNamespace(namespaceSpec));
|
||||||
WaitUntilNamespaceCreated();
|
WaitUntilNamespaceCreated();
|
||||||
}
|
|
||||||
|
|
||||||
private void DeleteNamespace()
|
CreatePolicy();
|
||||||
{
|
|
||||||
if (IsTestNamespaceOnline())
|
|
||||||
{
|
|
||||||
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private string K8sNamespace
|
|
||||||
{
|
|
||||||
get { return cluster.Configuration.K8sNamespace; }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool IsTestNamespaceOnline()
|
private bool IsTestNamespaceOnline()
|
||||||
{
|
{
|
||||||
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
|
return IsNamespaceOnline(K8sTestNamespace);
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool IsNamespaceOnline(string name)
|
||||||
|
{
|
||||||
|
return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void CreatePolicy()
|
||||||
|
{
|
||||||
|
client.Run(c =>
|
||||||
|
{
|
||||||
|
var body = new V1NetworkPolicy
|
||||||
|
{
|
||||||
|
Metadata = new V1ObjectMeta
|
||||||
|
{
|
||||||
|
Name = "isolate-policy",
|
||||||
|
NamespaceProperty = K8sTestNamespace
|
||||||
|
},
|
||||||
|
Spec = new V1NetworkPolicySpec
|
||||||
|
{
|
||||||
|
PodSelector = new V1LabelSelector
|
||||||
|
{
|
||||||
|
MatchLabels = GetSelector()
|
||||||
|
},
|
||||||
|
PolicyTypes = new[]
|
||||||
|
{
|
||||||
|
"Ingress",
|
||||||
|
"Egress"
|
||||||
|
},
|
||||||
|
Ingress = new List<V1NetworkPolicyIngressRule>
|
||||||
|
{
|
||||||
|
new V1NetworkPolicyIngressRule
|
||||||
|
{
|
||||||
|
FromProperty = new List<V1NetworkPolicyPeer>
|
||||||
|
{
|
||||||
|
new V1NetworkPolicyPeer
|
||||||
|
{
|
||||||
|
NamespaceSelector = new V1LabelSelector
|
||||||
|
{
|
||||||
|
MatchLabels = GetMyNamespaceSelector()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Egress = new List<V1NetworkPolicyEgressRule>
|
||||||
|
{
|
||||||
|
new V1NetworkPolicyEgressRule
|
||||||
|
{
|
||||||
|
To = new List<V1NetworkPolicyPeer>
|
||||||
|
{
|
||||||
|
new V1NetworkPolicyPeer
|
||||||
|
{
|
||||||
|
NamespaceSelector = new V1LabelSelector
|
||||||
|
{
|
||||||
|
MatchLabels = GetMyNamespaceSelector()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
@ -141,7 +228,7 @@ namespace KubernetesWorkflow
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
|
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace));
|
||||||
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
|
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
|
||||||
|
|
||||||
return deploymentSpec.Metadata.Name;
|
return deploymentSpec.Metadata.Name;
|
||||||
|
@ -149,7 +236,7 @@ namespace KubernetesWorkflow
|
||||||
|
|
||||||
private void DeleteDeployment(string deploymentName)
|
private void DeleteDeployment(string deploymentName)
|
||||||
{
|
{
|
||||||
client.DeleteNamespacedDeployment(deploymentName, K8sNamespace);
|
client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace));
|
||||||
WaitUntilDeploymentOffline(deploymentName);
|
WaitUntilDeploymentOffline(deploymentName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -168,12 +255,18 @@ namespace KubernetesWorkflow
|
||||||
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
|
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private IDictionary<string, string> GetMyNamespaceSelector()
|
||||||
|
{
|
||||||
|
return new Dictionary<string, string> { { "name", "thatisincorrect" } };
|
||||||
|
}
|
||||||
|
|
||||||
private V1ObjectMeta CreateDeploymentMetadata()
|
private V1ObjectMeta CreateDeploymentMetadata()
|
||||||
{
|
{
|
||||||
return new V1ObjectMeta
|
return new V1ObjectMeta
|
||||||
{
|
{
|
||||||
Name = "deploy-" + workflowNumberSource.WorkflowNumber,
|
Name = "deploy-" + workflowNumberSource.WorkflowNumber,
|
||||||
NamespaceProperty = K8sNamespace
|
NamespaceProperty = K8sTestNamespace,
|
||||||
|
Labels = GetSelector()
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -257,14 +350,14 @@ namespace KubernetesWorkflow
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
client.CreateNamespacedService(serviceSpec, K8sNamespace);
|
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
|
||||||
|
|
||||||
return (serviceSpec.Metadata.Name, result);
|
return (serviceSpec.Metadata.Name, result);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void DeleteService(string serviceName)
|
private void DeleteService(string serviceName)
|
||||||
{
|
{
|
||||||
client.DeleteNamespacedService(serviceName, K8sNamespace);
|
client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
|
||||||
}
|
}
|
||||||
|
|
||||||
private V1ObjectMeta CreateServiceMetadata()
|
private V1ObjectMeta CreateServiceMetadata()
|
||||||
|
@ -272,7 +365,7 @@ namespace KubernetesWorkflow
|
||||||
return new V1ObjectMeta
|
return new V1ObjectMeta
|
||||||
{
|
{
|
||||||
Name = "service-" + workflowNumberSource.WorkflowNumber,
|
Name = "service-" + workflowNumberSource.WorkflowNumber,
|
||||||
NamespaceProperty = K8sNamespace
|
NamespaceProperty = K8sTestNamespace
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -323,11 +416,16 @@ namespace KubernetesWorkflow
|
||||||
WaitUntil(() => !IsTestNamespaceOnline());
|
WaitUntil(() => !IsTestNamespaceOnline());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void WaitUntilNamespaceDeleted(string name)
|
||||||
|
{
|
||||||
|
WaitUntil(() => !IsNamespaceOnline(name));
|
||||||
|
}
|
||||||
|
|
||||||
private void WaitUntilDeploymentOnline(string deploymentName)
|
private void WaitUntilDeploymentOnline(string deploymentName)
|
||||||
{
|
{
|
||||||
WaitUntil(() =>
|
WaitUntil(() =>
|
||||||
{
|
{
|
||||||
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
|
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace));
|
||||||
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
|
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -336,7 +434,7 @@ namespace KubernetesWorkflow
|
||||||
{
|
{
|
||||||
WaitUntil(() =>
|
WaitUntil(() =>
|
||||||
{
|
{
|
||||||
var deployments = client.ListNamespacedDeployment(K8sNamespace);
|
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace));
|
||||||
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
|
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
|
||||||
return deployment == null || deployment.Status.AvailableReplicas == 0;
|
return deployment == null || deployment.Status.AvailableReplicas == 0;
|
||||||
});
|
});
|
||||||
|
@ -346,7 +444,7 @@ namespace KubernetesWorkflow
|
||||||
{
|
{
|
||||||
WaitUntil(() =>
|
WaitUntil(() =>
|
||||||
{
|
{
|
||||||
var pods = client.ListNamespacedPod(K8sNamespace).Items;
|
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
|
||||||
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
|
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
|
||||||
return pod == null;
|
return pod == null;
|
||||||
});
|
});
|
||||||
|
@ -369,7 +467,7 @@ namespace KubernetesWorkflow
|
||||||
|
|
||||||
private (string, string) FetchNewPod()
|
private (string, string) FetchNewPod()
|
||||||
{
|
{
|
||||||
var pods = client.ListNamespacedPod(K8sNamespace).Items;
|
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
|
||||||
|
|
||||||
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
|
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
|
||||||
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");
|
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<RootNamespace>KubernetesWorkflow</RootNamespace>
|
<RootNamespace>KubernetesWorkflow</RootNamespace>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
|
|
@ -8,14 +8,16 @@ namespace KubernetesWorkflow
|
||||||
private readonly WorkflowNumberSource numberSource;
|
private readonly WorkflowNumberSource numberSource;
|
||||||
private readonly K8sCluster cluster;
|
private readonly K8sCluster cluster;
|
||||||
private readonly KnownK8sPods knownK8SPods;
|
private readonly KnownK8sPods knownK8SPods;
|
||||||
|
private readonly string testNamespace;
|
||||||
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
|
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
|
||||||
|
|
||||||
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
|
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
this.numberSource = numberSource;
|
this.numberSource = numberSource;
|
||||||
this.cluster = cluster;
|
this.cluster = cluster;
|
||||||
this.knownK8SPods = knownK8SPods;
|
this.knownK8SPods = knownK8SPods;
|
||||||
|
this.testNamespace = testNamespace;
|
||||||
}
|
}
|
||||||
|
|
||||||
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
|
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
|
||||||
|
@ -62,10 +64,24 @@ namespace KubernetesWorkflow
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void DeleteTestResources()
|
||||||
|
{
|
||||||
|
K8s(controller =>
|
||||||
|
{
|
||||||
|
controller.DeleteTestNamespace();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
|
private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
|
||||||
{
|
{
|
||||||
log.Debug();
|
log.Debug();
|
||||||
return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r), startupConfig)).ToArray();
|
return recipes.Select(r =>
|
||||||
|
{
|
||||||
|
var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
|
||||||
|
log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
|
||||||
|
|
||||||
|
return new RunningContainer(runningPod, r, servicePorts, startupConfig);
|
||||||
|
}).ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
|
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
|
||||||
|
@ -82,14 +98,14 @@ namespace KubernetesWorkflow
|
||||||
|
|
||||||
private void K8s(Action<K8sController> action)
|
private void K8s(Action<K8sController> action)
|
||||||
{
|
{
|
||||||
var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
|
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
|
||||||
action(controller);
|
action(controller);
|
||||||
controller.Dispose();
|
controller.Dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
private T K8s<T>(Func<K8sController, T> action)
|
private T K8s<T>(Func<K8sController, T> action)
|
||||||
{
|
{
|
||||||
var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
|
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
|
||||||
var result = action(controller);
|
var result = action(controller);
|
||||||
controller.Dispose();
|
controller.Dispose();
|
||||||
return result;
|
return result;
|
||||||
|
|
|
@ -6,25 +6,26 @@ namespace KubernetesWorkflow
|
||||||
public class WorkflowCreator
|
public class WorkflowCreator
|
||||||
{
|
{
|
||||||
private readonly NumberSource numberSource = new NumberSource(0);
|
private readonly NumberSource numberSource = new NumberSource(0);
|
||||||
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
|
|
||||||
private readonly NumberSource containerNumberSource = new NumberSource(0);
|
private readonly NumberSource containerNumberSource = new NumberSource(0);
|
||||||
private readonly KnownK8sPods knownPods = new KnownK8sPods();
|
private readonly KnownK8sPods knownPods = new KnownK8sPods();
|
||||||
private readonly K8sCluster cluster;
|
private readonly K8sCluster cluster;
|
||||||
private readonly BaseLog log;
|
private readonly BaseLog log;
|
||||||
|
private readonly string testNamespace;
|
||||||
|
|
||||||
public WorkflowCreator(BaseLog log, Configuration configuration)
|
public WorkflowCreator(BaseLog log, Configuration configuration)
|
||||||
{
|
{
|
||||||
cluster = new K8sCluster(configuration);
|
cluster = new K8sCluster(configuration);
|
||||||
this.log = log;
|
this.log = log;
|
||||||
|
testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
|
||||||
}
|
}
|
||||||
|
|
||||||
public StartupWorkflow CreateWorkflow()
|
public StartupWorkflow CreateWorkflow()
|
||||||
{
|
{
|
||||||
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
|
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
|
||||||
servicePortNumberSource,
|
ApplicationLifecycle.Instance.GetServiceNumberSource(),
|
||||||
containerNumberSource);
|
containerNumberSource);
|
||||||
|
|
||||||
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods);
|
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<RootNamespace>Logging</RootNamespace>
|
<RootNamespace>Logging</RootNamespace>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
|
@ -10,16 +10,13 @@ namespace NethereumWorkflow
|
||||||
{
|
{
|
||||||
public class NethereumInteraction
|
public class NethereumInteraction
|
||||||
{
|
{
|
||||||
private readonly List<Task> openTasks = new List<Task>();
|
|
||||||
private readonly BaseLog log;
|
private readonly BaseLog log;
|
||||||
private readonly Web3 web3;
|
private readonly Web3 web3;
|
||||||
private readonly string rootAccount;
|
|
||||||
|
|
||||||
internal NethereumInteraction(BaseLog log, Web3 web3, string rootAccount)
|
internal NethereumInteraction(BaseLog log, Web3 web3)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
this.web3 = web3;
|
this.web3 = web3;
|
||||||
this.rootAccount = rootAccount;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public string GetTokenAddress(string marketplaceAddress)
|
public string GetTokenAddress(string marketplaceAddress)
|
||||||
|
@ -31,29 +28,13 @@ namespace NethereumWorkflow
|
||||||
return Time.Wait(handler.QueryAsync<string>(marketplaceAddress, function));
|
return Time.Wait(handler.QueryAsync<string>(marketplaceAddress, function));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void TransferWeiTo(string account, decimal amount)
|
public void MintTestTokens(string[] accounts, decimal amount, string tokenAddress)
|
||||||
{
|
{
|
||||||
log.Debug($"{amount} --> {account}");
|
if (amount < 1 || accounts.Length < 1) throw new ArgumentException("Invalid arguments for MintTestTokens");
|
||||||
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance");
|
|
||||||
|
|
||||||
var value = ToHexBig(amount);
|
var tasks = accounts.Select(a => MintTokens(a, amount, tokenAddress));
|
||||||
var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value));
|
|
||||||
openTasks.Add(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void MintTestTokens(string account, decimal amount, string tokenAddress)
|
Task.WaitAll(tasks.ToArray());
|
||||||
{
|
|
||||||
log.Debug($"({tokenAddress}) {amount} --> {account}");
|
|
||||||
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
|
|
||||||
|
|
||||||
var function = new MintTokensFunction
|
|
||||||
{
|
|
||||||
Holder = account,
|
|
||||||
Amount = ToBig(amount)
|
|
||||||
};
|
|
||||||
|
|
||||||
var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
|
|
||||||
openTasks.Add(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public decimal GetBalance(string tokenAddress, string account)
|
public decimal GetBalance(string tokenAddress, string account)
|
||||||
|
@ -68,38 +49,45 @@ namespace NethereumWorkflow
|
||||||
return ToDecimal(Time.Wait(handler.QueryAsync<BigInteger>(tokenAddress, function)));
|
return ToDecimal(Time.Wait(handler.QueryAsync<BigInteger>(tokenAddress, function)));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void WaitForAllTransactions()
|
public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
|
||||||
{
|
{
|
||||||
var tasks = openTasks.ToArray();
|
try
|
||||||
openTasks.Clear();
|
{
|
||||||
|
return IsBlockNumberOK() && IsContractAvailable(marketplaceAddress, marketplaceAbi);
|
||||||
Task.WaitAll(tasks);
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void EnsureSynced(string marketplaceAddress, string marketplaceAbi)
|
private Task MintTokens(string account, decimal amount, string tokenAddress)
|
||||||
{
|
{
|
||||||
WaitUntilSynced();
|
log.Debug($"({tokenAddress}) {amount} --> {account}");
|
||||||
WaitForContract(marketplaceAddress, marketplaceAbi);
|
if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
|
||||||
|
|
||||||
|
var function = new MintTokensFunction
|
||||||
|
{
|
||||||
|
Holder = account,
|
||||||
|
Amount = ToBig(amount)
|
||||||
|
};
|
||||||
|
|
||||||
|
var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
|
||||||
|
return handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void WaitUntilSynced()
|
private bool IsBlockNumberOK()
|
||||||
{
|
{
|
||||||
log.Debug();
|
log.Debug();
|
||||||
Time.WaitUntil(() =>
|
|
||||||
{
|
|
||||||
var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
|
var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
|
||||||
var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
|
var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
|
||||||
var numberOfBlocks = ToDecimal(number);
|
var numberOfBlocks = ToDecimal(number);
|
||||||
return !sync.IsSyncing && numberOfBlocks > 256;
|
return !sync.IsSyncing && numberOfBlocks > 256;
|
||||||
|
|
||||||
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void WaitForContract(string marketplaceAddress, string marketplaceAbi)
|
private bool IsContractAvailable(string marketplaceAddress, string marketplaceAbi)
|
||||||
{
|
{
|
||||||
log.Debug();
|
log.Debug();
|
||||||
Time.WaitUntil(() =>
|
|
||||||
{
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
|
var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
|
||||||
|
@ -109,7 +97,6 @@ namespace NethereumWorkflow
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private HexBigInteger ToHexBig(decimal amount)
|
private HexBigInteger ToHexBig(decimal amount)
|
||||||
|
|
|
@ -8,21 +8,19 @@ namespace NethereumWorkflow
|
||||||
private readonly BaseLog log;
|
private readonly BaseLog log;
|
||||||
private readonly string ip;
|
private readonly string ip;
|
||||||
private readonly int port;
|
private readonly int port;
|
||||||
private readonly string rootAccount;
|
|
||||||
private readonly string privateKey;
|
private readonly string privateKey;
|
||||||
|
|
||||||
public NethereumInteractionCreator(BaseLog log, string ip, int port, string rootAccount, string privateKey)
|
public NethereumInteractionCreator(BaseLog log, string ip, int port, string privateKey)
|
||||||
{
|
{
|
||||||
this.log = log;
|
this.log = log;
|
||||||
this.ip = ip;
|
this.ip = ip;
|
||||||
this.port = port;
|
this.port = port;
|
||||||
this.rootAccount = rootAccount;
|
|
||||||
this.privateKey = privateKey;
|
this.privateKey = privateKey;
|
||||||
}
|
}
|
||||||
|
|
||||||
public NethereumInteraction CreateWorkflow()
|
public NethereumInteraction CreateWorkflow()
|
||||||
{
|
{
|
||||||
return new NethereumInteraction(log, CreateWeb3(), rootAccount);
|
return new NethereumInteraction(log, CreateWeb3());
|
||||||
}
|
}
|
||||||
|
|
||||||
private Web3 CreateWeb3()
|
private Web3 CreateWeb3()
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<RootNamespace>NethereumWorkflow</RootNamespace>
|
<RootNamespace>NethereumWorkflow</RootNamespace>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
|
|
@ -71,13 +71,13 @@ namespace Tests.BasicTests
|
||||||
requiredCollateral: 10.TestTokens(),
|
requiredCollateral: 10.TestTokens(),
|
||||||
minRequiredNumberOfNodes: 1,
|
minRequiredNumberOfNodes: 1,
|
||||||
proofProbability: 5,
|
proofProbability: 5,
|
||||||
duration: TimeSpan.FromMinutes(2));
|
duration: TimeSpan.FromMinutes(1));
|
||||||
|
|
||||||
Time.Sleep(TimeSpan.FromMinutes(1));
|
Time.Sleep(TimeSpan.FromSeconds(10));
|
||||||
|
|
||||||
seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
|
seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
|
||||||
|
|
||||||
Time.Sleep(TimeSpan.FromMinutes(2));
|
Time.Sleep(TimeSpan.FromMinutes(1));
|
||||||
|
|
||||||
seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
|
seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
|
||||||
buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
|
buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
using DistTestCore;
|
||||||
|
using NUnit.Framework;
|
||||||
|
using Utils;
|
||||||
|
|
||||||
|
namespace Tests.BasicTests
|
||||||
|
{
|
||||||
|
// Warning!
|
||||||
|
// This is a test to check network-isolation in the test-infrastructure.
|
||||||
|
// It requires parallelism(2) or greater to run.
|
||||||
|
[TestFixture]
|
||||||
|
public class NetworkIsolationTest : DistTest
|
||||||
|
{
|
||||||
|
private IOnlineCodexNode? node = null;
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void SetUpANodeAndWait()
|
||||||
|
{
|
||||||
|
node = SetupCodexNode();
|
||||||
|
|
||||||
|
Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ForeignNodeConnects()
|
||||||
|
{
|
||||||
|
var myNode = SetupCodexNode();
|
||||||
|
|
||||||
|
Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
myNode.ConnectToPeer(node!);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Good! This connection should be prohibited by the network isolation policy.
|
||||||
|
node = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " +
|
||||||
|
"This may cause cross-test interference. Network isolation policy should be applied. Test infra failure.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -72,13 +72,13 @@ namespace Tests.BasicTests
|
||||||
|
|
||||||
private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
|
private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
|
||||||
{
|
{
|
||||||
//var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
|
var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
|
||||||
var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
|
var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
|
||||||
|
|
||||||
//Log.Debug($"Looking for {b.id} in engine-peers [{enginePeers}]");
|
Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
|
||||||
Log.Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
|
Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
|
||||||
|
|
||||||
//Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
|
Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
|
||||||
Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
|
Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,15 +7,23 @@ namespace Tests.BasicTests
|
||||||
[TestFixture]
|
[TestFixture]
|
||||||
public class TwoClientTests : DistTest
|
public class TwoClientTests : DistTest
|
||||||
{
|
{
|
||||||
[Test]
|
[TestCase(1)]
|
||||||
public void TwoClientsOnePodTest()
|
[TestCase(2)]
|
||||||
|
[TestCase(3)]
|
||||||
|
[TestCase(4)]
|
||||||
|
[TestCase(5)]
|
||||||
|
[TestCase(6)]
|
||||||
|
[TestCase(7)]
|
||||||
|
[TestCase(8)]
|
||||||
|
[TestCase(9)]
|
||||||
|
public void TwoClientsOnePodTest(int size)
|
||||||
{
|
{
|
||||||
var group = SetupCodexNodes(2);
|
var group = SetupCodexNodes(2);
|
||||||
|
|
||||||
var primary = group[0];
|
var primary = group[0];
|
||||||
var secondary = group[1];
|
var secondary = group[1];
|
||||||
|
|
||||||
PerformTwoClientTest(primary, secondary);
|
PerformTwoClientTest(primary, secondary, size.MB());
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
|
@ -38,10 +46,15 @@ namespace Tests.BasicTests
|
||||||
}
|
}
|
||||||
|
|
||||||
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
|
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
|
||||||
|
{
|
||||||
|
PerformTwoClientTest(primary, secondary, 1.MB());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary, ByteSize size)
|
||||||
{
|
{
|
||||||
primary.ConnectToPeer(secondary);
|
primary.ConnectToPeer(secondary);
|
||||||
|
|
||||||
var testFile = GenerateTestFile(1.MB());
|
var testFile = GenerateTestFile(size);
|
||||||
|
|
||||||
var contentId = primary.UploadFile(testFile);
|
var contentId = primary.UploadFile(testFile);
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
[assembly: LevelOfParallelism(1)]
|
||||||
|
namespace Tests
|
||||||
|
{
|
||||||
|
}
|
|
@ -1,7 +1,7 @@
|
||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
<RootNamespace>Utils</RootNamespace>
|
<RootNamespace>Utils</RootNamespace>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
|
Loading…
Reference in New Issue