Merge branch 'feature/parallel-tests'

This commit is contained in:
benbierens 2023-05-05 08:33:24 +02:00
commit 3aba6d5082
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
49 changed files with 631 additions and 359 deletions

View File

@ -6,10 +6,12 @@ namespace DistTestCore.Codex
public class CodexAccess public class CodexAccess
{ {
private readonly BaseLog log; private readonly BaseLog log;
private readonly ITimeSet timeSet;
public CodexAccess(BaseLog log, RunningContainer runningContainer) public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
{ {
this.log = log; this.log = log;
this.timeSet = timeSet;
Container = runningContainer; Container = runningContainer;
} }
@ -40,11 +42,29 @@ namespace DistTestCore.Codex
return Http().HttpPostJson($"storage/request/{contentId}", request); return Http().HttpPostJson($"storage/request/{contentId}", request);
} }
public void EnsureOnline()
{
try
{
var debugInfo = GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
var nodePeerId = debugInfo.id;
var nodeName = Container.Name;
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
}
catch (Exception e)
{
log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
}
}
private Http Http() private Http Http()
{ {
var ip = Container.Pod.Cluster.IP; var ip = Container.Pod.Cluster.IP;
var port = Container.ServicePorts[0].Number; var port = Container.ServicePorts[0].Number;
return new Http(log, ip, port, baseUrl: "/api/codex/v1"); return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
} }
public string ConnectToPeer(string peerId, string peerMultiAddress) public string ConnectToPeer(string peerId, string peerMultiAddress)

View File

@ -1,12 +1,17 @@
using DistTestCore.Marketplace; using System.Runtime.InteropServices;
using DistTestCore.Marketplace;
using KubernetesWorkflow; using KubernetesWorkflow;
namespace DistTestCore.Codex namespace DistTestCore.Codex
{ {
public class CodexContainerRecipe : ContainerRecipeFactory public class CodexContainerRecipe : ContainerRecipeFactory
{ {
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635"; #if Arm64
public const string DockerImage = "thatbenbierens/codexlocal:latest"; public const string DockerImage = "emizzle/nim-codex-arm64:sha-c7af585";
#else
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
public const string DockerImage = "thatbenbierens/codexlocal:latest";
#endif
public const string MetricsPortTag = "metrics_port"; public const string MetricsPortTag = "metrics_port";
protected override string Image => DockerImage; protected override string Image => DockerImage;

View File

@ -62,29 +62,15 @@ namespace DistTestCore
return $"group:[{Containers.Describe()}]"; return $"group:[{Containers.Describe()}]";
} }
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) public void EnsureOnline()
{ {
var access = new CodexAccess(lifecycle.Log, c); foreach (var node in Nodes) node.CodexAccess.EnsureOnline();
EnsureOnline(access);
return factory.CreateOnlineCodexNode(access, this);
} }
private void EnsureOnline(CodexAccess access) private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{ {
try var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
{ return factory.CreateOnlineCodexNode(access, this);
var debugInfo = access.GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
var nodePeerId = debugInfo.id;
var nodeName = access.Container.Name;
lifecycle.Log.AddStringReplace(nodePeerId, $"___{nodeName}___");
}
catch (Exception e)
{
lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
}
} }
} }
} }

View File

@ -44,7 +44,7 @@ namespace DistTestCore
public void DeleteAllResources() public void DeleteAllResources()
{ {
var workflow = CreateWorkflow(); var workflow = CreateWorkflow();
workflow.DeleteAllResources(); workflow.DeleteTestResources();
RunningGroups.Clear(); RunningGroups.Clear();
} }
@ -74,6 +74,7 @@ namespace DistTestCore
{ {
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
RunningGroups.Add(group); RunningGroups.Add(group);
group.EnsureOnline();
return group; return group;
} }

View File

@ -4,13 +4,13 @@ namespace DistTestCore
{ {
public class Configuration public class Configuration
{ {
public KubernetesWorkflow.Configuration GetK8sConfiguration() public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{ {
return new KubernetesWorkflow.Configuration( return new KubernetesWorkflow.Configuration(
k8sNamespace: "codex-test-ns", k8sNamespacePrefix: "ct-",
kubeConfigFile: null, kubeConfigFile: null,
operationTimeout: Timing.K8sOperationTimeout(), operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: Timing.K8sServiceDelay(), retryDelay: timeSet.WaitForK8sServiceDelay(),
locationMap: new[] locationMap: new[]
{ {
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"), new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),

View File

@ -6,23 +6,25 @@ using KubernetesWorkflow;
using Logging; using Logging;
using NUnit.Framework; using NUnit.Framework;
using System.Reflection; using System.Reflection;
using Utils;
namespace DistTestCore namespace DistTestCore
{ {
[SetUpFixture] [SetUpFixture]
[Parallelizable(ParallelScope.All)]
public abstract class DistTest public abstract class DistTest
{ {
private readonly Configuration configuration = new Configuration(); private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies; private readonly Assembly[] testAssemblies;
private FixtureLog fixtureLog = null!; private readonly FixtureLog fixtureLog;
private TestLifecycle lifecycle = null!; private readonly object lifecycleLock = new object();
private DateTime testStart = DateTime.MinValue; private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
public DistTest() public DistTest()
{ {
var assemblies = AppDomain.CurrentDomain.GetAssemblies(); var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
} }
[OneTimeSetUp] [OneTimeSetUp]
@ -30,14 +32,11 @@ namespace DistTestCore
{ {
// Previous test run may have been interrupted. // Previous test run may have been interrupted.
// Begin by cleaning everything up. // Begin by cleaning everything up.
Timing.UseLongTimeouts = false;
fixtureLog = new FixtureLog(configuration.GetLogConfig());
try try
{ {
Stopwatch.Measure(fixtureLog, "Global setup", () => Stopwatch.Measure(fixtureLog, "Global setup", () =>
{ {
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration()); var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
wc.CreateWorkflow().DeleteAllResources(); wc.CreateWorkflow().DeleteAllResources();
}); });
} }
@ -57,8 +56,6 @@ namespace DistTestCore
[SetUp] [SetUp]
public void SetUpDistTest() public void SetUpDistTest()
{ {
Timing.UseLongTimeouts = ShouldUseLongTimeouts();
if (GlobalTestFailure.HasFailed) if (GlobalTestFailure.HasFailed)
{ {
Assert.Inconclusive("Skip test: Previous test failed during clean up."); Assert.Inconclusive("Skip test: Previous test failed during clean up.");
@ -85,7 +82,7 @@ namespace DistTestCore
public TestFile GenerateTestFile(ByteSize size) public TestFile GenerateTestFile(ByteSize size)
{ {
return lifecycle.FileManager.GenerateTestFile(size); return Get().FileManager.GenerateTestFile(size);
} }
public IOnlineCodexNode SetupCodexBootstrapNode() public IOnlineCodexNode SetupCodexBootstrapNode()
@ -128,12 +125,58 @@ namespace DistTestCore
public ICodexNodeGroup BringOnline(ICodexSetup codexSetup) public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
{ {
return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup); return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
} }
protected BaseLog Log protected void Log(string msg)
{ {
get { return lifecycle.Log; } TestContext.Progress.WriteLine(msg);
Get().Log.Log(msg);
}
protected void Debug(string msg)
{
TestContext.Progress.WriteLine(msg);
Get().Log.Debug(msg);
}
private TestLifecycle Get()
{
lock (lifecycleLock)
{
return lifecycles[GetCurrentTestName()];
}
}
private void CreateNewTestLifecycle()
{
var testName = GetCurrentTestName();
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
{
lock (lifecycleLock)
{
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
}
});
}
private void DisposeTestLifecycle()
{
var lifecycle = Get();
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure(lifecycle);
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private ITimeSet GetTimeSet()
{
if (ShouldUseLongTimeouts()) return new LongTimeSet();
return new DefaultTimeSet();
} }
private bool ShouldUseLongTimeouts() private bool ShouldUseLongTimeouts()
@ -151,28 +194,7 @@ namespace DistTestCore
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null); return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
} }
private void CreateNewTestLifecycle() private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
{
Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
{
lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
testStart = DateTime.UtcNow;
});
}
private void DisposeTestLifecycle()
{
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure();
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private void IncludeLogsAndMetricsOnTestFailure()
{ {
var result = TestContext.CurrentContext.Result; var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
@ -182,8 +204,8 @@ namespace DistTestCore
if (IsDownloadingLogsAndMetricsEnabled()) if (IsDownloadingLogsAndMetricsEnabled())
{ {
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure..."); lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
DownloadAllLogs(); DownloadAllLogs(lifecycle);
DownloadAllMetrics(); DownloadAllMetrics(lifecycle);
} }
else else
{ {
@ -192,25 +214,19 @@ namespace DistTestCore
} }
} }
private string GetTestDuration() private void DownloadAllLogs(TestLifecycle lifecycle)
{ {
var testDuration = DateTime.UtcNow - testStart; OnEachCodexNode(lifecycle, node =>
return Time.FormatDuration(testDuration);
}
private void DownloadAllLogs()
{
OnEachCodexNode(node =>
{ {
lifecycle.DownloadLog(node); lifecycle.DownloadLog(node);
}); });
} }
private void DownloadAllMetrics() private void DownloadAllMetrics(TestLifecycle lifecycle)
{ {
var metricsDownloader = new MetricsDownloader(lifecycle.Log); var metricsDownloader = new MetricsDownloader(lifecycle.Log);
OnEachCodexNode(node => OnEachCodexNode(lifecycle, node =>
{ {
var m = node.Metrics as MetricsAccess; var m = node.Metrics as MetricsAccess;
if (m != null) if (m != null)
@ -220,7 +236,7 @@ namespace DistTestCore
}); });
} }
private void OnEachCodexNode(Action<OnlineCodexNode> action) private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
{ {
var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
foreach (var node in allNodes) foreach (var node in allNodes)

View File

@ -1,10 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<RootNamespace>DistTestCore</RootNamespace> <RootNamespace>DistTestCore</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
</PropertyGroup>
<PropertyGroup Condition="'$(IsArm64)'=='true'">
<DefineConstants>Arm64</DefineConstants>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>

View File

@ -1,5 +1,6 @@
using Logging; using Logging;
using NUnit.Framework; using NUnit.Framework;
using Utils;
namespace DistTestCore namespace DistTestCore
{ {
@ -13,13 +14,14 @@ namespace DistTestCore
public class FileManager : IFileManager public class FileManager : IFileManager
{ {
public const int ChunkSize = 1024 * 1024; public const int ChunkSize = 1024 * 1024;
private static NumberSource folderNumberSource = new NumberSource(0);
private readonly Random random = new Random(); private readonly Random random = new Random();
private readonly TestLog log; private readonly TestLog log;
private readonly string folder; private readonly string folder;
public FileManager(TestLog log, Configuration configuration) public FileManager(TestLog log, Configuration configuration)
{ {
folder = configuration.GetFileManagerFolder(); folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
EnsureDirectory(); EnsureDirectory();
this.log = log; this.log = log;

View File

@ -36,13 +36,8 @@ namespace DistTestCore
var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log); var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
foreach (var account in companionNode.Accounts) var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
{ interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei);
interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
}
interaction.WaitForAllTransactions();
} }
private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode) private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)

View File

@ -10,13 +10,15 @@ namespace DistTestCore
public class Http public class Http
{ {
private readonly BaseLog log; private readonly BaseLog log;
private readonly ITimeSet timeSet;
private readonly string ip; private readonly string ip;
private readonly int port; private readonly int port;
private readonly string baseUrl; private readonly string baseUrl;
public Http(BaseLog log, string ip, int port, string baseUrl) public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
{ {
this.log = log; this.log = log;
this.timeSet = timeSet;
this.ip = ip; this.ip = ip;
this.port = port; this.port = port;
this.baseUrl = baseUrl; this.baseUrl = baseUrl;
@ -103,7 +105,7 @@ namespace DistTestCore
log.Debug($"({url}) = '{message}'", 3); log.Debug($"({url}) = '{message}'", 3);
} }
private static T Retry<T>(Func<T> operation) private T Retry<T>(Func<T> operation)
{ {
var retryCounter = 0; var retryCounter = 0;
@ -115,9 +117,9 @@ namespace DistTestCore
} }
catch (Exception exception) catch (Exception exception)
{ {
Timing.HttpCallRetryDelay(); timeSet.HttpCallRetryDelay();
retryCounter++; retryCounter++;
if (retryCounter > Timing.HttpCallRetryCount()) if (retryCounter > timeSet.HttpCallRetryCount())
{ {
Assert.Fail(exception.ToString()); Assert.Fail(exception.ToString());
throw; throw;
@ -140,10 +142,10 @@ namespace DistTestCore
} }
} }
private static HttpClient GetClient() private HttpClient GetClient()
{ {
var client = new HttpClient(); var client = new HttpClient();
client.Timeout = Timing.HttpCallTimeout(); client.Timeout = timeSet.HttpCallTimeout();
return client; return client;
} }
} }

View File

@ -4,7 +4,11 @@ namespace DistTestCore.Marketplace
{ {
public class CodexContractsContainerRecipe : ContainerRecipeFactory public class CodexContractsContainerRecipe : ContainerRecipeFactory
{ {
public const string DockerImage = "thatbenbierens/codex-contracts-deployment"; #if Arm64
public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
#else
public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint";
#endif
public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json"; public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";

View File

@ -19,13 +19,14 @@ namespace DistTestCore.Marketplace
this.container = container; this.container = container;
} }
public string ExtractAccount(int? orderNumber) public AllGethAccounts ExtractAccounts()
{ {
log.Debug(); log.Debug();
var account = Retry(() => FetchAccount(orderNumber)); var accountsCsv = Retry(() => FetchAccountsCsv());
if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); if (string.IsNullOrEmpty(accountsCsv)) throw new InvalidOperationException("Unable to fetch accounts.csv for geth node. Test infra failure.");
return account; var lines = accountsCsv.Split('\n');
return new AllGethAccounts(lines.Select(ParseLineToAccount).ToArray());
} }
public string ExtractPubKey() public string ExtractPubKey()
@ -37,15 +38,6 @@ namespace DistTestCore.Marketplace
return pubKey; return pubKey;
} }
public string ExtractPrivateKey(int? orderNumber)
{
log.Debug();
var privKey = Retry(() => FetchPrivateKey(orderNumber));
if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
return privKey;
}
public string ExtractMarketplaceAddress() public string ExtractMarketplaceAddress()
{ {
log.Debug(); log.Debug();
@ -88,14 +80,9 @@ namespace DistTestCore.Marketplace
} }
} }
private string FetchAccount(int? orderNumber) private string FetchAccountsCsv()
{ {
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber)); return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
}
private string FetchPrivateKey(int? orderNumber)
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber));
} }
private string FetchMarketplaceAddress() private string FetchMarketplaceAddress()
@ -120,6 +107,15 @@ namespace DistTestCore.Marketplace
workflow.DownloadContainerLog(container, enodeFinder); workflow.DownloadContainerLog(container, enodeFinder);
return enodeFinder.GetPubKey(); return enodeFinder.GetPubKey();
} }
private GethAccount ParseLineToAccount(string l)
{
var tokens = l.Replace("\r", "").Split(',');
if (tokens.Length != 2) throw new InvalidOperationException();
var account = tokens[0];
var privateKey = tokens[1];
return new GethAccount(account, privateKey);
}
} }
public class PubKeyFinder : LogHandler, ILogHandler public class PubKeyFinder : LogHandler, ILogHandler

View File

@ -6,19 +6,19 @@ namespace DistTestCore.Marketplace
{ {
public class GethBootstrapNodeInfo public class GethBootstrapNodeInfo
{ {
public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string pubKey, string privateKey, Port discoveryPort) public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
{ {
RunningContainers = runningContainers; RunningContainers = runningContainers;
Account = account; AllAccounts = allAccounts;
Account = allAccounts.Accounts[0];
PubKey = pubKey; PubKey = pubKey;
PrivateKey = privateKey;
DiscoveryPort = discoveryPort; DiscoveryPort = discoveryPort;
} }
public RunningContainers RunningContainers { get; } public RunningContainers RunningContainers { get; }
public string Account { get; } public AllGethAccounts AllAccounts { get; }
public GethAccount Account { get; }
public string PubKey { get; } public string PubKey { get; }
public string PrivateKey { get; }
public Port DiscoveryPort { get; } public Port DiscoveryPort { get; }
public NethereumInteraction StartInteraction(BaseLog log) public NethereumInteraction StartInteraction(BaseLog log)
@ -26,10 +26,19 @@ namespace DistTestCore.Marketplace
var ip = RunningContainers.RunningPod.Cluster.IP; var ip = RunningContainers.RunningPod.Cluster.IP;
var port = RunningContainers.Containers[0].ServicePorts[0].Number; var port = RunningContainers.Containers[0].ServicePorts[0].Number;
var account = Account; var account = Account;
var privateKey = PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey); var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey);
return creator.CreateWorkflow(); return creator.CreateWorkflow();
} }
} }
public class AllGethAccounts
{
public GethAccount[] Accounts { get; }
public AllGethAccounts(GethAccount[] accounts)
{
Accounts = accounts;
}
}
} }

View File

@ -20,20 +20,20 @@ namespace DistTestCore.Marketplace
var bootstrapContainer = containers.Containers[0]; var bootstrapContainer = containers.Containers[0];
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer); var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
var account = extractor.ExtractAccount(null); var accounts = extractor.ExtractAccounts();
var pubKey = extractor.ExtractPubKey(); var pubKey = extractor.ExtractPubKey();
var privateKey = extractor.ExtractPrivateKey(null);
var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
LogEnd($"Geth bootstrap node started with account '{account}'"); LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
return new GethBootstrapNodeInfo(containers, account, pubKey, privateKey, discoveryPort); return result;
} }
private StartupConfig CreateBootstrapStartupConfig() private StartupConfig CreateBootstrapStartupConfig()
{ {
var config = new StartupConfig(); var config = new StartupConfig();
config.Add(new GethStartupConfig(true, null!, 0)); config.Add(new GethStartupConfig(true, null!, 0, 0));
return config; return config;
} }
} }

View File

@ -6,30 +6,29 @@ namespace DistTestCore.Marketplace
{ {
public class GethCompanionNodeInfo public class GethCompanionNodeInfo
{ {
public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts) public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
{ {
RunningContainer = runningContainer; RunningContainer = runningContainer;
Accounts = accounts; Accounts = accounts;
} }
public RunningContainer RunningContainer { get; } public RunningContainer RunningContainer { get; }
public GethCompanionAccount[] Accounts { get; } public GethAccount[] Accounts { get; }
public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account) public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
{ {
var ip = RunningContainer.Pod.Cluster.IP; var ip = RunningContainer.Pod.Cluster.IP;
var port = RunningContainer.ServicePorts[0].Number; var port = RunningContainer.ServicePorts[0].Number;
var accountStr = account.Account;
var privateKey = account.PrivateKey; var privateKey = account.PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey); var creator = new NethereumInteractionCreator(log, ip, port, privateKey);
return creator.CreateWorkflow(); return creator.CreateWorkflow();
} }
} }
public class GethCompanionAccount public class GethAccount
{ {
public GethCompanionAccount(string account, string privateKey) public GethAccount(string account, string privateKey)
{ {
Account = account; Account = account;
PrivateKey = privateKey; PrivateKey = privateKey;

View File

@ -5,6 +5,8 @@ namespace DistTestCore.Marketplace
{ {
public class GethCompanionNodeStarter : BaseStarter public class GethCompanionNodeStarter : BaseStarter
{ {
private int companionAccountIndex = 0;
public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator) : base(lifecycle, workflowCreator)
{ {
@ -14,53 +16,43 @@ namespace DistTestCore.Marketplace
{ {
LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes."); LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes); var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
var workflow = workflowCreator.CreateWorkflow(); var workflow = workflowCreator.CreateWorkflow();
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
WaitForAccountCreation(codexSetup.NumberOfNodes);
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure."); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
var container = containers.Containers[0]; var container = containers.Containers[0];
var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes); var node = CreateCompanionInfo(container, marketplace, config);
EnsureCompanionNodeIsSynced(node, marketplace); EnsureCompanionNodeIsSynced(node, marketplace);
LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]"); LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
return node; return node;
} }
private void WaitForAccountCreation(int numberOfNodes) private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
{ {
// We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files var accounts = ExtractAccounts(marketplace, config);
// we will be trying to read in 'ExtractAccount', later on in the start-up process.
Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes));
}
private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts)
{
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray();
return new GethCompanionNodeInfo(container, accounts); return new GethCompanionNodeInfo(container, accounts);
} }
private IEnumerable<GethCompanionAccount> ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts) private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
{ {
for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1); return marketplace.Bootstrap.AllAccounts.Accounts
} .Skip(1 + config.CompanionAccountStartIndex)
.Take(config.NumberOfCompanionAccounts)
private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber) .ToArray();
{
var account = extractor.ExtractAccount(orderNumber);
var privKey = extractor.ExtractPrivateKey(orderNumber);
return new GethCompanionAccount(account, privKey);
} }
private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace) private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
{ {
try try
{ {
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First()); Time.WaitUntil(() =>
interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi); {
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
} }
catch (Exception e) catch (Exception e)
{ {
@ -68,10 +60,17 @@ namespace DistTestCore.Marketplace
} }
} }
private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts) private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
{
var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
companionAccountIndex += numberOfAccounts;
return config;
}
private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
{ {
var config = new StartupConfig(); var config = new StartupConfig();
config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts)); config.Add(gethConfig);
return config; return config;
} }
} }

View File

@ -4,22 +4,17 @@ namespace DistTestCore.Marketplace
{ {
public class GethContainerRecipe : ContainerRecipeFactory public class GethContainerRecipe : ContainerRecipeFactory
{ {
public const string DockerImage = "thatbenbierens/geth-confenv:latest"; #if Arm64
public const string DockerImage = "emizzle/geth-confenv:latest";
#else
public const string DockerImage = "thatbenbierens/geth-confenv:onethousand";
#endif
public const string HttpPortTag = "http_port"; public const string HttpPortTag = "http_port";
public const string DiscoveryPortTag = "disc_port"; public const string DiscoveryPortTag = "disc_port";
private const string defaultArgs = "--ipcdisable --syncmode full"; private const string defaultArgs = "--ipcdisable --syncmode full";
public static string GetAccountFilename(int? orderNumber) public const string AccountsFilename = "accounts.csv";
{
if (orderNumber == null) return "account_string.txt";
return $"account_string_{orderNumber.Value}.txt";
}
public static string GetPrivateKeyFilename(int? orderNumber)
{
if (orderNumber == null) return "private.key";
return $"private_{orderNumber.Value}.key";
}
protected override string Image => DockerImage; protected override string Image => DockerImage;
@ -46,14 +41,17 @@ namespace DistTestCore.Marketplace
private string CreateBootstapArgs(Port discovery) private string CreateBootstapArgs(Port discovery)
{ {
AddEnvVar("IS_BOOTSTRAP", "1"); AddEnvVar("ENABLE_MINER", "1");
UnlockAccounts(0, 1);
var exposedPort = AddExposedPort(tag: HttpPortTag); var exposedPort = AddExposedPort(tag: HttpPortTag);
return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}"; return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
} }
private string CreateCompanionArgs(Port discovery, GethStartupConfig config) private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
{ {
AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString()); UnlockAccounts(
config.CompanionAccountStartIndex + 1,
config.NumberOfCompanionAccounts);
var port = AddInternalPort(); var port = AddInternalPort();
var authRpc = AddInternalPort(); var authRpc = AddInternalPort();
@ -66,5 +64,15 @@ namespace DistTestCore.Marketplace
return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}"; return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
} }
private void UnlockAccounts(int startIndex, int numberOfAccounts)
{
if (startIndex < 0) throw new ArgumentException();
if (numberOfAccounts < 1) throw new ArgumentException();
if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
}
} }
} }

View File

@ -2,15 +2,17 @@
{ {
public class GethStartupConfig public class GethStartupConfig
{ {
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts) public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
{ {
IsBootstrapNode = isBootstrapNode; IsBootstrapNode = isBootstrapNode;
BootstrapNode = bootstrapNode; BootstrapNode = bootstrapNode;
CompanionAccountStartIndex = companionAccountStartIndex;
NumberOfCompanionAccounts = numberOfCompanionAccounts; NumberOfCompanionAccounts = numberOfCompanionAccounts;
} }
public bool IsBootstrapNode { get; } public bool IsBootstrapNode { get; }
public GethBootstrapNodeInfo BootstrapNode { get; } public GethBootstrapNodeInfo BootstrapNode { get; }
public int CompanionAccountStartIndex { get; }
public int NumberOfCompanionAccounts { get; } public int NumberOfCompanionAccounts { get; }
} }
} }

View File

@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace
{ {
private readonly TestLog log; private readonly TestLog log;
private readonly MarketplaceNetwork marketplaceNetwork; private readonly MarketplaceNetwork marketplaceNetwork;
private readonly GethCompanionAccount account; private readonly GethAccount account;
private readonly CodexAccess codexAccess; private readonly CodexAccess codexAccess;
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess) public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
{ {
this.log = log; this.log = log;
this.marketplaceNetwork = marketplaceNetwork; this.marketplaceNetwork = marketplaceNetwork;

View File

@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace
return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access); return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
} }
private GethCompanionAccount GetGethCompanionNode(CodexAccess access) private GethAccount GetGethCompanionNode(CodexAccess access)
{ {
var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount); var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
return (GethCompanionAccount)account; return (GethAccount)account;
} }
} }
} }

View File

@ -14,12 +14,14 @@ namespace DistTestCore.Metrics
public class MetricsAccess : IMetricsAccess public class MetricsAccess : IMetricsAccess
{ {
private readonly TestLog log; private readonly TestLog log;
private readonly ITimeSet timeSet;
private readonly MetricsQuery query; private readonly MetricsQuery query;
private readonly RunningContainer node; private readonly RunningContainer node;
public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node) public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
{ {
this.log = log; this.log = log;
this.timeSet = timeSet;
this.query = query; this.query = query;
this.node = node; this.node = node;
} }
@ -47,7 +49,7 @@ namespace DistTestCore.Metrics
{ {
var mostRecent = GetMostRecent(metricName); var mostRecent = GetMostRecent(metricName);
if (mostRecent != null) return mostRecent; if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
{ {
Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
throw new TimeoutException(); throw new TimeoutException();

View File

@ -28,8 +28,8 @@ namespace DistTestCore.Metrics
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{ {
var query = new MetricsQuery(lifecycle.Log, prometheusContainer); var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
return new MetricsAccess(lifecycle.Log, query, codexContainer); return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
} }
} }
} }

View File

@ -9,12 +9,13 @@ namespace DistTestCore.Metrics
{ {
private readonly Http http; private readonly Http http;
public MetricsQuery(BaseLog log, RunningContainers runningContainers) public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
{ {
RunningContainers = runningContainers; RunningContainers = runningContainers;
http = new Http( http = new Http(
log, log,
timeSet,
runningContainers.RunningPod.Cluster.IP, runningContainers.RunningPod.Cluster.IP,
runningContainers.Containers[0].ServicePorts[0].Number, runningContainers.Containers[0].ServicePorts[0].Number,
"api/v1"); "api/v1");

View File

@ -92,6 +92,10 @@ namespace DistTestCore
public ICodexSetup BringOffline() public ICodexSetup BringOffline()
{ {
if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
"individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
"available for codex-nodes in groups of 1.");
return Group.BringOffline(); return Group.BringOffline();
} }

View File

@ -1,25 +1,30 @@
using DistTestCore.Logs; using DistTestCore.Logs;
using KubernetesWorkflow; using KubernetesWorkflow;
using Logging; using Logging;
using Utils;
namespace DistTestCore namespace DistTestCore
{ {
public class TestLifecycle public class TestLifecycle
{ {
private readonly WorkflowCreator workflowCreator; private readonly WorkflowCreator workflowCreator;
private DateTime testStart = DateTime.MinValue;
public TestLifecycle(TestLog log, Configuration configuration) public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
{ {
Log = log; Log = log;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration()); TimeSet = timeSet;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
FileManager = new FileManager(Log, configuration); FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this, workflowCreator); CodexStarter = new CodexStarter(this, workflowCreator);
PrometheusStarter = new PrometheusStarter(this, workflowCreator); PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator); GethStarter = new GethStarter(this, workflowCreator);
testStart = DateTime.UtcNow;
} }
public TestLog Log { get; } public TestLog Log { get; }
public ITimeSet TimeSet { get; }
public FileManager FileManager { get; } public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; } public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; } public PrometheusStarter PrometheusStarter { get; }
@ -42,5 +47,11 @@ namespace DistTestCore
return new CodexNodeLog(subFile, node); return new CodexNodeLog(subFile, node);
} }
public string GetTestDuration()
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
}
} }
} }

View File

@ -8,53 +8,11 @@ namespace DistTestCore
{ {
} }
public static class Timing
{
public static bool UseLongTimeouts { get; set; }
public static TimeSpan HttpCallTimeout()
{
return GetTimes().HttpCallTimeout();
}
public static int HttpCallRetryCount()
{
return GetTimes().HttpCallRetryCount();
}
public static void HttpCallRetryDelay()
{
Time.Sleep(GetTimes().HttpCallRetryDelay());
}
public static TimeSpan K8sServiceDelay()
{
return GetTimes().WaitForK8sServiceDelay();
}
public static TimeSpan K8sOperationTimeout()
{
return GetTimes().K8sOperationTimeout();
}
public static TimeSpan WaitForMetricTimeout()
{
return GetTimes().WaitForMetricTimeout();
}
private static ITimeSet GetTimes()
{
if (UseLongTimeouts) return new LongTimeSet();
return new DefaultTimeSet();
}
}
public interface ITimeSet public interface ITimeSet
{ {
TimeSpan HttpCallTimeout(); TimeSpan HttpCallTimeout();
int HttpCallRetryCount(); int HttpCallRetryCount();
TimeSpan HttpCallRetryDelay(); void HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay(); TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout(); TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout(); TimeSpan WaitForMetricTimeout();
@ -72,9 +30,9 @@ namespace DistTestCore
return 5; return 5;
} }
public TimeSpan HttpCallRetryDelay() public void HttpCallRetryDelay()
{ {
return TimeSpan.FromSeconds(3); Time.Sleep(TimeSpan.FromSeconds(3));
} }
public TimeSpan WaitForK8sServiceDelay() public TimeSpan WaitForK8sServiceDelay()
@ -105,9 +63,9 @@ namespace DistTestCore
return 2; return 2;
} }
public TimeSpan HttpCallRetryDelay() public void HttpCallRetryDelay()
{ {
return TimeSpan.FromMinutes(5); Time.Sleep(TimeSpan.FromMinutes(5));
} }
public TimeSpan WaitForK8sServiceDelay() public TimeSpan WaitForK8sServiceDelay()

View File

@ -0,0 +1,40 @@
using Utils;
namespace KubernetesWorkflow
{
public class ApplicationLifecycle
{
private static object instanceLock = new object();
private static ApplicationLifecycle? instance;
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource namespaceNumberSource = new NumberSource(0);
private ApplicationLifecycle()
{
}
public static ApplicationLifecycle Instance
{
// I know singletons are quite evil. But we need to be sure this object is created only once
// and persists for the entire application lifecycle.
get
{
lock (instanceLock)
{
if (instance == null) instance = new ApplicationLifecycle();
return instance;
}
}
}
public NumberSource GetServiceNumberSource()
{
return servicePortNumberSource;
}
public string GetTestNamespace()
{
return namespaceNumberSource.GetNextNumber().ToString("D5");
}
}
}

View File

@ -5,7 +5,7 @@ namespace KubernetesWorkflow
{ {
public class CommandRunner public class CommandRunner
{ {
private readonly Kubernetes client; private readonly K8sClient client;
private readonly string k8sNamespace; private readonly string k8sNamespace;
private readonly RunningPod pod; private readonly RunningPod pod;
private readonly string containerName; private readonly string containerName;
@ -13,7 +13,7 @@ namespace KubernetesWorkflow
private readonly string[] arguments; private readonly string[] arguments;
private readonly List<string> lines = new List<string>(); private readonly List<string> lines = new List<string>();
public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments) public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
{ {
this.client = client; this.client = client;
this.k8sNamespace = k8sNamespace; this.k8sNamespace = k8sNamespace;
@ -27,8 +27,8 @@ namespace KubernetesWorkflow
{ {
var input = new[] { command }.Concat(arguments).ToArray(); var input = new[] { command }.Concat(arguments).ToArray();
Time.Wait(client.NamespacedPodExecAsync( Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())); pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
} }
public string GetStdOut() public string GetStdOut()

View File

@ -2,16 +2,16 @@
{ {
public class Configuration public class Configuration
{ {
public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
{ {
K8sNamespace = k8sNamespace; K8sNamespacePrefix = k8sNamespacePrefix;
KubeConfigFile = kubeConfigFile; KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout; OperationTimeout = operationTimeout;
RetryDelay = retryDelay; RetryDelay = retryDelay;
LocationMap = locationMap; LocationMap = locationMap;
} }
public string K8sNamespace { get; } public string K8sNamespacePrefix { get; }
public string? KubeConfigFile { get; } public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; } public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; } public TimeSpan RetryDelay { get; }

View File

@ -24,6 +24,14 @@
{ {
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag); return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
} }
public override string ToString()
{
return $"(container-recipe: {Name}, image: {Image}, " +
$"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
$"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
$"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, ";
}
} }
public class Port public class Port

View File

@ -0,0 +1,36 @@
using k8s;
namespace KubernetesWorkflow
{
public class K8sClient
{
private readonly Kubernetes client;
private static readonly object clientLock = new object();
public K8sClient(KubernetesClientConfiguration config)
{
client = new Kubernetes(config);
}
public void Run(Action<Kubernetes> action)
{
lock (clientLock)
{
action(client);
}
}
public T Run<T>(Func<Kubernetes, T> action)
{
lock (clientLock)
{
return action(client);
}
}
public void Dispose()
{
client.Dispose();
}
}
}

View File

@ -11,16 +11,18 @@ namespace KubernetesWorkflow
private readonly K8sCluster cluster; private readonly K8sCluster cluster;
private readonly KnownK8sPods knownPods; private readonly KnownK8sPods knownPods;
private readonly WorkflowNumberSource workflowNumberSource; private readonly WorkflowNumberSource workflowNumberSource;
private readonly Kubernetes client; private readonly K8sClient client;
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource) public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
{ {
this.log = log; this.log = log;
this.cluster = cluster; this.cluster = cluster;
this.knownPods = knownPods; this.knownPods = knownPods;
this.workflowNumberSource = workflowNumberSource; this.workflowNumberSource = workflowNumberSource;
client = new K8sClient(cluster.GetK8sClientConfig());
client = new Kubernetes(cluster.GetK8sClientConfig()); K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
log.Debug($"Test namespace: '{K8sTestNamespace}'");
} }
public void Dispose() public void Dispose()
@ -52,14 +54,14 @@ namespace KubernetesWorkflow
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{ {
log.Debug(); log.Debug();
using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
logHandler.Log(stream); logHandler.Log(stream);
} }
public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args) public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
{ {
log.Debug($"{containerName}: {command} ({string.Join(",", args)})"); log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args); var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
runner.Run(); runner.Run();
return runner.GetStdOut(); return runner.GetStdOut();
} }
@ -67,13 +69,43 @@ namespace KubernetesWorkflow
public void DeleteAllResources() public void DeleteAllResources()
{ {
log.Debug(); log.Debug();
DeleteNamespace();
var all = client.Run(c => c.ListNamespace().Items);
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
foreach (var ns in namespaces)
{
DeleteNamespace(ns);
}
foreach (var ns in namespaces)
{
WaitUntilNamespaceDeleted(ns);
}
}
public void DeleteTestNamespace()
{
log.Debug();
if (IsTestNamespaceOnline())
{
client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0));
}
WaitUntilNamespaceDeleted(); WaitUntilNamespaceDeleted();
} }
public void DeleteNamespace(string ns)
{
log.Debug();
if (IsNamespaceOnline(ns))
{
client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
}
}
#region Namespace management #region Namespace management
private string K8sTestNamespace { get; }
private void EnsureTestNamespace() private void EnsureTestNamespace()
{ {
if (IsTestNamespaceOnline()) return; if (IsTestNamespaceOnline()) return;
@ -83,30 +115,85 @@ namespace KubernetesWorkflow
ApiVersion = "v1", ApiVersion = "v1",
Metadata = new V1ObjectMeta Metadata = new V1ObjectMeta
{ {
Name = K8sNamespace, Name = K8sTestNamespace,
Labels = new Dictionary<string, string> { { "name", K8sNamespace } } Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
} }
}; };
client.CreateNamespace(namespaceSpec); client.Run(c => c.CreateNamespace(namespaceSpec));
WaitUntilNamespaceCreated(); WaitUntilNamespaceCreated();
}
private void DeleteNamespace() CreatePolicy();
{
if (IsTestNamespaceOnline())
{
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
}
}
private string K8sNamespace
{
get { return cluster.Configuration.K8sNamespace; }
} }
private bool IsTestNamespaceOnline() private bool IsTestNamespaceOnline()
{ {
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); return IsNamespaceOnline(K8sTestNamespace);
}
private bool IsNamespaceOnline(string name)
{
return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name));
}
private void CreatePolicy()
{
client.Run(c =>
{
var body = new V1NetworkPolicy
{
Metadata = new V1ObjectMeta
{
Name = "isolate-policy",
NamespaceProperty = K8sTestNamespace
},
Spec = new V1NetworkPolicySpec
{
PodSelector = new V1LabelSelector
{
MatchLabels = GetSelector()
},
PolicyTypes = new[]
{
"Ingress",
"Egress"
},
Ingress = new List<V1NetworkPolicyIngressRule>
{
new V1NetworkPolicyIngressRule
{
FromProperty = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
NamespaceSelector = new V1LabelSelector
{
MatchLabels = GetMyNamespaceSelector()
}
}
}
}
},
Egress = new List<V1NetworkPolicyEgressRule>
{
new V1NetworkPolicyEgressRule
{
To = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
NamespaceSelector = new V1LabelSelector
{
MatchLabels = GetMyNamespaceSelector()
}
}
}
}
}
}
};
c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace);
});
} }
#endregion #endregion
@ -141,7 +228,7 @@ namespace KubernetesWorkflow
} }
}; };
client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace));
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
return deploymentSpec.Metadata.Name; return deploymentSpec.Metadata.Name;
@ -149,7 +236,7 @@ namespace KubernetesWorkflow
private void DeleteDeployment(string deploymentName) private void DeleteDeployment(string deploymentName)
{ {
client.DeleteNamespacedDeployment(deploymentName, K8sNamespace); client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace));
WaitUntilDeploymentOffline(deploymentName); WaitUntilDeploymentOffline(deploymentName);
} }
@ -168,12 +255,18 @@ namespace KubernetesWorkflow
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } }; return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
} }
private IDictionary<string, string> GetMyNamespaceSelector()
{
return new Dictionary<string, string> { { "name", "thatisincorrect" } };
}
private V1ObjectMeta CreateDeploymentMetadata() private V1ObjectMeta CreateDeploymentMetadata()
{ {
return new V1ObjectMeta return new V1ObjectMeta
{ {
Name = "deploy-" + workflowNumberSource.WorkflowNumber, Name = "deploy-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace NamespaceProperty = K8sTestNamespace,
Labels = GetSelector()
}; };
} }
@ -257,14 +350,14 @@ namespace KubernetesWorkflow
} }
}; };
client.CreateNamespacedService(serviceSpec, K8sNamespace); client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
return (serviceSpec.Metadata.Name, result); return (serviceSpec.Metadata.Name, result);
} }
private void DeleteService(string serviceName) private void DeleteService(string serviceName)
{ {
client.DeleteNamespacedService(serviceName, K8sNamespace); client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
} }
private V1ObjectMeta CreateServiceMetadata() private V1ObjectMeta CreateServiceMetadata()
@ -272,7 +365,7 @@ namespace KubernetesWorkflow
return new V1ObjectMeta return new V1ObjectMeta
{ {
Name = "service-" + workflowNumberSource.WorkflowNumber, Name = "service-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace NamespaceProperty = K8sTestNamespace
}; };
} }
@ -323,11 +416,16 @@ namespace KubernetesWorkflow
WaitUntil(() => !IsTestNamespaceOnline()); WaitUntil(() => !IsTestNamespaceOnline());
} }
private void WaitUntilNamespaceDeleted(string name)
{
WaitUntil(() => !IsNamespaceOnline(name));
}
private void WaitUntilDeploymentOnline(string deploymentName) private void WaitUntilDeploymentOnline(string deploymentName)
{ {
WaitUntil(() => WaitUntil(() =>
{ {
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace));
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
}); });
} }
@ -336,7 +434,7 @@ namespace KubernetesWorkflow
{ {
WaitUntil(() => WaitUntil(() =>
{ {
var deployments = client.ListNamespacedDeployment(K8sNamespace); var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace));
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
return deployment == null || deployment.Status.AvailableReplicas == 0; return deployment == null || deployment.Status.AvailableReplicas == 0;
}); });
@ -346,7 +444,7 @@ namespace KubernetesWorkflow
{ {
WaitUntil(() => WaitUntil(() =>
{ {
var pods = client.ListNamespacedPod(K8sNamespace).Items; var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName); var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
return pod == null; return pod == null;
}); });
@ -369,7 +467,7 @@ namespace KubernetesWorkflow
private (string, string) FetchNewPod() private (string, string) FetchNewPod()
{ {
var pods = client.ListNamespacedPod(K8sNamespace).Items; var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure."); if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<RootNamespace>KubernetesWorkflow</RootNamespace> <RootNamespace>KubernetesWorkflow</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>

View File

@ -8,14 +8,16 @@ namespace KubernetesWorkflow
private readonly WorkflowNumberSource numberSource; private readonly WorkflowNumberSource numberSource;
private readonly K8sCluster cluster; private readonly K8sCluster cluster;
private readonly KnownK8sPods knownK8SPods; private readonly KnownK8sPods knownK8SPods;
private readonly string testNamespace;
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods) internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
{ {
this.log = log; this.log = log;
this.numberSource = numberSource; this.numberSource = numberSource;
this.cluster = cluster; this.cluster = cluster;
this.knownK8SPods = knownK8SPods; this.knownK8SPods = knownK8SPods;
this.testNamespace = testNamespace;
} }
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@ -62,10 +64,24 @@ namespace KubernetesWorkflow
}); });
} }
public void DeleteTestResources()
{
K8s(controller =>
{
controller.DeleteTestNamespace();
});
}
private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig) private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
{ {
log.Debug(); log.Debug();
return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r), startupConfig)).ToArray(); return recipes.Select(r =>
{
var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
return new RunningContainer(runningPod, r, servicePorts, startupConfig);
}).ToArray();
} }
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@ -74,7 +90,7 @@ namespace KubernetesWorkflow
var result = new List<ContainerRecipe>(); var result = new List<ContainerRecipe>();
for (var i = 0; i < numberOfContainers; i++) for (var i = 0; i < numberOfContainers; i++)
{ {
result.Add(recipeFactory.CreateRecipe(i ,numberSource.GetContainerNumber(), componentFactory, startupConfig)); result.Add(recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig));
} }
return result.ToArray(); return result.ToArray();
@ -82,14 +98,14 @@ namespace KubernetesWorkflow
private void K8s(Action<K8sController> action) private void K8s(Action<K8sController> action)
{ {
var controller = new K8sController(log, cluster, knownK8SPods, numberSource); var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
action(controller); action(controller);
controller.Dispose(); controller.Dispose();
} }
private T K8s<T>(Func<K8sController, T> action) private T K8s<T>(Func<K8sController, T> action)
{ {
var controller = new K8sController(log, cluster, knownK8SPods, numberSource); var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
var result = action(controller); var result = action(controller);
controller.Dispose(); controller.Dispose();
return result; return result;

View File

@ -6,25 +6,26 @@ namespace KubernetesWorkflow
public class WorkflowCreator public class WorkflowCreator
{ {
private readonly NumberSource numberSource = new NumberSource(0); private readonly NumberSource numberSource = new NumberSource(0);
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly NumberSource containerNumberSource = new NumberSource(0);
private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly K8sCluster cluster; private readonly K8sCluster cluster;
private readonly BaseLog log; private readonly BaseLog log;
private readonly string testNamespace;
public WorkflowCreator(BaseLog log, Configuration configuration) public WorkflowCreator(BaseLog log, Configuration configuration)
{ {
cluster = new K8sCluster(configuration); cluster = new K8sCluster(configuration);
this.log = log; this.log = log;
testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
} }
public StartupWorkflow CreateWorkflow() public StartupWorkflow CreateWorkflow()
{ {
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
servicePortNumberSource, ApplicationLifecycle.Instance.GetServiceNumberSource(),
containerNumberSource); containerNumberSource);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods); return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
} }
} }
} }

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<RootNamespace>Logging</RootNamespace> <RootNamespace>Logging</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>

View File

@ -10,16 +10,13 @@ namespace NethereumWorkflow
{ {
public class NethereumInteraction public class NethereumInteraction
{ {
private readonly List<Task> openTasks = new List<Task>();
private readonly BaseLog log; private readonly BaseLog log;
private readonly Web3 web3; private readonly Web3 web3;
private readonly string rootAccount;
internal NethereumInteraction(BaseLog log, Web3 web3, string rootAccount) internal NethereumInteraction(BaseLog log, Web3 web3)
{ {
this.log = log; this.log = log;
this.web3 = web3; this.web3 = web3;
this.rootAccount = rootAccount;
} }
public string GetTokenAddress(string marketplaceAddress) public string GetTokenAddress(string marketplaceAddress)
@ -31,29 +28,13 @@ namespace NethereumWorkflow
return Time.Wait(handler.QueryAsync<string>(marketplaceAddress, function)); return Time.Wait(handler.QueryAsync<string>(marketplaceAddress, function));
} }
public void TransferWeiTo(string account, decimal amount) public void MintTestTokens(string[] accounts, decimal amount, string tokenAddress)
{ {
log.Debug($"{amount} --> {account}"); if (amount < 1 || accounts.Length < 1) throw new ArgumentException("Invalid arguments for MintTestTokens");
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance");
var value = ToHexBig(amount); var tasks = accounts.Select(a => MintTokens(a, amount, tokenAddress));
var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value));
openTasks.Add(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId));
}
public void MintTestTokens(string account, decimal amount, string tokenAddress) Task.WaitAll(tasks.ToArray());
{
log.Debug($"({tokenAddress}) {amount} --> {account}");
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
var function = new MintTokensFunction
{
Holder = account,
Amount = ToBig(amount)
};
var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
openTasks.Add(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function));
} }
public decimal GetBalance(string tokenAddress, string account) public decimal GetBalance(string tokenAddress, string account)
@ -68,48 +49,54 @@ namespace NethereumWorkflow
return ToDecimal(Time.Wait(handler.QueryAsync<BigInteger>(tokenAddress, function))); return ToDecimal(Time.Wait(handler.QueryAsync<BigInteger>(tokenAddress, function)));
} }
public void WaitForAllTransactions() public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
{ {
var tasks = openTasks.ToArray(); try
openTasks.Clear(); {
return IsBlockNumberOK() && IsContractAvailable(marketplaceAddress, marketplaceAbi);
Task.WaitAll(tasks); }
catch
{
return false;
}
} }
public void EnsureSynced(string marketplaceAddress, string marketplaceAbi) private Task MintTokens(string account, decimal amount, string tokenAddress)
{ {
WaitUntilSynced(); log.Debug($"({tokenAddress}) {amount} --> {account}");
WaitForContract(marketplaceAddress, marketplaceAbi); if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
var function = new MintTokensFunction
{
Holder = account,
Amount = ToBig(amount)
};
var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
return handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function);
} }
private void WaitUntilSynced() private bool IsBlockNumberOK()
{ {
log.Debug(); log.Debug();
Time.WaitUntil(() => var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
{ var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync()); var numberOfBlocks = ToDecimal(number);
var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); return !sync.IsSyncing && numberOfBlocks > 256;
var numberOfBlocks = ToDecimal(number);
return !sync.IsSyncing && numberOfBlocks > 256;
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
} }
private void WaitForContract(string marketplaceAddress, string marketplaceAbi) private bool IsContractAvailable(string marketplaceAddress, string marketplaceAbi)
{ {
log.Debug(); log.Debug();
Time.WaitUntil(() => try
{ {
try var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
{ return contract != null;
var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress); }
return contract != null; catch
} {
catch return false;
{ }
return false;
}
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
} }
private HexBigInteger ToHexBig(decimal amount) private HexBigInteger ToHexBig(decimal amount)

View File

@ -8,21 +8,19 @@ namespace NethereumWorkflow
private readonly BaseLog log; private readonly BaseLog log;
private readonly string ip; private readonly string ip;
private readonly int port; private readonly int port;
private readonly string rootAccount;
private readonly string privateKey; private readonly string privateKey;
public NethereumInteractionCreator(BaseLog log, string ip, int port, string rootAccount, string privateKey) public NethereumInteractionCreator(BaseLog log, string ip, int port, string privateKey)
{ {
this.log = log; this.log = log;
this.ip = ip; this.ip = ip;
this.port = port; this.port = port;
this.rootAccount = rootAccount;
this.privateKey = privateKey; this.privateKey = privateKey;
} }
public NethereumInteraction CreateWorkflow() public NethereumInteraction CreateWorkflow()
{ {
return new NethereumInteraction(log, CreateWeb3(), rootAccount); return new NethereumInteraction(log, CreateWeb3());
} }
private Web3 CreateWeb3() private Web3 CreateWeb3()

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<RootNamespace>NethereumWorkflow</RootNamespace> <RootNamespace>NethereumWorkflow</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>

View File

@ -71,13 +71,13 @@ namespace Tests.BasicTests
requiredCollateral: 10.TestTokens(), requiredCollateral: 10.TestTokens(),
minRequiredNumberOfNodes: 1, minRequiredNumberOfNodes: 1,
proofProbability: 5, proofProbability: 5,
duration: TimeSpan.FromMinutes(2)); duration: TimeSpan.FromMinutes(1));
Time.Sleep(TimeSpan.FromMinutes(1)); Time.Sleep(TimeSpan.FromSeconds(10));
seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed."); seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
Time.Sleep(TimeSpan.FromMinutes(2)); Time.Sleep(TimeSpan.FromMinutes(1));
seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage."); seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage."); buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");

View File

@ -0,0 +1,45 @@
using DistTestCore;
using NUnit.Framework;
using Utils;
namespace Tests.BasicTests
{
// Warning!
// This is a test to check network-isolation in the test-infrastructure.
// It requires parallelism(2) or greater to run.
[TestFixture]
public class NetworkIsolationTest : DistTest
{
private IOnlineCodexNode? node = null;
[Test]
public void SetUpANodeAndWait()
{
node = SetupCodexNode();
Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
}
[Test]
public void ForeignNodeConnects()
{
var myNode = SetupCodexNode();
Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
try
{
myNode.ConnectToPeer(node!);
}
catch
{
// Good! This connection should be prohibited by the network isolation policy.
node = null;
return;
}
Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " +
"This may cause cross-test interference. Network isolation policy should be applied. Test infra failure.");
}
}
}

View File

@ -72,13 +72,13 @@ namespace Tests.BasicTests
private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b) private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
{ {
//var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
//Log.Debug($"Looking for {b.id} in engine-peers [{enginePeers}]"); Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
Log.Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
//Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
} }
} }

View File

@ -7,15 +7,23 @@ namespace Tests.BasicTests
[TestFixture] [TestFixture]
public class TwoClientTests : DistTest public class TwoClientTests : DistTest
{ {
[Test] [TestCase(1)]
public void TwoClientsOnePodTest() [TestCase(2)]
[TestCase(3)]
[TestCase(4)]
[TestCase(5)]
[TestCase(6)]
[TestCase(7)]
[TestCase(8)]
[TestCase(9)]
public void TwoClientsOnePodTest(int size)
{ {
var group = SetupCodexNodes(2); var group = SetupCodexNodes(2);
var primary = group[0]; var primary = group[0];
var secondary = group[1]; var secondary = group[1];
PerformTwoClientTest(primary, secondary); PerformTwoClientTest(primary, secondary, size.MB());
} }
[Test] [Test]
@ -38,10 +46,15 @@ namespace Tests.BasicTests
} }
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{
PerformTwoClientTest(primary, secondary, 1.MB());
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary, ByteSize size)
{ {
primary.ConnectToPeer(secondary); primary.ConnectToPeer(secondary);
var testFile = GenerateTestFile(1.MB()); var testFile = GenerateTestFile(size);
var contentId = primary.UploadFile(testFile); var contentId = primary.UploadFile(testFile);

6
Tests/Parallelism.cs Normal file
View File

@ -0,0 +1,6 @@
using NUnit.Framework;
[assembly: LevelOfParallelism(1)]
namespace Tests
{
}

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
</PropertyGroup> </PropertyGroup>

View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net7.0</TargetFramework>
<RootNamespace>Utils</RootNamespace> <RootNamespace>Utils</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>