codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/parallel-tests'

This commit is contained in:
benbierens 2023-05-05 08:33:24 +02:00
parent c1f1daa6fe 87f3b9ec00
commit 3aba6d5082
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
49 changed files with 631 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
DistTestCore/Codex/CodexAccess.cs
View File

@ -6,10 +6,12 @@ namespace DistTestCore.Codex
public class CodexAccess
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
public CodexAccess(BaseLog log, RunningContainer runningContainer)
public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
{
this.log = log;
this.timeSet = timeSet;
Container = runningContainer;
}
@ -40,11 +42,29 @@ namespace DistTestCore.Codex
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public void EnsureOnline()
{
try
{
var debugInfo = GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
var nodePeerId = debugInfo.id;
var nodeName = Container.Name;
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
}
catch (Exception e)
{
log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
}
}
private Http Http()
{
var ip = Container.Pod.Cluster.IP;
var port = Container.ServicePorts[0].Number;
return new Http(log, ip, port, baseUrl: "/api/codex/v1");
return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)

11
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -1,12 +1,17 @@
using DistTestCore.Marketplace;
using System.Runtime.InteropServices;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore.Codex
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
public const string DockerImage = "thatbenbierens/codexlocal:latest";
#if Arm64
public const string DockerImage = "emizzle/nim-codex-arm64:sha-c7af585";
#else
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
public const string DockerImage = "thatbenbierens/codexlocal:latest";
#endif
public const string MetricsPortTag = "metrics_port";
protected override string Image => DockerImage;

24
DistTestCore/CodexNodeGroup.cs
View File

@ -62,29 +62,15 @@ namespace DistTestCore
return $"group:[{Containers.Describe()}]";
}
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
public void EnsureOnline()
{
var access = new CodexAccess(lifecycle.Log, c);
EnsureOnline(access);
return factory.CreateOnlineCodexNode(access, this);
foreach (var node in Nodes) node.CodexAccess.EnsureOnline();
}
private void EnsureOnline(CodexAccess access)
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{
try
{
var debugInfo = access.GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
var nodePeerId = debugInfo.id;
var nodeName = access.Container.Name;
lifecycle.Log.AddStringReplace(nodePeerId, $"___{nodeName}___");
}
catch (Exception e)
{
lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
}
var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
return factory.CreateOnlineCodexNode(access, this);
}
}
}

3
DistTestCore/CodexStarter.cs
View File

@ -44,7 +44,7 @@ namespace DistTestCore
public void DeleteAllResources()
{
var workflow = CreateWorkflow();
workflow.DeleteAllResources();
workflow.DeleteTestResources();
RunningGroups.Clear();
}
@ -74,6 +74,7 @@ namespace DistTestCore
{
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
RunningGroups.Add(group);
group.EnsureOnline();
return group;
}

8
DistTestCore/Configuration.cs
View File

@ -4,13 +4,13 @@ namespace DistTestCore
{
public class Configuration
{
public KubernetesWorkflow.Configuration GetK8sConfiguration()
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{
return new KubernetesWorkflow.Configuration(
k8sNamespace: "codex-test-ns",
k8sNamespacePrefix: "ct-",
kubeConfigFile: null,
operationTimeout: Timing.K8sOperationTimeout(),
retryDelay: Timing.K8sServiceDelay(),
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay(),
locationMap: new[]
{
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),

114
DistTestCore/DistTest.cs
View File

@ -6,23 +6,25 @@ using KubernetesWorkflow;
using Logging;
using NUnit.Framework;
using System.Reflection;
using Utils;
namespace DistTestCore
{
[SetUpFixture]
[Parallelizable(ParallelScope.All)]
public abstract class DistTest
{
private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies;
private FixtureLog fixtureLog = null!;
private TestLifecycle lifecycle = null!;
private DateTime testStart = DateTime.MinValue;
private readonly FixtureLog fixtureLog;
private readonly object lifecycleLock = new object();
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
public DistTest()
{
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
}
[OneTimeSetUp]
@ -30,14 +32,11 @@ namespace DistTestCore
{
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
Timing.UseLongTimeouts = false;
fixtureLog = new FixtureLog(configuration.GetLogConfig());
try
{
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration());
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
wc.CreateWorkflow().DeleteAllResources();
});
}
@ -57,8 +56,6 @@ namespace DistTestCore
[SetUp]
public void SetUpDistTest()
{
Timing.UseLongTimeouts = ShouldUseLongTimeouts();
if (GlobalTestFailure.HasFailed)
{
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
@ -85,7 +82,7 @@ namespace DistTestCore
public TestFile GenerateTestFile(ByteSize size)
{
return lifecycle.FileManager.GenerateTestFile(size);
return Get().FileManager.GenerateTestFile(size);
}
public IOnlineCodexNode SetupCodexBootstrapNode()
@ -128,12 +125,58 @@ namespace DistTestCore
public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
{
return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup);
return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
}
protected BaseLog Log
protected void Log(string msg)
{
get { return lifecycle.Log; }
TestContext.Progress.WriteLine(msg);
Get().Log.Log(msg);
}
protected void Debug(string msg)
{
TestContext.Progress.WriteLine(msg);
Get().Log.Debug(msg);
}
private TestLifecycle Get()
{
lock (lifecycleLock)
{
return lifecycles[GetCurrentTestName()];
}
}
private void CreateNewTestLifecycle()
{
var testName = GetCurrentTestName();
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
{
lock (lifecycleLock)
{
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
}
});
}
private void DisposeTestLifecycle()
{
var lifecycle = Get();
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure(lifecycle);
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private ITimeSet GetTimeSet()
{
if (ShouldUseLongTimeouts()) return new LongTimeSet();
return new DefaultTimeSet();
}
private bool ShouldUseLongTimeouts()
@ -151,28 +194,7 @@ namespace DistTestCore
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
}
private void CreateNewTestLifecycle()
{
Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
{
lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
testStart = DateTime.UtcNow;
});
}
private void DisposeTestLifecycle()
{
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure();
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private void IncludeLogsAndMetricsOnTestFailure()
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
@ -182,8 +204,8 @@ namespace DistTestCore
if (IsDownloadingLogsAndMetricsEnabled())
{
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
DownloadAllLogs();
DownloadAllMetrics();
DownloadAllLogs(lifecycle);
DownloadAllMetrics(lifecycle);
}
else
{
@ -192,25 +214,19 @@ namespace DistTestCore
}
}
private string GetTestDuration()
private void DownloadAllLogs(TestLifecycle lifecycle)
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
}
private void DownloadAllLogs()
{
OnEachCodexNode(node =>
OnEachCodexNode(lifecycle, node =>
{
lifecycle.DownloadLog(node);
});
}
private void DownloadAllMetrics()
private void DownloadAllMetrics(TestLifecycle lifecycle)
{
var metricsDownloader = new MetricsDownloader(lifecycle.Log);
OnEachCodexNode(node =>
OnEachCodexNode(lifecycle, node =>
{
var m = node.Metrics as MetricsAccess;
if (m != null)
@ -220,7 +236,7 @@ namespace DistTestCore
});
}
private void OnEachCodexNode(Action<OnlineCodexNode> action)
private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
{
var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
foreach (var node in allNodes)

6
DistTestCore/DistTestCore.csproj
View File

@ -1,10 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>DistTestCore</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
</PropertyGroup>
<PropertyGroup Condition="'$(IsArm64)'=='true'">
<DefineConstants>Arm64</DefineConstants>
</PropertyGroup>
<ItemGroup>

4
DistTestCore/FileManager.cs
View File

@ -1,5 +1,6 @@
using Logging;
using NUnit.Framework;
using Utils;
namespace DistTestCore
{
@ -13,13 +14,14 @@ namespace DistTestCore
public class FileManager : IFileManager
{
public const int ChunkSize = 1024 * 1024;
private static NumberSource folderNumberSource = new NumberSource(0);
private readonly Random random = new Random();
private readonly TestLog log;
private readonly string folder;
public FileManager(TestLog log, Configuration configuration)
{
folder = configuration.GetFileManagerFolder();
folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
EnsureDirectory();
this.log = log;

9
DistTestCore/GethStarter.cs
View File

@ -36,13 +36,8 @@ namespace DistTestCore
var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
foreach (var account in companionNode.Accounts)
{
interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei);
interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
}
interaction.WaitForAllTransactions();
var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
}
private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)

14
DistTestCore/Http.cs
View File

@ -10,13 +10,15 @@ namespace DistTestCore
public class Http
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
private readonly string ip;
private readonly int port;
private readonly string baseUrl;
public Http(BaseLog log, string ip, int port, string baseUrl)
public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
{
this.log = log;
this.timeSet = timeSet;
this.ip = ip;
this.port = port;
this.baseUrl = baseUrl;
@ -103,7 +105,7 @@ namespace DistTestCore
log.Debug($"({url}) = '{message}'", 3);
}
private static T Retry<T>(Func<T> operation)
private T Retry<T>(Func<T> operation)
{
var retryCounter = 0;
@ -115,9 +117,9 @@ namespace DistTestCore
}
catch (Exception exception)
{
Timing.HttpCallRetryDelay();
timeSet.HttpCallRetryDelay();
retryCounter++;
if (retryCounter > Timing.HttpCallRetryCount())
if (retryCounter > timeSet.HttpCallRetryCount())
{
Assert.Fail(exception.ToString());
throw;
@ -140,10 +142,10 @@ namespace DistTestCore
}
}
private static HttpClient GetClient()
private HttpClient GetClient()
{
var client = new HttpClient();
client.Timeout = Timing.HttpCallTimeout();
client.Timeout = timeSet.HttpCallTimeout();
return client;
}
}

6
DistTestCore/Marketplace/CodexContractsContainerRecipe.cs
View File

@ -4,7 +4,11 @@ namespace DistTestCore.Marketplace
{
public class CodexContractsContainerRecipe : ContainerRecipeFactory
{
public const string DockerImage = "thatbenbierens/codex-contracts-deployment";
#if Arm64
public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
#else
public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint";
#endif
public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";

36
DistTestCore/Marketplace/ContainerInfoExtractor.cs
View File

@ -19,13 +19,14 @@ namespace DistTestCore.Marketplace
this.container = container;
}
public string ExtractAccount(int? orderNumber)
public AllGethAccounts ExtractAccounts()
{
log.Debug();
var account = Retry(() => FetchAccount(orderNumber));
if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
var accountsCsv = Retry(() => FetchAccountsCsv());
if (string.IsNullOrEmpty(accountsCsv)) throw new InvalidOperationException("Unable to fetch accounts.csv for geth node. Test infra failure.");
return account;
var lines = accountsCsv.Split('\n');
return new AllGethAccounts(lines.Select(ParseLineToAccount).ToArray());
}
public string ExtractPubKey()
@ -37,15 +38,6 @@ namespace DistTestCore.Marketplace
return pubKey;
}
public string ExtractPrivateKey(int? orderNumber)
{
log.Debug();
var privKey = Retry(() => FetchPrivateKey(orderNumber));
if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
return privKey;
}
public string ExtractMarketplaceAddress()
{
log.Debug();
@ -88,14 +80,9 @@ namespace DistTestCore.Marketplace
}
}
private string FetchAccount(int? orderNumber)
private string FetchAccountsCsv()
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber));
}
private string FetchPrivateKey(int? orderNumber)
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber));
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
}
private string FetchMarketplaceAddress()
@ -120,6 +107,15 @@ namespace DistTestCore.Marketplace
workflow.DownloadContainerLog(container, enodeFinder);
return enodeFinder.GetPubKey();
}
private GethAccount ParseLineToAccount(string l)
{
var tokens = l.Replace("\r", "").Split(',');
if (tokens.Length != 2) throw new InvalidOperationException();
var account = tokens[0];
var privateKey = tokens[1];
return new GethAccount(account, privateKey);
}
}
public class PubKeyFinder : LogHandler, ILogHandler

23
DistTestCore/Marketplace/GethBootstrapNodeInfo.cs
View File

@ -6,19 +6,19 @@ namespace DistTestCore.Marketplace
{
public class GethBootstrapNodeInfo
{
public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string pubKey, string privateKey, Port discoveryPort)
public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
{
RunningContainers = runningContainers;
Account = account;
AllAccounts = allAccounts;
Account = allAccounts.Accounts[0];
PubKey = pubKey;
PrivateKey = privateKey;
DiscoveryPort = discoveryPort;
}
public RunningContainers RunningContainers { get; }
public string Account { get; }
public AllGethAccounts AllAccounts { get; }
public GethAccount Account { get; }
public string PubKey { get; }
public string PrivateKey { get; }
public Port DiscoveryPort { get; }
public NethereumInteraction StartInteraction(BaseLog log)
@ -26,10 +26,19 @@ namespace DistTestCore.Marketplace
var ip = RunningContainers.RunningPod.Cluster.IP;
var port = RunningContainers.Containers[0].ServicePorts[0].Number;
var account = Account;
var privateKey = PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey);
var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey);
return creator.CreateWorkflow();
}
}
public class AllGethAccounts
{
public GethAccount[] Accounts { get; }
public AllGethAccounts(GethAccount[] accounts)
{
Accounts = accounts;
}
}
}

10
DistTestCore/Marketplace/GethBootstrapNodeStarter.cs
View File

@ -20,20 +20,20 @@ namespace DistTestCore.Marketplace
var bootstrapContainer = containers.Containers[0];
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
var account = extractor.ExtractAccount(null);
var accounts = extractor.ExtractAccounts();
var pubKey = extractor.ExtractPubKey();
var privateKey = extractor.ExtractPrivateKey(null);
var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
LogEnd($"Geth bootstrap node started with account '{account}'");
LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
return new GethBootstrapNodeInfo(containers, account, pubKey, privateKey, discoveryPort);
return result;
}
private StartupConfig CreateBootstrapStartupConfig()
{
var config = new StartupConfig();
config.Add(new GethStartupConfig(true, null!, 0));
config.Add(new GethStartupConfig(true, null!, 0, 0));
return config;
}
}

13
DistTestCore/Marketplace/GethCompanionNodeInfo.cs
View File

@ -6,30 +6,29 @@ namespace DistTestCore.Marketplace
{
public class GethCompanionNodeInfo
{
public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts)
public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
{
RunningContainer = runningContainer;
Accounts = accounts;
}
public RunningContainer RunningContainer { get; }
public GethCompanionAccount[] Accounts { get; }
public GethAccount[] Accounts { get; }
public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account)
public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
{
var ip = RunningContainer.Pod.Cluster.IP;
var port = RunningContainer.ServicePorts[0].Number;
var accountStr = account.Account;
var privateKey = account.PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey);
var creator = new NethereumInteractionCreator(log, ip, port, privateKey);
return creator.CreateWorkflow();
}
}
public class GethCompanionAccount
public class GethAccount
{
public GethCompanionAccount(string account, string privateKey)
public GethAccount(string account, string privateKey)
{
Account = account;
PrivateKey = privateKey;

53
DistTestCore/Marketplace/GethCompanionNodeStarter.cs
View File

@ -5,6 +5,8 @@ namespace DistTestCore.Marketplace
{
public class GethCompanionNodeStarter : BaseStarter
{
private int companionAccountIndex = 0;
public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
{
@ -14,53 +16,43 @@ namespace DistTestCore.Marketplace
{
LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
var workflow = workflowCreator.CreateWorkflow();
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
WaitForAccountCreation(codexSetup.NumberOfNodes);
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
var container = containers.Containers[0];
var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes);
var node = CreateCompanionInfo(container, marketplace, config);
EnsureCompanionNodeIsSynced(node, marketplace);
LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
return node;
}
private void WaitForAccountCreation(int numberOfNodes)
private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
{
// We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files
// we will be trying to read in 'ExtractAccount', later on in the start-up process.
Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes));
}
private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts)
{
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray();
var accounts = ExtractAccounts(marketplace, config);
return new GethCompanionNodeInfo(container, accounts);
}
private IEnumerable<GethCompanionAccount> ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts)
private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
{
for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1);
}
private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber)
{
var account = extractor.ExtractAccount(orderNumber);
var privKey = extractor.ExtractPrivateKey(orderNumber);
return new GethCompanionAccount(account, privKey);
return marketplace.Bootstrap.AllAccounts.Accounts
.Skip(1 + config.CompanionAccountStartIndex)
.Take(config.NumberOfCompanionAccounts)
.ToArray();
}
private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
{
try
{
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
Time.WaitUntil(() =>
{
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
}
catch (Exception e)
{
@ -68,10 +60,17 @@ namespace DistTestCore.Marketplace
}
}
private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
{
var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
companionAccountIndex += numberOfAccounts;
return config;
}
private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
{
var config = new StartupConfig();
config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts));
config.Add(gethConfig);
return config;
}
}

36
DistTestCore/Marketplace/GethContainerRecipe.cs
View File

@ -4,22 +4,17 @@ namespace DistTestCore.Marketplace
{
public class GethContainerRecipe : ContainerRecipeFactory
{
public const string DockerImage = "thatbenbierens/geth-confenv:latest";
#if Arm64
public const string DockerImage = "emizzle/geth-confenv:latest";
#else
public const string DockerImage = "thatbenbierens/geth-confenv:onethousand";
#endif
public const string HttpPortTag = "http_port";
public const string DiscoveryPortTag = "disc_port";
private const string defaultArgs = "--ipcdisable --syncmode full";
public static string GetAccountFilename(int? orderNumber)
{
if (orderNumber == null) return "account_string.txt";
return $"account_string_{orderNumber.Value}.txt";
}
public static string GetPrivateKeyFilename(int? orderNumber)
{
if (orderNumber == null) return "private.key";
return $"private_{orderNumber.Value}.key";
}
public const string AccountsFilename = "accounts.csv";
protected override string Image => DockerImage;
@ -46,14 +41,17 @@ namespace DistTestCore.Marketplace
private string CreateBootstapArgs(Port discovery)
{
AddEnvVar("IS_BOOTSTRAP", "1");
AddEnvVar("ENABLE_MINER", "1");
UnlockAccounts(0, 1);
var exposedPort = AddExposedPort(tag: HttpPortTag);
return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
}
private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
{
AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString());
UnlockAccounts(
config.CompanionAccountStartIndex + 1,
config.NumberOfCompanionAccounts);
var port = AddInternalPort();
var authRpc = AddInternalPort();
@ -66,5 +64,15 @@ namespace DistTestCore.Marketplace
return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
}
private void UnlockAccounts(int startIndex, int numberOfAccounts)
{
if (startIndex < 0) throw new ArgumentException();
if (numberOfAccounts < 1) throw new ArgumentException();
if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
}
}
}

4
DistTestCore/Marketplace/GethStartupConfig.cs
View File

@ -2,15 +2,17 @@
{
public class GethStartupConfig
{
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts)
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
{
IsBootstrapNode = isBootstrapNode;
BootstrapNode = bootstrapNode;
CompanionAccountStartIndex = companionAccountStartIndex;
NumberOfCompanionAccounts = numberOfCompanionAccounts;
}
public bool IsBootstrapNode { get; }
public GethBootstrapNodeInfo BootstrapNode { get; }
public int CompanionAccountStartIndex { get; }
public int NumberOfCompanionAccounts { get; }
}
}

4
DistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace
{
private readonly TestLog log;
private readonly MarketplaceNetwork marketplaceNetwork;
private readonly GethCompanionAccount account;
private readonly GethAccount account;
private readonly CodexAccess codexAccess;
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess)
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
{
this.log = log;
this.marketplaceNetwork = marketplaceNetwork;

6
DistTestCore/Marketplace/MarketplaceAccessFactory.cs
View File

@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace
return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
}
private GethCompanionAccount GetGethCompanionNode(CodexAccess access)
private GethAccount GetGethCompanionNode(CodexAccess access)
{
var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount);
return (GethCompanionAccount)account;
var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
return (GethAccount)account;
}
}
}

6
DistTestCore/Metrics/MetricsAccess.cs
View File

@ -14,12 +14,14 @@ namespace DistTestCore.Metrics
public class MetricsAccess : IMetricsAccess
{
private readonly TestLog log;
private readonly ITimeSet timeSet;
private readonly MetricsQuery query;
private readonly RunningContainer node;
public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node)
public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
{
this.log = log;
this.timeSet = timeSet;
this.query = query;
this.node = node;
}
@ -47,7 +49,7 @@ namespace DistTestCore.Metrics
{
var mostRecent = GetMostRecent(metricName);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
{
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
throw new TimeoutException();

4
DistTestCore/Metrics/MetricsAccessFactory.cs
View File

@ -28,8 +28,8 @@ namespace DistTestCore.Metrics
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{
var query = new MetricsQuery(lifecycle.Log, prometheusContainer);
return new MetricsAccess(lifecycle.Log, query, codexContainer);
var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
}
}
}

3
DistTestCore/Metrics/MetricsQuery.cs
View File

@ -9,12 +9,13 @@ namespace DistTestCore.Metrics
{
private readonly Http http;
public MetricsQuery(BaseLog log, RunningContainers runningContainers)
public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
{
RunningContainers = runningContainers;
http = new Http(
log,
timeSet,
runningContainers.RunningPod.Cluster.IP,
runningContainers.Containers[0].ServicePorts[0].Number,
"api/v1");

4
DistTestCore/OnlineCodexNode.cs
View File

@ -92,6 +92,10 @@ namespace DistTestCore
public ICodexSetup BringOffline()
{
if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
"individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
"available for codex-nodes in groups of 1.");
return Group.BringOffline();
}

15
DistTestCore/TestLifecycle.cs
View File

@ -1,25 +1,30 @@
using DistTestCore.Logs;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore
{
public class TestLifecycle
{
private readonly WorkflowCreator workflowCreator;
private DateTime testStart = DateTime.MinValue;
public TestLifecycle(TestLog log, Configuration configuration)
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
{
Log = log;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration());
TimeSet = timeSet;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this, workflowCreator);
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator);
testStart = DateTime.UtcNow;
}
public TestLog Log { get; }
public ITimeSet TimeSet { get; }
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }
@ -42,5 +47,11 @@ namespace DistTestCore
return new CodexNodeLog(subFile, node);
}
public string GetTestDuration()
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
}
}
}

52
DistTestCore/Timing.cs
View File

@ -8,53 +8,11 @@ namespace DistTestCore
{
}
public static class Timing
{
public static bool UseLongTimeouts { get; set; }
public static TimeSpan HttpCallTimeout()
{
return GetTimes().HttpCallTimeout();
}
public static int HttpCallRetryCount()
{
return GetTimes().HttpCallRetryCount();
}
public static void HttpCallRetryDelay()
{
Time.Sleep(GetTimes().HttpCallRetryDelay());
}
public static TimeSpan K8sServiceDelay()
{
return GetTimes().WaitForK8sServiceDelay();
}
public static TimeSpan K8sOperationTimeout()
{
return GetTimes().K8sOperationTimeout();
}
public static TimeSpan WaitForMetricTimeout()
{
return GetTimes().WaitForMetricTimeout();
}
private static ITimeSet GetTimes()
{
if (UseLongTimeouts) return new LongTimeSet();
return new DefaultTimeSet();
}
}
public interface ITimeSet
{
TimeSpan HttpCallTimeout();
int HttpCallRetryCount();
TimeSpan HttpCallRetryDelay();
void HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout();
@ -72,9 +30,9 @@ namespace DistTestCore
return 5;
}
public TimeSpan HttpCallRetryDelay()
public void HttpCallRetryDelay()
{
return TimeSpan.FromSeconds(3);
Time.Sleep(TimeSpan.FromSeconds(3));
}
public TimeSpan WaitForK8sServiceDelay()
@ -105,9 +63,9 @@ namespace DistTestCore
return 2;
}
public TimeSpan HttpCallRetryDelay()
public void HttpCallRetryDelay()
{
return TimeSpan.FromMinutes(5);
Time.Sleep(TimeSpan.FromMinutes(5));
}
public TimeSpan WaitForK8sServiceDelay()

40
KubernetesWorkflow/ApplicationLifecycle.cs Normal file
View File

@ -0,0 +1,40 @@
using Utils;
namespace KubernetesWorkflow
{
public class ApplicationLifecycle
{
private static object instanceLock = new object();
private static ApplicationLifecycle? instance;
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource namespaceNumberSource = new NumberSource(0);
private ApplicationLifecycle()
{
}
public static ApplicationLifecycle Instance
{
// I know singletons are quite evil. But we need to be sure this object is created only once
// and persists for the entire application lifecycle.
get
{
lock (instanceLock)
{
if (instance == null) instance = new ApplicationLifecycle();
return instance;
}
}
}
public NumberSource GetServiceNumberSource()
{
return servicePortNumberSource;
}
public string GetTestNamespace()
{
return namespaceNumberSource.GetNextNumber().ToString("D5");
}
}
}

8
KubernetesWorkflow/CommandRunner.cs
View File

@ -5,7 +5,7 @@ namespace KubernetesWorkflow
{
public class CommandRunner
{
private readonly Kubernetes client;
private readonly K8sClient client;
private readonly string k8sNamespace;
private readonly RunningPod pod;
private readonly string containerName;
@ -13,7 +13,7 @@ namespace KubernetesWorkflow
private readonly string[] arguments;
private readonly List<string> lines = new List<string>();
public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
{
this.client = client;
this.k8sNamespace = k8sNamespace;
@ -27,8 +27,8 @@ namespace KubernetesWorkflow
{
var input = new[] { command }.Concat(arguments).ToArray();
Time.Wait(client.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
}
public string GetStdOut()

6
KubernetesWorkflow/Configuration.cs
View File

@ -2,16 +2,16 @@
{
public class Configuration
{
public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
{
K8sNamespace = k8sNamespace;
K8sNamespacePrefix = k8sNamespacePrefix;
KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout;
RetryDelay = retryDelay;
LocationMap = locationMap;
}
public string K8sNamespace { get; }
public string K8sNamespacePrefix { get; }
public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; }

8
KubernetesWorkflow/ContainerRecipe.cs
View File

@ -24,6 +24,14 @@
{
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
}
public override string ToString()
{
return $"(container-recipe: {Name}, image: {Image}, " +
$"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
$"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
$"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, ";
}
}
public class Port

36
KubernetesWorkflow/K8sClient.cs Normal file
View File

@ -0,0 +1,36 @@
using k8s;
namespace KubernetesWorkflow
{
public class K8sClient
{
private readonly Kubernetes client;
private static readonly object clientLock = new object();
public K8sClient(KubernetesClientConfiguration config)
{
client = new Kubernetes(config);
}
public void Run(Action<Kubernetes> action)
{
lock (clientLock)
{
action(client);
}
}
public T Run<T>(Func<Kubernetes, T> action)
{
lock (clientLock)
{
return action(client);
}
}
public void Dispose()
{
client.Dispose();
}
}
}

162
KubernetesWorkflow/K8sController.cs
View File

@ -11,16 +11,18 @@ namespace KubernetesWorkflow
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownPods;
private readonly WorkflowNumberSource workflowNumberSource;
private readonly Kubernetes client;
private readonly K8sClient client;
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
{
this.log = log;
this.cluster = cluster;
this.knownPods = knownPods;
this.workflowNumberSource = workflowNumberSource;
client = new K8sClient(cluster.GetK8sClientConfig());
client = new Kubernetes(cluster.GetK8sClientConfig());
K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
log.Debug($"Test namespace: '{K8sTestNamespace}'");
}
public void Dispose()
@ -52,14 +54,14 @@ namespace KubernetesWorkflow
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{
log.Debug();
using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name);
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
logHandler.Log(stream);
}
public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
{
log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
runner.Run();
return runner.GetStdOut();
}
@ -67,13 +69,43 @@ namespace KubernetesWorkflow
public void DeleteAllResources()
{
log.Debug();
DeleteNamespace();
var all = client.Run(c => c.ListNamespace().Items);
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
foreach (var ns in namespaces)
{
DeleteNamespace(ns);
}
foreach (var ns in namespaces)
{
WaitUntilNamespaceDeleted(ns);
}
}
public void DeleteTestNamespace()
{
log.Debug();
if (IsTestNamespaceOnline())
{
client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0));
}
WaitUntilNamespaceDeleted();
}
public void DeleteNamespace(string ns)
{
log.Debug();
if (IsNamespaceOnline(ns))
{
client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
}
}
#region Namespace management
private string K8sTestNamespace { get; }
private void EnsureTestNamespace()
{
if (IsTestNamespaceOnline()) return;
@ -83,30 +115,85 @@ namespace KubernetesWorkflow
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = K8sNamespace,
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
Name = K8sTestNamespace,
Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
}
};
client.CreateNamespace(namespaceSpec);
client.Run(c => c.CreateNamespace(namespaceSpec));
WaitUntilNamespaceCreated();
}
private void DeleteNamespace()
{
if (IsTestNamespaceOnline())
{
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
}
}
private string K8sNamespace
{
get { return cluster.Configuration.K8sNamespace; }
CreatePolicy();
}
private bool IsTestNamespaceOnline()
{
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
return IsNamespaceOnline(K8sTestNamespace);
}
private bool IsNamespaceOnline(string name)
{
return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name));
}
private void CreatePolicy()
{
client.Run(c =>
{
var body = new V1NetworkPolicy
{
Metadata = new V1ObjectMeta
{
Name = "isolate-policy",
NamespaceProperty = K8sTestNamespace
},
Spec = new V1NetworkPolicySpec
{
PodSelector = new V1LabelSelector
{
MatchLabels = GetSelector()
},
PolicyTypes = new[]
{
"Ingress",
"Egress"
},
Ingress = new List<V1NetworkPolicyIngressRule>
{
new V1NetworkPolicyIngressRule
{
FromProperty = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
NamespaceSelector = new V1LabelSelector
{
MatchLabels = GetMyNamespaceSelector()
}
}
}
}
},
Egress = new List<V1NetworkPolicyEgressRule>
{
new V1NetworkPolicyEgressRule
{
To = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
NamespaceSelector = new V1LabelSelector
{
MatchLabels = GetMyNamespaceSelector()
}
}
}
}
}
}
};
c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace);
});
}
#endregion
@ -141,7 +228,7 @@ namespace KubernetesWorkflow
}
};
client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace));
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
return deploymentSpec.Metadata.Name;
@ -149,7 +236,7 @@ namespace KubernetesWorkflow
private void DeleteDeployment(string deploymentName)
{
client.DeleteNamespacedDeployment(deploymentName, K8sNamespace);
client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace));
WaitUntilDeploymentOffline(deploymentName);
}
@ -168,12 +255,18 @@ namespace KubernetesWorkflow
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
}
private IDictionary<string, string> GetMyNamespaceSelector()
{
return new Dictionary<string, string> { { "name", "thatisincorrect" } };
}
private V1ObjectMeta CreateDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "deploy-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace
NamespaceProperty = K8sTestNamespace,
Labels = GetSelector()
};
}
@ -257,14 +350,14 @@ namespace KubernetesWorkflow
}
};
client.CreateNamespacedService(serviceSpec, K8sNamespace);
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
return (serviceSpec.Metadata.Name, result);
}
private void DeleteService(string serviceName)
{
client.DeleteNamespacedService(serviceName, K8sNamespace);
client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
}
private V1ObjectMeta CreateServiceMetadata()
@ -272,7 +365,7 @@ namespace KubernetesWorkflow
return new V1ObjectMeta
{
Name = "service-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace
NamespaceProperty = K8sTestNamespace
};
}
@ -323,11 +416,16 @@ namespace KubernetesWorkflow
WaitUntil(() => !IsTestNamespaceOnline());
}
private void WaitUntilNamespaceDeleted(string name)
{
WaitUntil(() => !IsNamespaceOnline(name));
}
private void WaitUntilDeploymentOnline(string deploymentName)
{
WaitUntil(() =>
{
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace));
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}
@ -336,7 +434,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var deployments = client.ListNamespacedDeployment(K8sNamespace);
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace));
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
@ -346,7 +444,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
return pod == null;
});
@ -369,7 +467,7 @@ namespace KubernetesWorkflow
private (string, string) FetchNewPod()
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");

2
KubernetesWorkflow/KubernetesWorkflow.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>KubernetesWorkflow</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

26
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -8,14 +8,16 @@ namespace KubernetesWorkflow
private readonly WorkflowNumberSource numberSource;
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownK8SPods;
private readonly string testNamespace;
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
{
this.log = log;
this.numberSource = numberSource;
this.cluster = cluster;
this.knownK8SPods = knownK8SPods;
this.testNamespace = testNamespace;
}
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@ -62,10 +64,24 @@ namespace KubernetesWorkflow
});
}
public void DeleteTestResources()
{
K8s(controller =>
{
controller.DeleteTestNamespace();
});
}
private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
{
log.Debug();
return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r), startupConfig)).ToArray();
return recipes.Select(r =>
{
var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
return new RunningContainer(runningPod, r, servicePorts, startupConfig);
}).ToArray();
}
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@ -74,7 +90,7 @@ namespace KubernetesWorkflow
var result = new List<ContainerRecipe>();
for (var i = 0; i < numberOfContainers; i++)
{
result.Add(recipeFactory.CreateRecipe(i ,numberSource.GetContainerNumber(), componentFactory, startupConfig));
result.Add(recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig));
}
return result.ToArray();
@ -82,14 +98,14 @@ namespace KubernetesWorkflow
private void K8s(Action<K8sController> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
action(controller);
controller.Dispose();
}
private T K8s<T>(Func<K8sController, T> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
var result = action(controller);
controller.Dispose();
return result;

7
KubernetesWorkflow/WorkflowCreator.cs
View File

@ -6,25 +6,26 @@ namespace KubernetesWorkflow
public class WorkflowCreator
{
private readonly NumberSource numberSource = new NumberSource(0);
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource containerNumberSource = new NumberSource(0);
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly K8sCluster cluster;
private readonly BaseLog log;
private readonly string testNamespace;
public WorkflowCreator(BaseLog log, Configuration configuration)
{
cluster = new K8sCluster(configuration);
this.log = log;
testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
}
public StartupWorkflow CreateWorkflow()
{
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
servicePortNumberSource,
ApplicationLifecycle.Instance.GetServiceNumberSource(),
containerNumberSource);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
}
}
}

2
Logging/Logging.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>Logging</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

2
LongTests/TestsLong.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

93
Nethereum/NethereumInteraction.cs
View File

@ -10,16 +10,13 @@ namespace NethereumWorkflow
{
public class NethereumInteraction
{
private readonly List<Task> openTasks = new List<Task>();
private readonly BaseLog log;
private readonly Web3 web3;
private readonly string rootAccount;
internal NethereumInteraction(BaseLog log, Web3 web3, string rootAccount)
internal NethereumInteraction(BaseLog log, Web3 web3)
{
this.log = log;
this.web3 = web3;
this.rootAccount = rootAccount;
}
public string GetTokenAddress(string marketplaceAddress)
@ -31,29 +28,13 @@ namespace NethereumWorkflow
return Time.Wait(handler.QueryAsync<string>(marketplaceAddress, function));
}
public void TransferWeiTo(string account, decimal amount)
public void MintTestTokens(string[] accounts, decimal amount, string tokenAddress)
{
log.Debug($"{amount} --> {account}");
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance");
if (amount < 1 || accounts.Length < 1) throw new ArgumentException("Invalid arguments for MintTestTokens");
var value = ToHexBig(amount);
var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value));
openTasks.Add(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId));
}
var tasks = accounts.Select(a => MintTokens(a, amount, tokenAddress));
public void MintTestTokens(string account, decimal amount, string tokenAddress)
{
log.Debug($"({tokenAddress}) {amount} --> {account}");
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
var function = new MintTokensFunction
{
Holder = account,
Amount = ToBig(amount)
};
var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
openTasks.Add(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function));
Task.WaitAll(tasks.ToArray());
}
public decimal GetBalance(string tokenAddress, string account)
@ -68,48 +49,54 @@ namespace NethereumWorkflow
return ToDecimal(Time.Wait(handler.QueryAsync<BigInteger>(tokenAddress, function)));
}
public void WaitForAllTransactions()
public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
{
var tasks = openTasks.ToArray();
openTasks.Clear();
Task.WaitAll(tasks);
try
{
return IsBlockNumberOK() && IsContractAvailable(marketplaceAddress, marketplaceAbi);
}
catch
{
return false;
}
}
public void EnsureSynced(string marketplaceAddress, string marketplaceAbi)
private Task MintTokens(string account, decimal amount, string tokenAddress)
{
WaitUntilSynced();
WaitForContract(marketplaceAddress, marketplaceAbi);
log.Debug($"({tokenAddress}) {amount} --> {account}");
if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
var function = new MintTokensFunction
{
Holder = account,
Amount = ToBig(amount)
};
var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
return handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function);
}
private void WaitUntilSynced()
private bool IsBlockNumberOK()
{
log.Debug();
Time.WaitUntil(() =>
{
var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
var numberOfBlocks = ToDecimal(number);
return !sync.IsSyncing && numberOfBlocks > 256;
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
var numberOfBlocks = ToDecimal(number);
return !sync.IsSyncing && numberOfBlocks > 256;
}
private void WaitForContract(string marketplaceAddress, string marketplaceAbi)
private bool IsContractAvailable(string marketplaceAddress, string marketplaceAbi)
{
log.Debug();
Time.WaitUntil(() =>
try
{
try
{
var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
return contract != null;
}
catch
{
return false;
}
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
return contract != null;
}
catch
{
return false;
}
}
private HexBigInteger ToHexBig(decimal amount)

6
Nethereum/NethereumInteractionCreator.cs
View File

@ -8,21 +8,19 @@ namespace NethereumWorkflow
private readonly BaseLog log;
private readonly string ip;
private readonly int port;
private readonly string rootAccount;
private readonly string privateKey;
public NethereumInteractionCreator(BaseLog log, string ip, int port, string rootAccount, string privateKey)
public NethereumInteractionCreator(BaseLog log, string ip, int port, string privateKey)
{
this.log = log;
this.ip = ip;
this.port = port;
this.rootAccount = rootAccount;
this.privateKey = privateKey;
}
public NethereumInteraction CreateWorkflow()
{
return new NethereumInteraction(log, CreateWeb3(), rootAccount);
return new NethereumInteraction(log, CreateWeb3());
}
private Web3 CreateWeb3()

6
Nethereum/NethereumWorkflow.csproj
View File

@ -1,16 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>NethereumWorkflow</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Logging\Logging.csproj" />
<ProjectReference Include="..\Utils\Utils.csproj" />

2
Tests/BasicTests/DownloadTests.cs
View File

@ -36,4 +36,4 @@ namespace Tests.ParallelTests
}
}
}
}
}

6
Tests/BasicTests/ExampleTests.cs
View File

@ -71,13 +71,13 @@ namespace Tests.BasicTests
requiredCollateral: 10.TestTokens(),
minRequiredNumberOfNodes: 1,
proofProbability: 5,
duration: TimeSpan.FromMinutes(2));
duration: TimeSpan.FromMinutes(1));
Time.Sleep(TimeSpan.FromMinutes(1));
Time.Sleep(TimeSpan.FromSeconds(10));
seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
Time.Sleep(TimeSpan.FromMinutes(2));
Time.Sleep(TimeSpan.FromMinutes(1));
seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");

45
Tests/BasicTests/NetworkIsolationTest.cs Normal file
View File

@ -0,0 +1,45 @@
using DistTestCore;
using NUnit.Framework;
using Utils;
namespace Tests.BasicTests
{
// Warning!
// This is a test to check network-isolation in the test-infrastructure.
// It requires parallelism(2) or greater to run.
[TestFixture]
public class NetworkIsolationTest : DistTest
{
private IOnlineCodexNode? node = null;
[Test]
public void SetUpANodeAndWait()
{
node = SetupCodexNode();
Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
}
[Test]
public void ForeignNodeConnects()
{
var myNode = SetupCodexNode();
Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
try
{
myNode.ConnectToPeer(node!);
}
catch
{
// Good! This connection should be prohibited by the network isolation policy.
node = null;
return;
}
Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " +
"This may cause cross-test interference. Network isolation policy should be applied. Test infra failure.");
}
}
}

8
Tests/BasicTests/PeerTests.cs
View File

@ -72,13 +72,13 @@ namespace Tests.BasicTests
private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
{
//var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
//Log.Debug($"Looking for {b.id} in engine-peers [{enginePeers}]");
Log.Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
//Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
}
}

21
Tests/BasicTests/TwoClientTests.cs
View File

@ -7,15 +7,23 @@ namespace Tests.BasicTests
[TestFixture]
public class TwoClientTests : DistTest
{
[Test]
public void TwoClientsOnePodTest()
[TestCase(1)]
[TestCase(2)]
[TestCase(3)]
[TestCase(4)]
[TestCase(5)]
[TestCase(6)]
[TestCase(7)]
[TestCase(8)]
[TestCase(9)]
public void TwoClientsOnePodTest(int size)
{
var group = SetupCodexNodes(2);
var primary = group[0];
var secondary = group[1];
PerformTwoClientTest(primary, secondary);
PerformTwoClientTest(primary, secondary, size.MB());
}
[Test]
@ -38,10 +46,15 @@ namespace Tests.BasicTests
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{
PerformTwoClientTest(primary, secondary, 1.MB());
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary, ByteSize size)
{
primary.ConnectToPeer(secondary);
var testFile = GenerateTestFile(1.MB());
var testFile = GenerateTestFile(size);
var contentId = primary.UploadFile(testFile);

2
Tests/BasicTests/UploadTests.cs
View File

@ -42,4 +42,4 @@ namespace Tests.ParallelTests
}
}
}
}
}

6
Tests/Parallelism.cs Normal file
View File

@ -0,0 +1,6 @@
using NUnit.Framework;
[assembly: LevelOfParallelism(1)]
namespace Tests
{
}

2
Tests/Tests.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Utils/Utils.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>Utils</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>