status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/tests

This commit is contained in:
Corbo12 2023-05-09 12:04:48 +02:00
parent 27e511f455 9c6f00dbce
commit 38e42f2ce9
53 changed files with 688 additions and 368 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
CONTRIBUTINGTESTS.MD Normal file
View File

@ -0,0 +1,24 @@
# Distributed System Tests for Nim-Codex
## Contributing tests
Do you want to write some tests for Codex using this distributed test setup? Great! Here's what you do.
1. Create a branch. Name it something descriptive, but start it with `tests/` please. [Example: `tests/data-redundancy`.]
1. Checkout your branch, and decide if your tests will be 'short' tests (minutes to hours), or 'long' tests (hours to days), or both! Create a folder for your tests in the matching folders (`/Tests`, `/LongTests`) and don't worry: You can always move your tests later if you like. [Example, short: `/Tests/DataRedundancy/`, long: `/LongTests/DataRedundancy/`]
1. Create one or more code files in your new folder, and write some tests! Here are some tips to help you get started. You can always take a look at the example tests found in [`/Tests/BasicTests/ExampleTests.cs`](/Tests/BasicTests/ExampleTests.cs)
1. Set up a standard NUnit test fixture.
1. Inherrit from `DistTest` or `AutoBootstrapDistTest`.
1. When using `DistTest`:
1. You must start your own Codex bootstrap node. You can use `SetupCodexBootstrapNode(...)` for this.
1. When you start other Codex nodes with `SetupCodexNodes(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
1. When using `AutoBootstrapDistTest`:
1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `DistTest` instead.
1. You can generate files of random test data by calling `GenerateTestFile(...)`.
1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
1. Tip: Commit often.
1. Once you're happy with your tests, please create a pull-request and ask (another) Codex core contributor to review your changes.

24
DistTestCore/Codex/CodexAccess.cs
View File

@ -6,10 +6,12 @@ namespace DistTestCore.Codex
public class CodexAccess
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
public CodexAccess(BaseLog log, RunningContainer runningContainer)
public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
{
this.log = log;
this.timeSet = timeSet;
Container = runningContainer;
}
@ -40,11 +42,29 @@ namespace DistTestCore.Codex
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public void EnsureOnline()
{
try
{
var debugInfo = GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
var nodePeerId = debugInfo.id;
var nodeName = Container.Name;
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
}
catch (Exception e)
{
log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
}
}
private Http Http()
{
var ip = Container.Pod.Cluster.IP;
var port = Container.ServicePorts[0].Number;
return new Http(log, ip, port, baseUrl: "/api/codex/v1");
return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)

11
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -1,12 +1,17 @@
using DistTestCore.Marketplace;
using System.Runtime.InteropServices;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore.Codex
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
public const string DockerImage = "thatbenbierens/codexlocal:latest";
#if Arm64
public const string DockerImage = "emizzle/nim-codex-arm64:sha-c7af585";
#else
//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
public const string DockerImage = "thatbenbierens/codexlocal:latest";
#endif
public const string MetricsPortTag = "metrics_port";
protected override string Image => DockerImage;

24
DistTestCore/CodexNodeGroup.cs
View File

@ -62,29 +62,15 @@ namespace DistTestCore
return $"group:[{Containers.Describe()}]";
}
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
public void EnsureOnline()
{
var access = new CodexAccess(lifecycle.Log, c);
EnsureOnline(access);
return factory.CreateOnlineCodexNode(access, this);
foreach (var node in Nodes) node.CodexAccess.EnsureOnline();
}
private void EnsureOnline(CodexAccess access)
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{
try
{
var debugInfo = access.GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
var nodePeerId = debugInfo.id;
var nodeName = access.Container.Name;
lifecycle.Log.AddStringReplace(nodePeerId, $"___{nodeName}___");
}
catch (Exception e)
{
lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
}
var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
return factory.CreateOnlineCodexNode(access, this);
}
}
}

3
DistTestCore/CodexStarter.cs
View File

@ -44,7 +44,7 @@ namespace DistTestCore
public void DeleteAllResources()
{
var workflow = CreateWorkflow();
workflow.DeleteAllResources();
workflow.DeleteTestResources();
RunningGroups.Clear();
}
@ -74,6 +74,7 @@ namespace DistTestCore
{
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
RunningGroups.Add(group);
group.EnsureOnline();
return group;
}

8
DistTestCore/Configuration.cs
View File

@ -4,13 +4,13 @@ namespace DistTestCore
{
public class Configuration
{
public KubernetesWorkflow.Configuration GetK8sConfiguration()
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{
return new KubernetesWorkflow.Configuration(
k8sNamespace: "codex-test-ns",
k8sNamespacePrefix: "ct-",
kubeConfigFile: null,
operationTimeout: Timing.K8sOperationTimeout(),
retryDelay: Timing.K8sServiceDelay(),
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay(),
locationMap: new[]
{
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),

114
DistTestCore/DistTest.cs
View File

@ -6,23 +6,25 @@ using KubernetesWorkflow;
using Logging;
using NUnit.Framework;
using System.Reflection;
using Utils;
namespace DistTestCore
{
[SetUpFixture]
[Parallelizable(ParallelScope.All)]
public abstract class DistTest
{
private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies;
private FixtureLog fixtureLog = null!;
private TestLifecycle lifecycle = null!;
private DateTime testStart = DateTime.MinValue;
private readonly FixtureLog fixtureLog;
private readonly object lifecycleLock = new object();
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
public DistTest()
{
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
}
[OneTimeSetUp]
@ -30,14 +32,11 @@ namespace DistTestCore
{
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
Timing.UseLongTimeouts = false;
fixtureLog = new FixtureLog(configuration.GetLogConfig());
try
{
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration());
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
wc.CreateWorkflow().DeleteAllResources();
});
}
@ -57,8 +56,6 @@ namespace DistTestCore
[SetUp]
public void SetUpDistTest()
{
Timing.UseLongTimeouts = ShouldUseLongTimeouts();
if (GlobalTestFailure.HasFailed)
{
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
@ -85,7 +82,7 @@ namespace DistTestCore
public TestFile GenerateTestFile(ByteSize size)
{
return lifecycle.FileManager.GenerateTestFile(size);
return Get().FileManager.GenerateTestFile(size);
}
public IOnlineCodexNode SetupCodexBootstrapNode()
@ -128,12 +125,58 @@ namespace DistTestCore
public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
{
return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup);
return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
}
protected BaseLog Log
protected void Log(string msg)
{
get { return lifecycle.Log; }
TestContext.Progress.WriteLine(msg);
Get().Log.Log(msg);
}
protected void Debug(string msg)
{
TestContext.Progress.WriteLine(msg);
Get().Log.Debug(msg);
}
private TestLifecycle Get()
{
lock (lifecycleLock)
{
return lifecycles[GetCurrentTestName()];
}
}
private void CreateNewTestLifecycle()
{
var testName = GetCurrentTestName();
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
{
lock (lifecycleLock)
{
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
}
});
}
private void DisposeTestLifecycle()
{
var lifecycle = Get();
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure(lifecycle);
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private ITimeSet GetTimeSet()
{
if (ShouldUseLongTimeouts()) return new LongTimeSet();
return new DefaultTimeSet();
}
private bool ShouldUseLongTimeouts()
@ -151,28 +194,7 @@ namespace DistTestCore
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
}
private void CreateNewTestLifecycle()
{
Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
{
lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
testStart = DateTime.UtcNow;
});
}
private void DisposeTestLifecycle()
{
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure();
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private void IncludeLogsAndMetricsOnTestFailure()
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
@ -182,8 +204,8 @@ namespace DistTestCore
if (IsDownloadingLogsAndMetricsEnabled())
{
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
DownloadAllLogs();
DownloadAllMetrics();
DownloadAllLogs(lifecycle);
DownloadAllMetrics(lifecycle);
}
else
{
@ -192,25 +214,19 @@ namespace DistTestCore
}
}
private string GetTestDuration()
private void DownloadAllLogs(TestLifecycle lifecycle)
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
}
private void DownloadAllLogs()
{
OnEachCodexNode(node =>
OnEachCodexNode(lifecycle, node =>
{
lifecycle.DownloadLog(node);
});
}
private void DownloadAllMetrics()
private void DownloadAllMetrics(TestLifecycle lifecycle)
{
var metricsDownloader = new MetricsDownloader(lifecycle.Log);
OnEachCodexNode(node =>
OnEachCodexNode(lifecycle, node =>
{
var m = node.Metrics as MetricsAccess;
if (m != null)
@ -220,7 +236,7 @@ namespace DistTestCore
});
}
private void OnEachCodexNode(Action<OnlineCodexNode> action)
private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
{
var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
foreach (var node in allNodes)

6
DistTestCore/DistTestCore.csproj
View File

@ -1,10 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>DistTestCore</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
</PropertyGroup>
<PropertyGroup Condition="'$(IsArm64)'=='true'">
<DefineConstants>Arm64</DefineConstants>
</PropertyGroup>
<ItemGroup>

4
DistTestCore/FileManager.cs
View File

@ -1,5 +1,6 @@
using Logging;
using NUnit.Framework;
using Utils;
namespace DistTestCore
{
@ -13,13 +14,14 @@ namespace DistTestCore
public class FileManager : IFileManager
{
public const int ChunkSize = 1024 * 1024;
private static NumberSource folderNumberSource = new NumberSource(0);
private readonly Random random = new Random();
private readonly TestLog log;
private readonly string folder;
public FileManager(TestLog log, Configuration configuration)
{
folder = configuration.GetFileManagerFolder();
folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
EnsureDirectory();
this.log = log;

9
DistTestCore/GethStarter.cs
View File

@ -36,13 +36,8 @@ namespace DistTestCore
var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
foreach (var account in companionNode.Accounts)
{
interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei);
interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
}
interaction.WaitForAllTransactions();
var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
}
private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)

14
DistTestCore/Http.cs
View File

@ -10,13 +10,15 @@ namespace DistTestCore
public class Http
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
private readonly string ip;
private readonly int port;
private readonly string baseUrl;
public Http(BaseLog log, string ip, int port, string baseUrl)
public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
{
this.log = log;
this.timeSet = timeSet;
this.ip = ip;
this.port = port;
this.baseUrl = baseUrl;
@ -103,7 +105,7 @@ namespace DistTestCore
log.Debug($"({url}) = '{message}'", 3);
}
private static T Retry<T>(Func<T> operation)
private T Retry<T>(Func<T> operation)
{
var retryCounter = 0;
@ -115,9 +117,9 @@ namespace DistTestCore
}
catch (Exception exception)
{
Timing.HttpCallRetryDelay();
timeSet.HttpCallRetryDelay();
retryCounter++;
if (retryCounter > Timing.HttpCallRetryCount())
if (retryCounter > timeSet.HttpCallRetryCount())
{
Assert.Fail(exception.ToString());
throw;
@ -140,10 +142,10 @@ namespace DistTestCore
}
}
private static HttpClient GetClient()
private HttpClient GetClient()
{
var client = new HttpClient();
client.Timeout = Timing.HttpCallTimeout();
client.Timeout = timeSet.HttpCallTimeout();
return client;
}
}

6
DistTestCore/Marketplace/CodexContractsContainerRecipe.cs
View File

@ -4,7 +4,11 @@ namespace DistTestCore.Marketplace
{
public class CodexContractsContainerRecipe : ContainerRecipeFactory
{
public const string DockerImage = "thatbenbierens/codex-contracts-deployment";
#if Arm64
public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
#else
public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint";
#endif
public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";

36
DistTestCore/Marketplace/ContainerInfoExtractor.cs
View File

@ -19,13 +19,14 @@ namespace DistTestCore.Marketplace
this.container = container;
}
public string ExtractAccount(int? orderNumber)
public AllGethAccounts ExtractAccounts()
{
log.Debug();
var account = Retry(() => FetchAccount(orderNumber));
if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
var accountsCsv = Retry(() => FetchAccountsCsv());
if (string.IsNullOrEmpty(accountsCsv)) throw new InvalidOperationException("Unable to fetch accounts.csv for geth node. Test infra failure.");
return account;
var lines = accountsCsv.Split('\n');
return new AllGethAccounts(lines.Select(ParseLineToAccount).ToArray());
}
public string ExtractPubKey()
@ -37,15 +38,6 @@ namespace DistTestCore.Marketplace
return pubKey;
}
public string ExtractPrivateKey(int? orderNumber)
{
log.Debug();
var privKey = Retry(() => FetchPrivateKey(orderNumber));
if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
return privKey;
}
public string ExtractMarketplaceAddress()
{
log.Debug();
@ -88,14 +80,9 @@ namespace DistTestCore.Marketplace
}
}
private string FetchAccount(int? orderNumber)
private string FetchAccountsCsv()
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber));
}
private string FetchPrivateKey(int? orderNumber)
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber));
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
}
private string FetchMarketplaceAddress()
@ -120,6 +107,15 @@ namespace DistTestCore.Marketplace
workflow.DownloadContainerLog(container, enodeFinder);
return enodeFinder.GetPubKey();
}
private GethAccount ParseLineToAccount(string l)
{
var tokens = l.Replace("\r", "").Split(',');
if (tokens.Length != 2) throw new InvalidOperationException();
var account = tokens[0];
var privateKey = tokens[1];
return new GethAccount(account, privateKey);
}
}
public class PubKeyFinder : LogHandler, ILogHandler

23
DistTestCore/Marketplace/GethBootstrapNodeInfo.cs
View File

@ -6,19 +6,19 @@ namespace DistTestCore.Marketplace
{
public class GethBootstrapNodeInfo
{
public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string pubKey, string privateKey, Port discoveryPort)
public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
{
RunningContainers = runningContainers;
Account = account;
AllAccounts = allAccounts;
Account = allAccounts.Accounts[0];
PubKey = pubKey;
PrivateKey = privateKey;
DiscoveryPort = discoveryPort;
}
public RunningContainers RunningContainers { get; }
public string Account { get; }
public AllGethAccounts AllAccounts { get; }
public GethAccount Account { get; }
public string PubKey { get; }
public string PrivateKey { get; }
public Port DiscoveryPort { get; }
public NethereumInteraction StartInteraction(BaseLog log)
@ -26,10 +26,19 @@ namespace DistTestCore.Marketplace
var ip = RunningContainers.RunningPod.Cluster.IP;
var port = RunningContainers.Containers[0].ServicePorts[0].Number;
var account = Account;
var privateKey = PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey);
var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey);
return creator.CreateWorkflow();
}
}
public class AllGethAccounts
{
public GethAccount[] Accounts { get; }
public AllGethAccounts(GethAccount[] accounts)
{
Accounts = accounts;
}
}
}

10
DistTestCore/Marketplace/GethBootstrapNodeStarter.cs
View File

@ -20,20 +20,20 @@ namespace DistTestCore.Marketplace
var bootstrapContainer = containers.Containers[0];
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
var account = extractor.ExtractAccount(null);
var accounts = extractor.ExtractAccounts();
var pubKey = extractor.ExtractPubKey();
var privateKey = extractor.ExtractPrivateKey(null);
var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
LogEnd($"Geth bootstrap node started with account '{account}'");
LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
return new GethBootstrapNodeInfo(containers, account, pubKey, privateKey, discoveryPort);
return result;
}
private StartupConfig CreateBootstrapStartupConfig()
{
var config = new StartupConfig();
config.Add(new GethStartupConfig(true, null!, 0));
config.Add(new GethStartupConfig(true, null!, 0, 0));
return config;
}
}

13
DistTestCore/Marketplace/GethCompanionNodeInfo.cs
View File

@ -6,30 +6,29 @@ namespace DistTestCore.Marketplace
{
public class GethCompanionNodeInfo
{
public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts)
public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
{
RunningContainer = runningContainer;
Accounts = accounts;
}
public RunningContainer RunningContainer { get; }
public GethCompanionAccount[] Accounts { get; }
public GethAccount[] Accounts { get; }
public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account)
public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
{
var ip = RunningContainer.Pod.Cluster.IP;
var port = RunningContainer.ServicePorts[0].Number;
var accountStr = account.Account;
var privateKey = account.PrivateKey;
var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey);
var creator = new NethereumInteractionCreator(log, ip, port, privateKey);
return creator.CreateWorkflow();
}
}
public class GethCompanionAccount
public class GethAccount
{
public GethCompanionAccount(string account, string privateKey)
public GethAccount(string account, string privateKey)
{
Account = account;
PrivateKey = privateKey;

53
DistTestCore/Marketplace/GethCompanionNodeStarter.cs
View File

@ -5,6 +5,8 @@ namespace DistTestCore.Marketplace
{
public class GethCompanionNodeStarter : BaseStarter
{
private int companionAccountIndex = 0;
public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
{
@ -14,53 +16,43 @@ namespace DistTestCore.Marketplace
{
LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
var workflow = workflowCreator.CreateWorkflow();
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
WaitForAccountCreation(codexSetup.NumberOfNodes);
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
var container = containers.Containers[0];
var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes);
var node = CreateCompanionInfo(container, marketplace, config);
EnsureCompanionNodeIsSynced(node, marketplace);
LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
return node;
}
private void WaitForAccountCreation(int numberOfNodes)
private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
{
// We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files
// we will be trying to read in 'ExtractAccount', later on in the start-up process.
Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes));
}
private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts)
{
var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray();
var accounts = ExtractAccounts(marketplace, config);
return new GethCompanionNodeInfo(container, accounts);
}
private IEnumerable<GethCompanionAccount> ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts)
private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
{
for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1);
}
private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber)
{
var account = extractor.ExtractAccount(orderNumber);
var privKey = extractor.ExtractPrivateKey(orderNumber);
return new GethCompanionAccount(account, privKey);
return marketplace.Bootstrap.AllAccounts.Accounts
.Skip(1 + config.CompanionAccountStartIndex)
.Take(config.NumberOfCompanionAccounts)
.ToArray();
}
private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
{
try
{
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
Time.WaitUntil(() =>
{
var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
}
catch (Exception e)
{
@ -68,10 +60,17 @@ namespace DistTestCore.Marketplace
}
}
private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
{
var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
companionAccountIndex += numberOfAccounts;
return config;
}
private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
{
var config = new StartupConfig();
config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts));
config.Add(gethConfig);
return config;
}
}

36
DistTestCore/Marketplace/GethContainerRecipe.cs
View File

@ -4,22 +4,17 @@ namespace DistTestCore.Marketplace
{
public class GethContainerRecipe : ContainerRecipeFactory
{
public const string DockerImage = "thatbenbierens/geth-confenv:latest";
#if Arm64
public const string DockerImage = "emizzle/geth-confenv:latest";
#else
public const string DockerImage = "thatbenbierens/geth-confenv:onethousand";
#endif
public const string HttpPortTag = "http_port";
public const string DiscoveryPortTag = "disc_port";
private const string defaultArgs = "--ipcdisable --syncmode full";
public static string GetAccountFilename(int? orderNumber)
{
if (orderNumber == null) return "account_string.txt";
return $"account_string_{orderNumber.Value}.txt";
}
public static string GetPrivateKeyFilename(int? orderNumber)
{
if (orderNumber == null) return "private.key";
return $"private_{orderNumber.Value}.key";
}
public const string AccountsFilename = "accounts.csv";
protected override string Image => DockerImage;
@ -46,14 +41,17 @@ namespace DistTestCore.Marketplace
private string CreateBootstapArgs(Port discovery)
{
AddEnvVar("IS_BOOTSTRAP", "1");
AddEnvVar("ENABLE_MINER", "1");
UnlockAccounts(0, 1);
var exposedPort = AddExposedPort(tag: HttpPortTag);
return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
}
private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
{
AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString());
UnlockAccounts(
config.CompanionAccountStartIndex + 1,
config.NumberOfCompanionAccounts);
var port = AddInternalPort();
var authRpc = AddInternalPort();
@ -66,5 +64,15 @@ namespace DistTestCore.Marketplace
return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
}
private void UnlockAccounts(int startIndex, int numberOfAccounts)
{
if (startIndex < 0) throw new ArgumentException();
if (numberOfAccounts < 1) throw new ArgumentException();
if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
}
}
}

4
DistTestCore/Marketplace/GethStartupConfig.cs
View File

@ -2,15 +2,17 @@
{
public class GethStartupConfig
{
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts)
public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
{
IsBootstrapNode = isBootstrapNode;
BootstrapNode = bootstrapNode;
CompanionAccountStartIndex = companionAccountStartIndex;
NumberOfCompanionAccounts = numberOfCompanionAccounts;
}
public bool IsBootstrapNode { get; }
public GethBootstrapNodeInfo BootstrapNode { get; }
public int CompanionAccountStartIndex { get; }
public int NumberOfCompanionAccounts { get; }
}
}

4
DistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace
{
private readonly TestLog log;
private readonly MarketplaceNetwork marketplaceNetwork;
private readonly GethCompanionAccount account;
private readonly GethAccount account;
private readonly CodexAccess codexAccess;
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess)
public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
{
this.log = log;
this.marketplaceNetwork = marketplaceNetwork;

6
DistTestCore/Marketplace/MarketplaceAccessFactory.cs
View File

@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace
return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
}
private GethCompanionAccount GetGethCompanionNode(CodexAccess access)
private GethAccount GetGethCompanionNode(CodexAccess access)
{
var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount);
return (GethCompanionAccount)account;
var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
return (GethAccount)account;
}
}
}

6
DistTestCore/Metrics/MetricsAccess.cs
View File

@ -14,12 +14,14 @@ namespace DistTestCore.Metrics
public class MetricsAccess : IMetricsAccess
{
private readonly TestLog log;
private readonly ITimeSet timeSet;
private readonly MetricsQuery query;
private readonly RunningContainer node;
public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node)
public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
{
this.log = log;
this.timeSet = timeSet;
this.query = query;
this.node = node;
}
@ -47,7 +49,7 @@ namespace DistTestCore.Metrics
{
var mostRecent = GetMostRecent(metricName);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
{
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
throw new TimeoutException();

4
DistTestCore/Metrics/MetricsAccessFactory.cs
View File

@ -28,8 +28,8 @@ namespace DistTestCore.Metrics
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{
var query = new MetricsQuery(lifecycle.Log, prometheusContainer);
return new MetricsAccess(lifecycle.Log, query, codexContainer);
var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
}
}
}

3
DistTestCore/Metrics/MetricsQuery.cs
View File

@ -9,12 +9,13 @@ namespace DistTestCore.Metrics
{
private readonly Http http;
public MetricsQuery(BaseLog log, RunningContainers runningContainers)
public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
{
RunningContainers = runningContainers;
http = new Http(
log,
timeSet,
runningContainers.RunningPod.Cluster.IP,
runningContainers.Containers[0].ServicePorts[0].Number,
"api/v1");

4
DistTestCore/OnlineCodexNode.cs
View File

@ -92,6 +92,10 @@ namespace DistTestCore
public ICodexSetup BringOffline()
{
if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
"individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
"available for codex-nodes in groups of 1.");
return Group.BringOffline();
}

15
DistTestCore/TestLifecycle.cs
View File

@ -1,25 +1,30 @@
using DistTestCore.Logs;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore
{
public class TestLifecycle
{
private readonly WorkflowCreator workflowCreator;
private DateTime testStart = DateTime.MinValue;
public TestLifecycle(TestLog log, Configuration configuration)
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
{
Log = log;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration());
TimeSet = timeSet;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this, workflowCreator);
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator);
testStart = DateTime.UtcNow;
}
public TestLog Log { get; }
public ITimeSet TimeSet { get; }
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }
@ -42,5 +47,11 @@ namespace DistTestCore
return new CodexNodeLog(subFile, node);
}
public string GetTestDuration()
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
}
}
}

52
DistTestCore/Timing.cs
View File

@ -8,53 +8,11 @@ namespace DistTestCore
{
}
public static class Timing
{
public static bool UseLongTimeouts { get; set; }
public static TimeSpan HttpCallTimeout()
{
return GetTimes().HttpCallTimeout();
}
public static int HttpCallRetryCount()
{
return GetTimes().HttpCallRetryCount();
}
public static void HttpCallRetryDelay()
{
Time.Sleep(GetTimes().HttpCallRetryDelay());
}
public static TimeSpan K8sServiceDelay()
{
return GetTimes().WaitForK8sServiceDelay();
}
public static TimeSpan K8sOperationTimeout()
{
return GetTimes().K8sOperationTimeout();
}
public static TimeSpan WaitForMetricTimeout()
{
return GetTimes().WaitForMetricTimeout();
}
private static ITimeSet GetTimes()
{
if (UseLongTimeouts) return new LongTimeSet();
return new DefaultTimeSet();
}
}
public interface ITimeSet
{
TimeSpan HttpCallTimeout();
int HttpCallRetryCount();
TimeSpan HttpCallRetryDelay();
void HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout();
@ -72,9 +30,9 @@ namespace DistTestCore
return 5;
}
public TimeSpan HttpCallRetryDelay()
public void HttpCallRetryDelay()
{
return TimeSpan.FromSeconds(3);
Time.Sleep(TimeSpan.FromSeconds(3));
}
public TimeSpan WaitForK8sServiceDelay()
@ -105,9 +63,9 @@ namespace DistTestCore
return 2;
}
public TimeSpan HttpCallRetryDelay()
public void HttpCallRetryDelay()
{
return TimeSpan.FromMinutes(5);
Time.Sleep(TimeSpan.FromMinutes(5));
}
public TimeSpan WaitForK8sServiceDelay()

40
KubernetesWorkflow/ApplicationLifecycle.cs Normal file
View File

@ -0,0 +1,40 @@
using Utils;
namespace KubernetesWorkflow
{
public class ApplicationLifecycle
{
private static object instanceLock = new object();
private static ApplicationLifecycle? instance;
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource namespaceNumberSource = new NumberSource(0);
private ApplicationLifecycle()
{
}
public static ApplicationLifecycle Instance
{
// I know singletons are quite evil. But we need to be sure this object is created only once
// and persists for the entire application lifecycle.
get
{
lock (instanceLock)
{
if (instance == null) instance = new ApplicationLifecycle();
return instance;
}
}
}
public NumberSource GetServiceNumberSource()
{
return servicePortNumberSource;
}
public string GetTestNamespace()
{
return namespaceNumberSource.GetNextNumber().ToString("D5");
}
}
}

8
KubernetesWorkflow/CommandRunner.cs
View File

@ -5,7 +5,7 @@ namespace KubernetesWorkflow
{
public class CommandRunner
{
private readonly Kubernetes client;
private readonly K8sClient client;
private readonly string k8sNamespace;
private readonly RunningPod pod;
private readonly string containerName;
@ -13,7 +13,7 @@ namespace KubernetesWorkflow
private readonly string[] arguments;
private readonly List<string> lines = new List<string>();
public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
{
this.client = client;
this.k8sNamespace = k8sNamespace;
@ -27,8 +27,8 @@ namespace KubernetesWorkflow
{
var input = new[] { command }.Concat(arguments).ToArray();
Time.Wait(client.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
}
public string GetStdOut()

6
KubernetesWorkflow/Configuration.cs
View File

@ -2,16 +2,16 @@
{
public class Configuration
{
public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
{
K8sNamespace = k8sNamespace;
K8sNamespacePrefix = k8sNamespacePrefix;
KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout;
RetryDelay = retryDelay;
LocationMap = locationMap;
}
public string K8sNamespace { get; }
public string K8sNamespacePrefix { get; }
public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; }

8
KubernetesWorkflow/ContainerRecipe.cs
View File

@ -24,6 +24,14 @@
{
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
}
public override string ToString()
{
return $"(container-recipe: {Name}, image: {Image}, " +
$"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
$"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
$"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, ";
}
}
public class Port

36
KubernetesWorkflow/K8sClient.cs Normal file
View File

@ -0,0 +1,36 @@
using k8s;
namespace KubernetesWorkflow
{
public class K8sClient
{
private readonly Kubernetes client;
private static readonly object clientLock = new object();
public K8sClient(KubernetesClientConfiguration config)
{
client = new Kubernetes(config);
}
public void Run(Action<Kubernetes> action)
{
lock (clientLock)
{
action(client);
}
}
public T Run<T>(Func<Kubernetes, T> action)
{
lock (clientLock)
{
return action(client);
}
}
public void Dispose()
{
client.Dispose();
}
}
}

162
KubernetesWorkflow/K8sController.cs
View File

@ -11,16 +11,18 @@ namespace KubernetesWorkflow
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownPods;
private readonly WorkflowNumberSource workflowNumberSource;
private readonly Kubernetes client;
private readonly K8sClient client;
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
{
this.log = log;
this.cluster = cluster;
this.knownPods = knownPods;
this.workflowNumberSource = workflowNumberSource;
client = new K8sClient(cluster.GetK8sClientConfig());
client = new Kubernetes(cluster.GetK8sClientConfig());
K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
log.Debug($"Test namespace: '{K8sTestNamespace}'");
}
public void Dispose()
@ -52,14 +54,14 @@ namespace KubernetesWorkflow
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{
log.Debug();
using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name);
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
logHandler.Log(stream);
}
public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
{
log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
runner.Run();
return runner.GetStdOut();
}
@ -67,13 +69,43 @@ namespace KubernetesWorkflow
public void DeleteAllResources()
{
log.Debug();
DeleteNamespace();
var all = client.Run(c => c.ListNamespace().Items);
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
foreach (var ns in namespaces)
{
DeleteNamespace(ns);
}
foreach (var ns in namespaces)
{
WaitUntilNamespaceDeleted(ns);
}
}
public void DeleteTestNamespace()
{
log.Debug();
if (IsTestNamespaceOnline())
{
client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0));
}
WaitUntilNamespaceDeleted();
}
public void DeleteNamespace(string ns)
{
log.Debug();
if (IsNamespaceOnline(ns))
{
client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
}
}
#region Namespace management
private string K8sTestNamespace { get; }
private void EnsureTestNamespace()
{
if (IsTestNamespaceOnline()) return;
@ -83,30 +115,85 @@ namespace KubernetesWorkflow
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = K8sNamespace,
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
Name = K8sTestNamespace,
Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
}
};
client.CreateNamespace(namespaceSpec);
client.Run(c => c.CreateNamespace(namespaceSpec));
WaitUntilNamespaceCreated();
}
private void DeleteNamespace()
{
if (IsTestNamespaceOnline())
{
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
}
}
private string K8sNamespace
{
get { return cluster.Configuration.K8sNamespace; }
CreatePolicy();
}
private bool IsTestNamespaceOnline()
{
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
return IsNamespaceOnline(K8sTestNamespace);
}
private bool IsNamespaceOnline(string name)
{
return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name));
}
private void CreatePolicy()
{
client.Run(c =>
{
var body = new V1NetworkPolicy
{
Metadata = new V1ObjectMeta
{
Name = "isolate-policy",
NamespaceProperty = K8sTestNamespace
},
Spec = new V1NetworkPolicySpec
{
PodSelector = new V1LabelSelector
{
MatchLabels = GetSelector()
},
PolicyTypes = new[]
{
"Ingress",
"Egress"
},
Ingress = new List<V1NetworkPolicyIngressRule>
{
new V1NetworkPolicyIngressRule
{
FromProperty = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
NamespaceSelector = new V1LabelSelector
{
MatchLabels = GetMyNamespaceSelector()
}
}
}
}
},
Egress = new List<V1NetworkPolicyEgressRule>
{
new V1NetworkPolicyEgressRule
{
To = new List<V1NetworkPolicyPeer>
{
new V1NetworkPolicyPeer
{
NamespaceSelector = new V1LabelSelector
{
MatchLabels = GetMyNamespaceSelector()
}
}
}
}
}
}
};
c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace);
});
}
#endregion
@ -141,7 +228,7 @@ namespace KubernetesWorkflow
}
};
client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace));
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
return deploymentSpec.Metadata.Name;
@ -149,7 +236,7 @@ namespace KubernetesWorkflow
private void DeleteDeployment(string deploymentName)
{
client.DeleteNamespacedDeployment(deploymentName, K8sNamespace);
client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace));
WaitUntilDeploymentOffline(deploymentName);
}
@ -168,12 +255,18 @@ namespace KubernetesWorkflow
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
}
private IDictionary<string, string> GetMyNamespaceSelector()
{
return new Dictionary<string, string> { { "name", "thatisincorrect" } };
}
private V1ObjectMeta CreateDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "deploy-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace
NamespaceProperty = K8sTestNamespace,
Labels = GetSelector()
};
}
@ -257,14 +350,14 @@ namespace KubernetesWorkflow
}
};
client.CreateNamespacedService(serviceSpec, K8sNamespace);
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
return (serviceSpec.Metadata.Name, result);
}
private void DeleteService(string serviceName)
{
client.DeleteNamespacedService(serviceName, K8sNamespace);
client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
}
private V1ObjectMeta CreateServiceMetadata()
@ -272,7 +365,7 @@ namespace KubernetesWorkflow
return new V1ObjectMeta
{
Name = "service-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sNamespace
NamespaceProperty = K8sTestNamespace
};
}
@ -323,11 +416,16 @@ namespace KubernetesWorkflow
WaitUntil(() => !IsTestNamespaceOnline());
}
private void WaitUntilNamespaceDeleted(string name)
{
WaitUntil(() => !IsNamespaceOnline(name));
}
private void WaitUntilDeploymentOnline(string deploymentName)
{
WaitUntil(() =>
{
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace));
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}
@ -336,7 +434,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var deployments = client.ListNamespacedDeployment(K8sNamespace);
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace));
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
@ -346,7 +444,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
return pod == null;
});
@ -369,7 +467,7 @@ namespace KubernetesWorkflow
private (string, string) FetchNewPod()
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");

2
KubernetesWorkflow/KubernetesWorkflow.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>KubernetesWorkflow</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

26
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -8,14 +8,16 @@ namespace KubernetesWorkflow
private readonly WorkflowNumberSource numberSource;
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownK8SPods;
private readonly string testNamespace;
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
{
this.log = log;
this.numberSource = numberSource;
this.cluster = cluster;
this.knownK8SPods = knownK8SPods;
this.testNamespace = testNamespace;
}
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@ -62,10 +64,24 @@ namespace KubernetesWorkflow
});
}
public void DeleteTestResources()
{
K8s(controller =>
{
controller.DeleteTestNamespace();
});
}
private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
{
log.Debug();
return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r), startupConfig)).ToArray();
return recipes.Select(r =>
{
var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
return new RunningContainer(runningPod, r, servicePorts, startupConfig);
}).ToArray();
}
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@ -74,7 +90,7 @@ namespace KubernetesWorkflow
var result = new List<ContainerRecipe>();
for (var i = 0; i < numberOfContainers; i++)
{
result.Add(recipeFactory.CreateRecipe(i ,numberSource.GetContainerNumber(), componentFactory, startupConfig));
result.Add(recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig));
}
return result.ToArray();
@ -82,14 +98,14 @@ namespace KubernetesWorkflow
private void K8s(Action<K8sController> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
action(controller);
controller.Dispose();
}
private T K8s<T>(Func<K8sController, T> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
var result = action(controller);
controller.Dispose();
return result;

7
KubernetesWorkflow/WorkflowCreator.cs
View File

@ -6,25 +6,26 @@ namespace KubernetesWorkflow
public class WorkflowCreator
{
private readonly NumberSource numberSource = new NumberSource(0);
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource containerNumberSource = new NumberSource(0);
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly K8sCluster cluster;
private readonly BaseLog log;
private readonly string testNamespace;
public WorkflowCreator(BaseLog log, Configuration configuration)
{
cluster = new K8sCluster(configuration);
this.log = log;
testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
}
public StartupWorkflow CreateWorkflow()
{
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
servicePortNumberSource,
ApplicationLifecycle.Instance.GetServiceNumberSource(),
containerNumberSource);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
}
}
}

21
LOCALSETUP.MD Normal file
View File

@ -0,0 +1,21 @@
# Distributed System Tests for Nim-Codex
## Local setup
These steps will help you set up everything you need to run and debug the tests on your local system.
### Installing the requirements.
1. Install dotnet v6.0 or newer. (If you install a newer version, consider updating the .csproj files by replacing all mention of `net6.0` with your version.)
1. Set up a nice C# IDE or plugin for your current IDE.
1. Install docker desktop.
1. In the docker-desktop settings, enable kubernetes. (This might take a few minutes.)
### Configure to taste.
The tests should run as-is. You can change the configuration. The items below explain the what and how.
1. Open `DistTestCore/Configuration.cs`.
1. `k8sNamespace` defines the Kubernetes namespace the tests will use. All Kubernetes resources used during the test will be created in it. At the beginning of a test run and at the end of each test, the namespace and all resources in it will be deleted.
1. `kubeConfigFile`. If you are using the Kubernetes cluster created in docker desktop, this field should be set to null. If you wish to use a different cluster, set this field to the path (absolute or relative) of your KubeConfig file.
1. `LogConfig(path, debugEnabled)`. Path defines the path (absolute or relative) where the tests logs will be saved. The debug flag allows you to enable additional logging. This is mostly useful when something's wrong with the test infra.
1. `FileManagerFolder` defines the path (absolute or relative) where the test infra will generate and store test data files. This folder will be deleted at the end of every test run.
### Running the tests
Most IDEs will let you run individual tests or test fixtures straight from the code file. If you want to run all the tests, you can use `dotnet test`. You can control which tests to run by specifying which folder of tests to run. `dotnet test Tests` will run only the tests in `/Tests` and exclude the long tests.

2
Logging/Logging.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>Logging</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

2
LongTests/TestsLong.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

93
Nethereum/NethereumInteraction.cs
View File

@ -10,16 +10,13 @@ namespace NethereumWorkflow
{
public class NethereumInteraction
{
private readonly List<Task> openTasks = new List<Task>();
private readonly BaseLog log;
private readonly Web3 web3;
private readonly string rootAccount;
internal NethereumInteraction(BaseLog log, Web3 web3, string rootAccount)
internal NethereumInteraction(BaseLog log, Web3 web3)
{
this.log = log;
this.web3 = web3;
this.rootAccount = rootAccount;
}
public string GetTokenAddress(string marketplaceAddress)
@ -31,29 +28,13 @@ namespace NethereumWorkflow
return Time.Wait(handler.QueryAsync<string>(marketplaceAddress, function));
}
public void TransferWeiTo(string account, decimal amount)
public void MintTestTokens(string[] accounts, decimal amount, string tokenAddress)
{
log.Debug($"{amount} --> {account}");
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance");
if (amount < 1 || accounts.Length < 1) throw new ArgumentException("Invalid arguments for MintTestTokens");
var value = ToHexBig(amount);
var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value));
openTasks.Add(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId));
}
var tasks = accounts.Select(a => MintTokens(a, amount, tokenAddress));
public void MintTestTokens(string account, decimal amount, string tokenAddress)
{
log.Debug($"({tokenAddress}) {amount} --> {account}");
if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
var function = new MintTokensFunction
{
Holder = account,
Amount = ToBig(amount)
};
var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
openTasks.Add(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function));
Task.WaitAll(tasks.ToArray());
}
public decimal GetBalance(string tokenAddress, string account)
@ -68,48 +49,54 @@ namespace NethereumWorkflow
return ToDecimal(Time.Wait(handler.QueryAsync<BigInteger>(tokenAddress, function)));
}
public void WaitForAllTransactions()
public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
{
var tasks = openTasks.ToArray();
openTasks.Clear();
Task.WaitAll(tasks);
try
{
return IsBlockNumberOK() && IsContractAvailable(marketplaceAddress, marketplaceAbi);
}
catch
{
return false;
}
}
public void EnsureSynced(string marketplaceAddress, string marketplaceAbi)
private Task MintTokens(string account, decimal amount, string tokenAddress)
{
WaitUntilSynced();
WaitForContract(marketplaceAddress, marketplaceAbi);
log.Debug($"({tokenAddress}) {amount} --> {account}");
if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
var function = new MintTokensFunction
{
Holder = account,
Amount = ToBig(amount)
};
var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
return handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function);
}
private void WaitUntilSynced()
private bool IsBlockNumberOK()
{
log.Debug();
Time.WaitUntil(() =>
{
var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
var numberOfBlocks = ToDecimal(number);
return !sync.IsSyncing && numberOfBlocks > 256;
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
var numberOfBlocks = ToDecimal(number);
return !sync.IsSyncing && numberOfBlocks > 256;
}
private void WaitForContract(string marketplaceAddress, string marketplaceAbi)
private bool IsContractAvailable(string marketplaceAddress, string marketplaceAbi)
{
log.Debug();
Time.WaitUntil(() =>
try
{
try
{
var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
return contract != null;
}
catch
{
return false;
}
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
return contract != null;
}
catch
{
return false;
}
}
private HexBigInteger ToHexBig(decimal amount)

6
Nethereum/NethereumInteractionCreator.cs
View File

@ -8,21 +8,19 @@ namespace NethereumWorkflow
private readonly BaseLog log;
private readonly string ip;
private readonly int port;
private readonly string rootAccount;
private readonly string privateKey;
public NethereumInteractionCreator(BaseLog log, string ip, int port, string rootAccount, string privateKey)
public NethereumInteractionCreator(BaseLog log, string ip, int port, string privateKey)
{
this.log = log;
this.ip = ip;
this.port = port;
this.rootAccount = rootAccount;
this.privateKey = privateKey;
}
public NethereumInteraction CreateWorkflow()
{
return new NethereumInteraction(log, CreateWeb3(), rootAccount);
return new NethereumInteraction(log, CreateWeb3());
}
private Web3 CreateWeb3()

2
Nethereum/NethereumWorkflow.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>NethereumWorkflow</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

28
README.md
View File

@ -1,18 +1,28 @@
# Distributed System Tests for Nim-Codex
Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducable environment.
Nim-Codex: https://github.com/status-im/nim-codex
Dotnet: v6.0
Kubernetes: v1.25.4
Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp
Nethereum: v4.14.0
Tests are built on dotnet v6.0 and Kubernetes v1.25.4, using dotnet-kubernetes SDK: https://github.com/kubernetes-client/csharp
## Tests
Tests are devided into two assemblies: `/Tests` and `/LongTests`.
`/Tests` is to be used for tests that take several minutes to hours to execute.
`/LongTests` is to be used for tests that take hours to days to execute.
## Requirement
TODO: All tests will eventually be running as part of a dedicated CI pipeline and kubernetes cluster. Currently, we're developing these tests and the infra-code to support it by running the whole thing locally.
At this moment, the tests require a local kubernetes cluster to be installed.
## Test logs
Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
## Run
## How to contribute tests
An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.MD).
Short tests: These tests may take minutes to an hour.
`dotnet test Tests`
Long tests: These may takes hours to days.
`dotnet test LongTests`
## Run the tests on your machine
Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/LOCALSETUP.MD). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
## Missing functionality
Surely the test-infra doesn't do everything we'll need it to do. If you're running into a limitation and would like to request a new feature for the test-infra, please create an issue.

6
Tests/BasicTests/ExampleTests.cs
View File

@ -71,13 +71,13 @@ namespace Tests.BasicTests
requiredCollateral: 10.TestTokens(),
minRequiredNumberOfNodes: 1,
proofProbability: 5,
duration: TimeSpan.FromMinutes(2));
duration: TimeSpan.FromMinutes(1));
Time.Sleep(TimeSpan.FromMinutes(1));
Time.Sleep(TimeSpan.FromSeconds(10));
seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
Time.Sleep(TimeSpan.FromMinutes(2));
Time.Sleep(TimeSpan.FromMinutes(1));
seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");

46
Tests/BasicTests/NetworkIsolationTest.cs Normal file
View File

@ -0,0 +1,46 @@
using DistTestCore;
using NUnit.Framework;
using Utils;
namespace Tests.BasicTests
{
// Warning!
// This is a test to check network-isolation in the test-infrastructure.
// It requires parallelism(2) or greater to run.
[TestFixture]
[Ignore("Disabled until a solution is implemented.")]
public class NetworkIsolationTest : DistTest
{
private IOnlineCodexNode? node = null;
[Test]
public void SetUpANodeAndWait()
{
node = SetupCodexNode();
Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
}
[Test]
public void ForeignNodeConnects()
{
var myNode = SetupCodexNode();
Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
try
{
myNode.ConnectToPeer(node!);
}
catch
{
// Good! This connection should be prohibited by the network isolation policy.
node = null;
return;
}
Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " +
"This may cause cross-test interference. Network isolation policy should be applied. Test infra failure.");
}
}
}

8
Tests/BasicTests/PeerTests.cs
View File

@ -72,13 +72,13 @@ namespace Tests.BasicTests
private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
{
//var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
//Log.Debug($"Looking for {b.id} in engine-peers [{enginePeers}]");
Log.Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
//Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
}
}

7
Tests/BasicTests/TwoClientTests.cs
View File

@ -38,10 +38,15 @@ namespace Tests.BasicTests
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{
PerformTwoClientTest(primary, secondary, 1.MB());
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary, ByteSize size)
{
primary.ConnectToPeer(secondary);
var testFile = GenerateTestFile(1.MB());
var testFile = GenerateTestFile(size);
var contentId = primary.UploadFile(testFile);

6
Tests/DurabilityTests/DurabilityTests.cs
View File

@ -11,12 +11,12 @@ namespace Tests.DurabilityTests
[Test]
public void BootstrapNodeDisappearsTest()
{
var bootstrapNode = SetupCodexNode();
var bootstrapNode = SetupCodexBootstrapNode();
var group = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode));
var primary = group[0];
var secondary = group[1];
// There is 1 minute of time for the nodes to connect to each other.
// There is 1 minute of time f or the nodes to connect to each other.
// (Should be easy, they're in the same pod.)
Time.Sleep(TimeSpan.FromMinutes(6));
bootstrapNode.BringOffline();
@ -31,7 +31,7 @@ namespace Tests.DurabilityTests
[Test]
public void DataRetentionTest()
{
var bootstrapNode = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace));
var bootstrapNode = SetupCodexBootstrapNode(s => s.WithLogLevel(CodexLogLevel.Trace));
var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));

6
Tests/Parallelism.cs Normal file
View File

@ -0,0 +1,6 @@
using NUnit.Framework;
[assembly: LevelOfParallelism(1)]
namespace Tests
{
}

2
Tests/Tests.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

2
Utils/Utils.csproj
View File

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<RootNamespace>Utils</RootNamespace>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>