codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extracts core from disttest core.

This commit is contained in:
benbierens 2023-09-12 13:32:06 +02:00
parent 32ad778a91
commit dc1bed6861
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
28 changed files with 140 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CodexPlugin/CodexAccess.cs
View File

@ -1,4 +1,4 @@
using DistTestCore;
using Core;
using KubernetesWorkflow;
using Utils;

2
CodexPlugin/CodexContainerRecipe.cs
View File

@ -1,5 +1,5 @@
//using DistTestCore.Marketplace;
using DistTestCore;
using Core;
using KubernetesWorkflow;
using Utils;

2
CodexPlugin/CodexNodeFactory.cs
View File

@ -1,4 +1,4 @@
using DistTestCore;
using Core;
namespace CodexPlugin
{

3
CodexPlugin/CodexNodeGroup.cs
View File

@ -1,6 +1,5 @@
using DistTestCore;
using Core;
using KubernetesWorkflow;
using Logging;
using System.Collections;
namespace CodexPlugin

11
CodexPlugin/CodexPlugin.cs
View File

@ -1,4 +1,4 @@
using DistTestCore;
using Core;
using KubernetesWorkflow;
using Logging;
@ -40,7 +40,13 @@ namespace CodexPlugin
public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
{
return null!;
return SetupCodexNodes(1, setup)[0];
}
public ICodexNodeGroup SetupCodexNodes(int number, Action<ICodexSetup> setup)
{
var rc = StartCodexNodes(number, setup);
return WrapCodexContainers(rc);
}
public ICodexNodeGroup SetupCodexNodes(int number)
@ -48,6 +54,5 @@ namespace CodexPlugin
var rc = StartCodexNodes(number, s => { });
return WrapCodexContainers(rc);
}
}
}

2
CodexPlugin/CodexPlugin.csproj
View File

@ -21,7 +21,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
<ProjectReference Include="..\Core\Core.csproj" />
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
</ItemGroup>

2
CodexPlugin/CodexStarter.cs
View File

@ -1,4 +1,4 @@
using DistTestCore;
using Core;
using KubernetesWorkflow;
using Logging;

19
CodexPlugin/OnlineCodexNode.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using DistTestCore.Logs;
using Core;
using FileUtils;
using Logging;
using NUnit.Framework;
@ -12,8 +11,8 @@ namespace CodexPlugin
string GetName();
CodexDebugResponse GetDebugInfo();
CodexDebugPeerResponse GetDebugPeer(string peerId);
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
ContentId UploadFile(TrackedFile file);
TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
void ConnectToPeer(IOnlineCodexNode node);
IDownloadedLog DownloadLog(int? tailLines = null);
//IMetricsAccess Metrics { get; }
@ -62,7 +61,7 @@ namespace CodexPlugin
return CodexAccess.GetDebugPeer(peerId);
}
public ContentId UploadFile(TestFile file)
public ContentId UploadFile(TrackedFile file)
{
using var fileStream = File.OpenRead(file.Filename);
@ -80,7 +79,7 @@ namespace CodexPlugin
return new ContentId(response);
}
public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
{
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
Log(logMessage);
@ -104,7 +103,11 @@ namespace CodexPlugin
public IDownloadedLog DownloadLog(int? tailLines = null)
{
return null!; // lifecycle.DownloadLog(CodexAccess.Container, tailLines);
var workflow = tools.CreateWorkflow();
var file = tools.GetLog().CreateSubfile();
var logHandler = new LogDownloadHandler(CodexAccess.GetName(), file);
workflow.DownloadContainerLog(CodexAccess.Container, logHandler);
return logHandler.DownloadLog();
}
public void BringOffline()
@ -142,7 +145,7 @@ namespace CodexPlugin
return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
}
private void DownloadToFile(string contentId, TestFile file)
private void DownloadToFile(string contentId, TrackedFile file)
{
using var fileStream = File.OpenWrite(file.Filename);
try

7
CodexPlugin/PluginInterfaceExtensions.cs
View File

@ -1,4 +1,4 @@
using DistTestCore;
using Core;
using KubernetesWorkflow;
namespace CodexPlugin
@ -20,6 +20,11 @@ namespace CodexPlugin
return Plugin(pluginInterface).SetupCodexNode(setup);
}
public static ICodexNodeGroup SetupCodexNodes(this PluginInterface pluginInterface, int number, Action<ICodexSetup> setup)
{
return Plugin(pluginInterface).SetupCodexNodes(number, setup);
}
public static ICodexNodeGroup SetupCodexNodes(this PluginInterface pluginInterface, int number)
{
return Plugin(pluginInterface).SetupCodexNodes(number);

14
Core/Core.csproj Normal file
View File

@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\FileUtils\FileUtils.csproj" />
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
</ItemGroup>
</Project>

2
DistTestCore/DefaultContainerRecipe.cs → Core/DefaultContainerRecipe.cs
View File

@ -1,7 +1,7 @@
using KubernetesWorkflow;
using Logging;
namespace DistTestCore
namespace Core
{
public abstract class DefaultContainerRecipe : ContainerRecipeFactory
{

16
DistTestCore/Logs/DownloadedLog.cs → Core/DownloadedLog.cs
View File

@ -1,11 +1,10 @@
using Logging;
using NUnit.Framework;
namespace DistTestCore.Logs
namespace Core
{
public interface IDownloadedLog
{
void AssertLogContains(string expectedString);
bool DoesLogContain(string expectedString);
string[] FindLinesThatContain(params string[] tags);
void DeleteFile();
}
@ -13,15 +12,13 @@ namespace DistTestCore.Logs
public class DownloadedLog : IDownloadedLog
{
private readonly LogFile logFile;
private readonly string owner;
public DownloadedLog(LogFile logFile, string owner)
public DownloadedLog(LogFile logFile)
{
this.logFile = logFile;
this.owner = owner;
}
public void AssertLogContains(string expectedString)
public bool DoesLogContain(string expectedString)
{
using var file = File.OpenRead(logFile.FullFilename);
using var streamReader = new StreamReader(file);
@ -29,11 +26,12 @@ namespace DistTestCore.Logs
var line = streamReader.ReadLine();
while (line != null)
{
if (line.Contains(expectedString)) return;
if (line.Contains(expectedString)) return true;
line = streamReader.ReadLine();
}
Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
//Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
return false;
}
public string[] FindLinesThatContain(params string[] tags)

2
DistTestCore/Http.cs → Core/Http.cs
View File

@ -5,7 +5,7 @@ using System.Net.Http.Headers;
using System.Net.Http.Json;
using Utils;
namespace DistTestCore
namespace Core
{
public class Http
{

10
DistTestCore/Logs/LogDownloadHandler.cs → Core/LogDownloadHandler.cs
View File

@ -1,25 +1,23 @@
using KubernetesWorkflow;
using Logging;
namespace DistTestCore.Logs
namespace Core
{
public class LogDownloadHandler : LogHandler, ILogHandler
{
private readonly RunningContainer container;
private readonly LogFile log;
public LogDownloadHandler(RunningContainer container, string description, LogFile log)
public LogDownloadHandler(string description, LogFile log)
{
this.container = container;
this.log = log;
log.Write($"{description} -->> {log.FullFilename}");
log.WriteRaw(description);
}
public DownloadedLog DownloadLog()
public IDownloadedLog DownloadLog()
{
return new DownloadedLog(log, container.Name);
return new DownloadedLog(log);
}
protected override void ProcessLine(string line)

2
DistTestCore/PluginInterface.cs → Core/PluginInterface.cs
View File

@ -1,4 +1,4 @@
namespace DistTestCore
namespace Core
{
public abstract class PluginInterface
{

2
DistTestCore/PluginManager.cs → Core/PluginManager.cs
View File

@ -4,7 +4,7 @@ using Logging;
using System.Reflection;
using Utils;
namespace DistTestCore
namespace Core
{
public class PluginManager
{

42
DistTestCore/Timing.cs → Core/TimeSet.cs
View File

@ -1,12 +1,5 @@
using NUnit.Framework;
namespace DistTestCore
namespace Core
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class UseLongTimeoutsAttribute : PropertyAttribute
{
}
public interface ITimeSet
{
TimeSpan HttpCallTimeout();
@ -49,37 +42,4 @@ namespace DistTestCore
return TimeSpan.FromSeconds(30);
}
}
public class LongTimeSet : ITimeSet
{
public TimeSpan HttpCallTimeout()
{
return TimeSpan.FromHours(2);
}
public TimeSpan HttpCallRetryTime()
{
return TimeSpan.FromHours(5);
}
public TimeSpan HttpCallRetryDelay()
{
return TimeSpan.FromSeconds(2);
}
public TimeSpan WaitForK8sServiceDelay()
{
return TimeSpan.FromSeconds(10);
}
public TimeSpan K8sOperationTimeout()
{
return TimeSpan.FromMinutes(15);
}
public TimeSpan WaitForMetricTimeout()
{
return TimeSpan.FromMinutes(5);
}
}
}

4
DistTestCore/Configuration.cs
View File

@ -1,6 +1,4 @@
using KubernetesWorkflow;
using System.Net.NetworkInformation;
using Utils;
using Core;
namespace DistTestCore
{

4
DistTestCore/DistTest.cs
View File

@ -1,4 +1,4 @@
using DistTestCore.Logs;
using Core;
using FileUtils;
using KubernetesWorkflow;
using Logging;
@ -92,7 +92,7 @@ namespace DistTestCore
}
}
public TestFile GenerateTestFile(ByteSize size, string label = "")
public TrackedFile GenerateTestFile(ByteSize size, string label = "")
{
return Get().FileManager.GenerateTestFile(size, label);
}

1
DistTestCore/DistTestCore.csproj
View File

@ -19,6 +19,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Core\Core.csproj" />
<ProjectReference Include="..\FileUtils\FileUtils.csproj" />
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
<ProjectReference Include="..\Logging\Logging.csproj" />

2
DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs → DistTestCore/DontDownloadLogsAndMetricsOnFailureAttribute.cs
View File

@ -1,6 +1,6 @@
using NUnit.Framework;
namespace DistTestCore.Logs
namespace DistTestCore
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute

37
DistTestCore/LongTimeSet.cs Normal file
View File

@ -0,0 +1,37 @@
using Core;
namespace DistTestCore
{
public class LongTimeSet : ITimeSet
{
public TimeSpan HttpCallTimeout()
{
return TimeSpan.FromHours(2);
}
public TimeSpan HttpCallRetryTime()
{
return TimeSpan.FromHours(5);
}
public TimeSpan HttpCallRetryDelay()
{
return TimeSpan.FromSeconds(2);
}
public TimeSpan WaitForK8sServiceDelay()
{
return TimeSpan.FromSeconds(10);
}
public TimeSpan K8sOperationTimeout()
{
return TimeSpan.FromMinutes(15);
}
public TimeSpan WaitForMetricTimeout()
{
return TimeSpan.FromMinutes(5);
}
}
}

9
DistTestCore/LongTimeoutsTestAttribute.cs Normal file
View File

@ -0,0 +1,9 @@
using NUnit.Framework;
namespace DistTestCore
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class UseLongTimeoutsAttribute : PropertyAttribute
{
}
}

3
DistTestCore/TestLifecycle.cs
View File

@ -1,4 +1,5 @@
using FileUtils;
using Core;
using FileUtils;
using KubernetesWorkflow;
using Logging;
using Utils;

22
FileUtils/FileManager.cs
View File

@ -6,8 +6,8 @@ namespace FileUtils
{
public interface IFileManager
{
TestFile CreateEmptyTestFile(string label = "");
TestFile GenerateTestFile(ByteSize size, string label = "");
TrackedFile CreateEmptyTestFile(string label = "");
TrackedFile GenerateTestFile(ByteSize size, string label = "");
void DeleteAllTestFiles();
void ScopedFiles(Action action);
T ScopedFiles<T>(Func<T> action);
@ -20,7 +20,7 @@ namespace FileUtils
private readonly Random random = new Random();
private readonly ILog log;
private readonly string folder;
private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
public FileManager(ILog log, string rootFolder)
{
@ -30,16 +30,16 @@ namespace FileUtils
this.log = log;
}
public TestFile CreateEmptyTestFile(string label = "")
public TrackedFile CreateEmptyTestFile(string label = "")
{
var path = Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin");
var result = new TestFile(log, path, label);
var result = new TrackedFile(log, path, label);
File.Create(result.Filename).Close();
if (fileSetStack.Any()) fileSetStack.Last().Add(result);
return result;
}
public TestFile GenerateTestFile(ByteSize size, string label)
public TrackedFile GenerateTestFile(ByteSize size, string label)
{
var sw = Stopwatch.Begin(log);
var result = GenerateFile(size, label);
@ -69,7 +69,7 @@ namespace FileUtils
private void PushFileSet()
{
fileSetStack.Add(new List<TestFile>());
fileSetStack.Add(new List<TrackedFile>());
}
private void PopFileSet()
@ -88,7 +88,7 @@ namespace FileUtils
}
}
private TestFile GenerateFile(ByteSize size, string label)
private TrackedFile GenerateFile(ByteSize size, string label)
{
var result = CreateEmptyTestFile(label);
CheckSpaceAvailable(result, size);
@ -97,7 +97,7 @@ namespace FileUtils
return result;
}
private void CheckSpaceAvailable(TestFile testFile, ByteSize size)
private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
{
var file = new FileInfo(testFile.Filename);
var drive = new DriveInfo(file.Directory!.Root.FullName);
@ -115,7 +115,7 @@ namespace FileUtils
}
}
private void GenerateFileBytes(TestFile result, ByteSize size)
private void GenerateFileBytes(TrackedFile result, ByteSize size)
{
long bytesLeft = size.SizeInBytes;
int chunkSize = ChunkSize;
@ -135,7 +135,7 @@ namespace FileUtils
}
}
private void AppendRandomBytesToFile(TestFile result, long length)
private void AppendRandomBytesToFile(TrackedFile result, long length)
{
var bytes = new byte[length];
random.NextBytes(bytes);

10
FileUtils/TestFile.cs → FileUtils/TrackedFile.cs
View File

@ -4,11 +4,11 @@ using Utils;
namespace FileUtils
{
public class TestFile
public class TrackedFile
{
private readonly ILog log;
public TestFile(ILog log, string filename, string label)
public TrackedFile(ILog log, string filename, string label)
{
this.log = log;
Filename = filename;
@ -18,7 +18,7 @@ namespace FileUtils
public string Filename { get; }
public string Label { get; }
public void AssertIsEqual(TestFile? actual)
public void AssertIsEqual(TrackedFile? actual)
{
var sw = Stopwatch.Begin(log);
try
@ -27,7 +27,7 @@ namespace FileUtils
}
finally
{
sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}");
sw.End($"{nameof(TrackedFile)}.{nameof(AssertIsEqual)}");
}
}
@ -38,7 +38,7 @@ namespace FileUtils
return $"'{Filename}'{sizePostfix}";
}
private void AssertEqual(TestFile? actual)
private void AssertEqual(TrackedFile? actual)
{
if (actual == null) Assert.Fail("TestFile is null.");
if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");

4
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -7,7 +7,7 @@ namespace KubernetesWorkflow
{
RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
void Stop(RunningContainers runningContainers);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
string ExecuteCommand(RunningContainer container, string command, params string[] args);
void DeleteNamespace();
void DeleteNamespacesStartingWith(string namespacePrefix);
@ -53,7 +53,7 @@ namespace KubernetesWorkflow
});
}
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
{
K8s(controller =>
{

10
cs-codex-dist-testing.sln
View File

@ -23,9 +23,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArgsUniform", "ArgsUniform\
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FileUtils", "FileUtils\FileUtils.csproj", "{ECC954DA-8D4E-49EE-83AD-80085A43DEEB}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FileUtils", "FileUtils\FileUtils.csproj", "{ECC954DA-8D4E-49EE-83AD-80085A43DEEB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexPlugin", "CodexPlugin\CodexPlugin.csproj", "{DE4E802C-288C-45C4-84B6-8A5A6A96EF49}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexPlugin", "CodexPlugin\CodexPlugin.csproj", "{DE4E802C-288C-45C4-84B6-8A5A6A96EF49}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Core", "Core\Core.csproj", "{F2BF34B3-C660-43EF-BD42-BC5C60237FC4}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -81,6 +83,10 @@ Global
{DE4E802C-288C-45C4-84B6-8A5A6A96EF49}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DE4E802C-288C-45C4-84B6-8A5A6A96EF49}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DE4E802C-288C-45C4-84B6-8A5A6A96EF49}.Release|Any CPU.Build.0 = Release|Any CPU
{F2BF34B3-C660-43EF-BD42-BC5C60237FC4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F2BF34B3-C660-43EF-BD42-BC5C60237FC4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F2BF34B3-C660-43EF-BD42-BC5C60237FC4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F2BF34B3-C660-43EF-BD42-BC5C60237FC4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE