status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

successful transcript for two-client test

This commit is contained in:
benbierens 2024-07-26 08:39:27 +02:00
parent d58d65b751
commit 927ccaa119
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
12 changed files with 206 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -1,4 +1,5 @@
using Core;
using CodexPlugin.Hooks;
using Core;
using FileUtils;
using GethPlugin;
using KubernetesWorkflow;
@ -12,6 +13,7 @@ namespace CodexPlugin
public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
{
string GetName();
string GetPeerId();
DebugInfo GetDebugInfo();
DebugPeer GetDebugPeer(string peerId);
ContentId UploadFile(TrackedFile file);
@ -40,20 +42,28 @@ namespace CodexPlugin
{
private const string UploadFailedMessage = "Unable to store block";
private readonly IPluginTools tools;
private readonly ICodexNodeHooks hooks;
private readonly EthAccount? ethAccount;
private readonly TransferSpeeds transferSpeeds;
private string peerId = string.Empty;
public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAccount? ethAccount)
public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks, EthAccount? ethAccount)
{
this.tools = tools;
this.ethAccount = ethAccount;
CodexAccess = codexAccess;
Group = group;
Marketplace = marketplaceAccess;
this.hooks = hooks;
Version = new DebugInfoVersion();
transferSpeeds = new TransferSpeeds();
}
public void Initialize()
{
hooks.OnNodeStarted(peerId, Container.Recipe.Image);
}
public RunningPod Pod { get { return CodexAccess.Container; } }
public RunningContainer Container { get { return Pod.Containers.Single(); } }
@ -95,6 +105,11 @@ namespace CodexPlugin
return Container.Name;
}
public string GetPeerId()
{
return peerId;
}
public DebugInfo GetDebugInfo()
{
var debugInfo = CodexAccess.GetDebugInfo();
@ -132,7 +147,9 @@ namespace CodexPlugin
Log($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
var cid = new ContentId(response);
hooks.OnFileUploaded(cid);
return cid;
}
public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
@ -148,6 +165,7 @@ namespace CodexPlugin
var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
hooks.OnFileDownloaded(contentId);
return file;
}
@ -185,6 +203,8 @@ namespace CodexPlugin
public void Stop(bool waitTillStopped)
{
Log("Stopping...");
hooks.OnNodeStopping();
CrashWatcher.Stop();
Group.Stop(this, waitTillStopped);
}
@ -192,7 +212,7 @@ namespace CodexPlugin
public void EnsureOnlineGetVersionResponse()
{
var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
var nodePeerId = debugInfo.Id;
peerId = debugInfo.Id;
var nodeName = CodexAccess.Container.Name;
if (!debugInfo.Version.IsValid())
@ -201,8 +221,8 @@ namespace CodexPlugin
}
var log = tools.GetLog();
log.AddStringReplace(nodePeerId, nodeName);
log.AddStringReplace(ToShortIdString(nodePeerId), nodeName);
log.AddStringReplace(peerId, nodeName);
log.AddStringReplace(ToShortIdString(peerId), nodeName);
log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
log.AddStringReplace(ToShortIdString(debugInfo.Table.LocalNode.NodeId), nodeName);
Version = debugInfo.Version;

10
ProjectPlugins/CodexPlugin/CodexNodeFactory.cs
View File

@ -1,4 +1,5 @@
using Core;
using CodexPlugin.Hooks;
using Core;
using GethPlugin;
using KubernetesWorkflow;
using KubernetesWorkflow.Types;
@ -14,17 +15,20 @@ namespace CodexPlugin
public class CodexNodeFactory : ICodexNodeFactory
{
private readonly IPluginTools tools;
private readonly CodexHooksFactory codexHooksFactory;
public CodexNodeFactory(IPluginTools tools)
public CodexNodeFactory(IPluginTools tools, CodexHooksFactory codexHooksFactory)
{
this.tools = tools;
this.codexHooksFactory = codexHooksFactory;
}
public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
{
var ethAccount = GetEthAccount(access);
var marketplaceAccess = GetMarketplaceAccess(access, ethAccount);
return new CodexNode(tools, access, group, marketplaceAccess, ethAccount);
var hooks = codexHooksFactory.CreateHooks(access.Container.Name);
return new CodexNode(tools, access, group, marketplaceAccess, hooks, ethAccount);
}
private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount)

1
ProjectPlugins/CodexPlugin/CodexNodeGroup.cs
View File

@ -79,6 +79,7 @@ namespace CodexPlugin
}
Version = first;
foreach (var node in Nodes) node.Initialize();
}
private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)

6
ProjectPlugins/CodexPlugin/CodexPlugin.cs
View File

@ -1,3 +1,4 @@
using CodexPlugin.Hooks;
using Core;
using KubernetesWorkflow.Types;
@ -57,6 +58,11 @@ namespace CodexPlugin
}
}
public void SetCodexHooksProvider(ICodexHooksProvider hooksProvider)
{
codexStarter.HooksFactory.Provider = hooksProvider;
}
private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)
{
var codexSetup = new CodexSetup(numberOfNodes);

1
ProjectPlugins/CodexPlugin/CodexPlugin.csproj
View File

@ -29,6 +29,7 @@
<ItemGroup>
<ProjectReference Include="..\..\Framework\Core\Core.csproj" />
<ProjectReference Include="..\..\Framework\KubernetesWorkflow\KubernetesWorkflow.csproj" />
<ProjectReference Include="..\..\Tools\OverwatchTranscript\OverwatchTranscript.csproj" />
<ProjectReference Include="..\CodexContractsPlugin\CodexContractsPlugin.csproj" />
<ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />
<ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />

7
ProjectPlugins/CodexPlugin/CodexStarter.cs
View File

@ -1,4 +1,5 @@
using Core;
using CodexPlugin.Hooks;
using Core;
using KubernetesWorkflow;
using KubernetesWorkflow.Types;
using Logging;
@ -19,6 +20,8 @@ namespace CodexPlugin
apiChecker = new ApiChecker(pluginTools);
}
public CodexHooksFactory HooksFactory { get; } = new CodexHooksFactory();
public RunningPod[] BringOnline(CodexSetup codexSetup)
{
LogSeparator();
@ -43,7 +46,7 @@ namespace CodexPlugin
public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
{
var codexNodeFactory = new CodexNodeFactory(pluginTools);
var codexNodeFactory = new CodexNodeFactory(pluginTools, HooksFactory);
var group = CreateCodexGroup(coreInterface, containers, codexNodeFactory);

8
ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs
View File

@ -1,4 +1,5 @@
using Core;
using CodexPlugin.Hooks;
using Core;
using KubernetesWorkflow.Types;
namespace CodexPlugin
@ -38,6 +39,11 @@ namespace CodexPlugin
return ci.StartCodexNodes(number, s => { });
}
public static void SetCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider)
{
Plugin(ci).SetCodexHooksProvider(hooksProvider);
}
private static CodexPlugin Plugin(CoreInterface ci)
{
return ci.GetPlugin<CodexPlugin>();

44
ProjectPlugins/CodexPlugin/Hooks/CodexHooksFactory.cs Normal file
View File

@ -0,0 +1,44 @@
namespace CodexPlugin.Hooks
{
public interface ICodexHooksProvider
{
ICodexNodeHooks CreateHooks(string nodeName);
}
public class CodexHooksFactory
{
public ICodexHooksProvider Provider { get; set; } = new DoNothingHooksProvider();
public ICodexNodeHooks CreateHooks(string nodeName)
{
return Provider.CreateHooks(nodeName);
}
}
public class DoNothingHooksProvider : ICodexHooksProvider
{
public ICodexNodeHooks CreateHooks(string nodeName)
{
return new DoNothingCodexHooks();
}
}
public class DoNothingCodexHooks : ICodexNodeHooks
{
public void OnFileDownloaded(ContentId cid)
{
}
public void OnFileUploaded(ContentId cid)
{
}
public void OnNodeStarted(string name, string image)
{
}
public void OnNodeStopping()
{
}
}
}

10
ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs Normal file
View File

@ -0,0 +1,10 @@
namespace CodexPlugin.Hooks
{
public interface ICodexNodeHooks
{
void OnNodeStarted(string peerId, string image);
void OnNodeStopping();
void OnFileUploaded(ContentId cid);
void OnFileDownloaded(ContentId cid);
}
}

91
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs
View File

@ -1,9 +1,10 @@
using Core;
using CodexPlugin.Hooks;
using Core;
using OverwatchTranscript;
namespace CodexPlugin.OverwatchSupport
{
public class CodexTranscriptWriter
public class CodexTranscriptWriter : ICodexHooksProvider
{
private readonly ITranscriptWriter transcriptWriter;
@ -12,9 +13,93 @@ namespace CodexPlugin.OverwatchSupport
this.transcriptWriter = transcriptWriter;
}
public void Finalize(string outputFilepath)
{
transcriptWriter.Write(outputFilepath);
}
public ICodexNodeHooks CreateHooks(string nodeName)
{
return new CodexNodeTranscriptWriter(transcriptWriter, nodeName);
}
public void ProcessLogs(IDownloadedLog[] downloadedLogs)
{
throw new NotImplementedException();
// which logs to which nodes?
// nodeIDs, peerIDs needed.
}
}
public class CodexNodeTranscriptWriter : ICodexNodeHooks
{
private readonly ITranscriptWriter writer;
private readonly string name;
private string peerId = string.Empty;
public CodexNodeTranscriptWriter(ITranscriptWriter writer, string name)
{
this.writer = writer;
this.name = name;
}
public void OnNodeStarted(string peerId, string image)
{
this.peerId = peerId;
WriteCodexEvent(e =>
{
e.NodeStarted = new NodeStartedEvent
{
Name = name,
Image = image,
Args = string.Empty
};
});
}
public void OnNodeStopping()
{
WriteCodexEvent(e =>
{
e.NodeStopped = new NodeStoppedEvent
{
Name = name
};
});
}
public void OnFileDownloaded(ContentId cid)
{
WriteCodexEvent(e =>
{
e.FileDownloaded = new FileDownloadedEvent
{
Cid = cid.Id
};
});
}
public void OnFileUploaded(ContentId cid)
{
WriteCodexEvent(e =>
{
e.FileUploaded = new FileUploadedEvent
{
Cid = cid.Id
};
});
}
private void WriteCodexEvent(Action<OverwatchCodexEvent> action)
{
if (string.IsNullOrEmpty(peerId)) throw new Exception("PeerId required");
var e = new OverwatchCodexEvent
{
PeerId = peerId
};
action(e);
writer.Add(DateTime.UtcNow, e);
}
}
}

1
ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs
View File

@ -9,6 +9,7 @@
[Serializable]
public class OverwatchCodexEvent
{
public string PeerId { get; set; } = string.Empty;
public ScenarioFinishedEvent? ScenarioFinished { get; set; }
public NodeStartedEvent? NodeStarted { get; set; }
public NodeStoppedEvent? NodeStopped { get; set; }

16
Tests/CodexTests/CodexDistTest.cs
View File

@ -7,6 +7,7 @@ using Core;
using DistTestCore;
using DistTestCore.Helpers;
using DistTestCore.Logs;
using Logging;
using MetricsPlugin;
using Newtonsoft.Json;
using NUnit.Framework.Constraints;
@ -39,7 +40,9 @@ namespace CodexTests
base.LifecycleStart(lifecycle);
if (!enableOverwatchTranscript) return;
writers.Add(lifecycle, new CodexTranscriptWriter(Transcript.NewWriter()));
var writer = new CodexTranscriptWriter(Transcript.NewWriter());
Ci.SetCodexHooksProvider(writer);
writers.Add(lifecycle, writer);
}
protected override void LifecycleStop(TestLifecycle lifecycle)
@ -51,6 +54,12 @@ namespace CodexTests
writers.Remove(lifecycle);
writer.ProcessLogs(lifecycle.DownloadAllLogs());
var file = lifecycle.Log.CreateSubfile("owts");
Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {file.FullFilename}", () =>
{
writer.Finalize(file.FullFilename);
});
}
public ICodexNode StartCodex()
@ -74,11 +83,6 @@ namespace CodexTests
{
setup(s);
OnCodexSetup(s);
if (enableOverwatchTranscript)
{
s.WithTranscriptWriter(writers[Get()]);
}
});
return group;