status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-01-11 09:06:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sets up starting event and bootstrap event

Browse Source
This commit is contained in:
benbierens 2024-07-29 10:16:37 +02:00
parent 9bebc23f6a
commit 87271f4f37
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
11 changed files with 133 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs
View File

@ -2,7 +2,7 @@
{
public class ContainerRecipe
{
public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
public ContainerRecipe(DateTime recipeCreatedUtc, int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
{
Number = number;
NameOverride = nameOverride;
@ -31,6 +31,7 @@
if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
}
public DateTime RecipeCreatedUtc { get; }
public string Name { get; }
public int Number { get; }
public string? NameOverride { get; }

2
Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs
View File

@ -25,7 +25,7 @@ namespace KubernetesWorkflow.Recipe
Initialize(config);
var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
var recipe = new ContainerRecipe(DateTime.UtcNow, containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
exposedPorts.ToArray(),
internalPorts.ToArray(),
envVars.ToArray(),

7
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -59,9 +59,14 @@ namespace CodexPlugin
transferSpeeds = new TransferSpeeds();
}
public void Awake()
{
hooks.OnNodeStarting(Container.Recipe.RecipeCreatedUtc, Container.Recipe.Image);
}
public void Initialize()
{
hooks.OnNodeStarted(peerId, Container.Recipe.Image);
hooks.OnNodeStarted(peerId);
}
public RunningPod Pod { get { return CodexAccess.Container; } }

4
ProjectPlugins/CodexPlugin/CodexNodeGroup.cs
View File

@ -86,7 +86,9 @@ namespace CodexPlugin
{
var watcher = factory.CreateCrashWatcher(c.Containers.Single());
var access = new CodexAccess(tools, c, watcher);
return factory.CreateOnlineCodexNode(access, this);
var node = factory.CreateOnlineCodexNode(access, this);
node.Awake();
return node;
}
}
}

6
ProjectPlugins/CodexPlugin/Hooks/CodexHooksFactory.cs
View File

@ -33,7 +33,11 @@
{
}
public void OnNodeStarted(string name, string image)
public void OnNodeStarted(string peerId)
{
}
public void OnNodeStarting(DateTime startUtc, string name)
{
}

3
ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs
View File

@ -2,7 +2,8 @@
{
public interface ICodexNodeHooks
{
void OnNodeStarted(string peerId, string image);
void OnNodeStarting(DateTime startUtc, string image);
void OnNodeStarted(string peerId);
void OnNodeStopping();
void OnFileUploaded(ContentId cid);
void OnFileDownloaded(ContentId cid);

3
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs
View File

@ -54,7 +54,8 @@ namespace CodexPlugin.OverwatchSupport
private readonly string peerId;
private readonly ILineConverter[] converters = new ILineConverter[]
{
new BlockReceivedLineConverter()
new BlockReceivedLineConverter(),
new BootstrapLineConverter()
};
public ConversionRunner(ITranscriptWriter writer, string peerId)

35
ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs
View File

@ -7,6 +7,7 @@ namespace CodexPlugin.OverwatchSupport
{
public class CodexTranscriptWriter : ICodexHooksProvider
{
private const string CodexHeaderKey = "cdx_h";
private readonly ITranscriptWriter writer;
private readonly CodexLogConverter converter;
private readonly NameIdMap nameIdMap = new NameIdMap();
@ -19,6 +20,8 @@ namespace CodexPlugin.OverwatchSupport
public void Finalize(string outputFilepath)
{
writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
writer.Write(outputFilepath);
we need:
@ -65,6 +68,14 @@ namespace CodexPlugin.OverwatchSupport
});
}
private OverwatchCodexHeader CreateCodexHeader()
{
return new OverwatchCodexHeader
{
TotalNumberOfNodes = nameIdMap.Size
};
}
private bool IsCodexLog(IDownloadedLog log)
{
return log.GetLinesContaining("Run Codex node").Any();
@ -85,7 +96,19 @@ namespace CodexPlugin.OverwatchSupport
this.name = name;
}
public void OnNodeStarted(string peerId, string image)
public void OnNodeStarting(DateTime startUtc, string image)
{
WriteCodexEvent(startUtc, e =>
{
e.NodeStarting = new NodeStartingEvent
{
Name = name,
Image = image
};
});
}
public void OnNodeStarted(string peerId)
{
this.peerId = peerId;
nameIdMap.Add(name, peerId);
@ -93,9 +116,6 @@ namespace CodexPlugin.OverwatchSupport
{
e.NodeStarted = new NodeStartedEvent
{
Name = name,
Image = image,
Args = string.Empty
};
});
}
@ -135,15 +155,18 @@ namespace CodexPlugin.OverwatchSupport
private void WriteCodexEvent(Action<OverwatchCodexEvent> action)
{
if (string.IsNullOrEmpty(peerId)) throw new Exception("PeerId required");
WriteCodexEvent(DateTime.UtcNow, action);
}
private void WriteCodexEvent(DateTime utc, Action<OverwatchCodexEvent> action)
{
var e = new OverwatchCodexEvent
{
PeerId = peerId
};
action(e);
writer.Add(DateTime.UtcNow, e);
writer.Add(utc, e);
}
}
}

64
ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs Normal file
View File

@ -0,0 +1,64 @@
namespace CodexPlugin.OverwatchSupport.LineConverters
{
public class BootstrapLineConverter : ILineConverter
{
public string Interest => "Starting codex node";
public void Process(CodexLogLine line, Action<Action<OverwatchCodexEvent>> addEvent)
{
// "(
// configFile: none(InputFile),
// logLevel: \"TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session,dialer,muxedupgrade,upgrade,identify;warn:contracts,clock;warn:serde,json,serialization\",
// logFormat: auto,
// metricsEnabled: false,
// metricsAddress: 127.0.0.1,
// metricsPort: 8008,
// dataDir: datadir5,
// circuitDir: /root/.cache/codex/circuits,
// listenAddrs: @[/ip4/0.0.0.0/tcp/8081],
// nat: 10.1.0.214,
// discoveryIp: 0.0.0.0,
// discoveryPort: 8080,
// netPrivKeyFile: \"key\",
// bootstrapNodes:
// @[(envelope: (publicKey: secp256k1 key (0414380858330307a4a59e8a1643512f80680dedb8c541674f40382be71c26556cd65b7e1775ec9fabfaf6d58d562b88a3c8afb969f8cc256db20e4c4c9e1f70a6),
// domain: \"libp2p-peer-record\",
// payloadType: @[3, 1],
// payload: @[10, 39, 0, 37, 8, 2, 18, 33, 2, 20, 56, 8, 88, 51, 3, 7, 164, 165, 158, 138, 22, 67, 81, 47, 128, 104, 13, 237, 184, 197, 65, 103, 79, 64, 56, 43, 231, 28, 38, 85, 108, 16, 254, 211, 141, 181, 6, 26, 11, 10, 9, 4, 10, 1, 0, 210, 145, 2, 31, 144],
// signature: 3045022100FA846871D96EDCA579990244B1C590E16B57A7BDB817908A2B580043F8A0B0280220342825DBE577E83C22CD59AA9099DE8F8DC86A38C659F60E8B9255126CF37FDE),
// data:
// (peerId: 16Uiu2HAkvnbgwdB2NmNe1uWGJxE3Ep3sDHxNW95W2rTPs2vfxvou,
// seqNo: 1721985534,
// addresses: @[(address: /ip4/10.1.0.210/udp/8080)]
// )
// )],
// maxPeers: 160,
// agentString: \"Codex\",
// apiBindAddress: \"0.0.0.0\",
// apiPort: 30035,
// apiCorsAllowedOrigin: none(string),
// repoKind: fs,
// storageQuota: 8589934592\'NByte,
// blockTtl: 1d,
// blockMaintenanceInterval: 10m,
// blockMaintenanceNumberOfBlocks: 1000,
// cacheSize: 0\'NByte,
// logFile: none(string),
// cmd: noCmd
// )"
var config = line.Attributes["config"];
var openIndex = config.IndexOf("peerId:") + 7;
var closeIndex = config.IndexOf(",", openIndex);
var bootPeerId = config.Substring(openIndex, closeIndex - openIndex);
addEvent(e =>
{
e.BootstrapConfig = new BootstrapConfigEvent
{
BootstrapPeerId = bootPeerId
};
});
}
}
}

16
ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs
View File

@ -11,8 +11,10 @@
{
public string PeerId { get; set; } = string.Empty;
public ScenarioFinishedEvent? ScenarioFinished { get; set; }
public NodeStartingEvent? NodeStarting { get; set; }
public NodeStartedEvent? NodeStarted { get; set; }
public NodeStoppedEvent? NodeStopped { get; set; }
public BootstrapConfigEvent? BootstrapConfig { get; set; }
public FileUploadedEvent? FileUploaded { get; set; }
public FileDownloadedEvent? FileDownloaded { get; set; }
public BlockReceivedEvent? BlockReceived { get; set; }
@ -28,11 +30,15 @@
}
[Serializable]
public class NodeStartedEvent
public class NodeStartingEvent
{
public string Name { get; set; } = string.Empty;
public string Image { get; set; } = string.Empty;
public string Args { get; set; } = string.Empty;
}
[Serializable]
public class NodeStartedEvent
{
}
[Serializable]
@ -41,6 +47,12 @@
public string Name { get; set; } = string.Empty;
}
[Serializable]
public class BootstrapConfigEvent
{
public string BootstrapPeerId { get; set; } = string.Empty;
}
[Serializable]
public class FileUploadedEvent
{

5
ProjectPlugins/CodexPlugin/OverwatchSupport/NameIdMap.cs
View File

@ -13,5 +13,10 @@
{
return map[name];
}
public int Size
{
get { return map.Count; }
}
}
}