cs-codex-dist-tests/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs

234 lines
6.5 KiB
C#
Raw Normal View History

using CodexPlugin.Hooks;
using KubernetesWorkflow;
using Logging;
2024-07-25 14:00:51 +00:00
using OverwatchTranscript;
using Utils;
2024-07-25 14:00:51 +00:00
namespace CodexPlugin.OverwatchSupport
{
public class CodexTranscriptWriter : ICodexHooksProvider
2024-07-25 14:00:51 +00:00
{
private const string CodexHeaderKey = "cdx_h";
private readonly ILog log;
2024-07-26 07:14:46 +00:00
private readonly ITranscriptWriter writer;
private readonly CodexLogConverter converter;
private readonly NameIdMap nameIdMap = new NameIdMap();
2024-07-25 14:00:51 +00:00
public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter)
2024-07-25 14:00:51 +00:00
{
this.log = log;
2024-07-26 07:14:46 +00:00
writer = transcriptWriter;
converter = new CodexLogConverter(writer, nameIdMap);
2024-07-25 14:00:51 +00:00
}
public void Finalize(string outputFilepath)
{
log.Log("Finalizing Codex transcript...");
writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
2024-07-26 07:14:46 +00:00
writer.Write(outputFilepath);
log.Log("Done");
}
public ICodexNodeHooks CreateHooks(string nodeName)
{
nodeName = Str.Between(nodeName, "'", "'");
2024-07-26 07:14:46 +00:00
return new CodexNodeTranscriptWriter(writer, nameIdMap, nodeName);
}
public void IncludeFile(string filepath)
{
writer.IncludeArtifact(filepath);
}
2024-07-25 14:00:51 +00:00
public void ProcessLogs(IDownloadedLog[] downloadedLogs)
{
foreach (var l in downloadedLogs)
2024-07-26 07:14:46 +00:00
{
log.Log("Include artifact: " + l.GetFilepath());
writer.IncludeArtifact(l.GetFilepath());
2024-07-26 08:56:22 +00:00
// Not all of these logs are necessarily Codex logs.
// Check, and process only the Codex ones.
if (IsCodexLog(l))
2024-07-26 08:56:22 +00:00
{
log.Log("Processing Codex log: " + l.GetFilepath());
converter.ProcessLog(l);
2024-07-26 08:56:22 +00:00
}
2024-07-26 07:14:46 +00:00
}
}
public void AddResult(bool success, string result)
{
writer.Add(DateTime.UtcNow, new OverwatchCodexEvent
{
2024-07-29 09:02:24 +00:00
Name = string.Empty,
PeerId = string.Empty,
ScenarioFinished = new ScenarioFinishedEvent
{
Success = success,
Result = result
}
});
}
2024-07-26 08:56:22 +00:00
private OverwatchCodexHeader CreateCodexHeader()
{
return new OverwatchCodexHeader
{
TotalNumberOfNodes = nameIdMap.Size
};
}
2024-07-26 08:56:22 +00:00
private bool IsCodexLog(IDownloadedLog log)
{
return log.GetLinesContaining("Run Codex node").Any();
}
}
public class CodexNodeTranscriptWriter : ICodexNodeHooks
{
private readonly ITranscriptWriter writer;
2024-07-26 07:14:46 +00:00
private readonly NameIdMap nameIdMap;
private readonly string name;
private string peerId = string.Empty;
private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>();
2024-07-26 07:14:46 +00:00
public CodexNodeTranscriptWriter(ITranscriptWriter writer, NameIdMap nameIdMap, string name)
{
this.writer = writer;
2024-07-26 07:14:46 +00:00
this.nameIdMap = nameIdMap;
this.name = name;
}
public void OnNodeStarting(DateTime startUtc, string image)
{
WriteCodexEvent(startUtc, e =>
{
e.NodeStarting = new NodeStartingEvent
{
Image = image
};
});
}
public void OnNodeStarted(string peerId)
{
this.peerId = peerId;
2024-07-26 07:14:46 +00:00
nameIdMap.Add(name, peerId);
WriteCodexEvent(e =>
{
e.NodeStarted = new NodeStartedEvent
{
};
});
}
public void OnNodeStopping()
{
WriteCodexEvent(e =>
{
e.NodeStopping = new NodeStoppingEvent
{
};
});
}
public void OnFileDownloading(ContentId cid)
{
WriteCodexEvent(e =>
{
e.FileDownloading = new FileDownloadingEvent
{
Cid = cid.Id
};
});
}
public void OnFileDownloaded(ByteSize size, ContentId cid)
{
WriteCodexEvent(e =>
{
e.FileDownloaded = new FileDownloadedEvent
{
Cid = cid.Id,
ByteSize = size.SizeInBytes
};
});
}
public void OnFileUploading(string uid, ByteSize size)
{
WriteCodexEvent(e =>
{
e.FileUploading = new FileUploadingEvent
{
UniqueId = uid,
ByteSize = size.SizeInBytes
};
});
}
public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
{
WriteCodexEvent(e =>
{
e.FileUploaded = new FileUploadedEvent
{
UniqueId = uid,
Cid = cid.Id,
ByteSize = size.SizeInBytes
};
});
}
private void WriteCodexEvent(Action<OverwatchCodexEvent> action)
{
WriteCodexEvent(DateTime.UtcNow, action);
}
private void WriteCodexEvent(DateTime utc, Action<OverwatchCodexEvent> action)
{
var e = new OverwatchCodexEvent
{
2024-07-29 09:02:24 +00:00
Name = name,
PeerId = peerId
};
action(e);
if (string.IsNullOrEmpty(peerId))
{
// If we don't know our peerId, don't write the events yet.
AddToCache(utc, e);
}
else
{
e.Write(utc, writer);
// Write any events that we cached when we didn't have our peerId yet.
WriteAndClearCache();
}
}
private void AddToCache(DateTime utc, OverwatchCodexEvent e)
{
pendingEvents.Add((utc, e));
}
private void WriteAndClearCache()
{
if (pendingEvents.Any())
{
foreach (var pair in pendingEvents)
{
pair.Item2.PeerId = peerId;
pair.Item2.Write(pair.Item1, writer);
}
pendingEvents.Clear();
}
2024-07-25 14:00:51 +00:00
}
}
}