2024-07-26 06:39:27 +00:00
|
|
|
|
using CodexPlugin.Hooks;
|
2024-08-01 08:39:06 +00:00
|
|
|
|
using KubernetesWorkflow;
|
|
|
|
|
using Logging;
|
2024-07-25 14:00:51 +00:00
|
|
|
|
using OverwatchTranscript;
|
2024-07-26 08:11:29 +00:00
|
|
|
|
using Utils;
|
2024-07-25 14:00:51 +00:00
|
|
|
|
|
|
|
|
|
namespace CodexPlugin.OverwatchSupport
|
|
|
|
|
{
|
2024-07-26 06:39:27 +00:00
|
|
|
|
public class CodexTranscriptWriter : ICodexHooksProvider
|
2024-07-25 14:00:51 +00:00
|
|
|
|
{
|
2024-07-29 08:16:37 +00:00
|
|
|
|
private const string CodexHeaderKey = "cdx_h";
|
2024-08-01 08:39:06 +00:00
|
|
|
|
private readonly ILog log;
|
2024-07-26 07:14:46 +00:00
|
|
|
|
private readonly ITranscriptWriter writer;
|
|
|
|
|
private readonly CodexLogConverter converter;
|
|
|
|
|
private readonly NameIdMap nameIdMap = new NameIdMap();
|
2024-07-25 14:00:51 +00:00
|
|
|
|
|
2024-08-01 08:39:06 +00:00
|
|
|
|
public CodexTranscriptWriter(ILog log, ITranscriptWriter transcriptWriter)
|
2024-07-25 14:00:51 +00:00
|
|
|
|
{
|
2024-08-01 08:39:06 +00:00
|
|
|
|
this.log = log;
|
2024-07-26 07:14:46 +00:00
|
|
|
|
writer = transcriptWriter;
|
|
|
|
|
converter = new CodexLogConverter(writer, nameIdMap);
|
2024-07-25 14:00:51 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-07-26 06:39:27 +00:00
|
|
|
|
public void Finalize(string outputFilepath)
|
|
|
|
|
{
|
2024-08-01 08:39:06 +00:00
|
|
|
|
log.Log("Finalizing Codex transcript...");
|
2024-07-29 08:16:37 +00:00
|
|
|
|
|
2024-08-01 08:39:06 +00:00
|
|
|
|
writer.AddHeader(CodexHeaderKey, CreateCodexHeader());
|
2024-07-26 07:14:46 +00:00
|
|
|
|
writer.Write(outputFilepath);
|
2024-08-01 08:39:06 +00:00
|
|
|
|
|
|
|
|
|
log.Log("Done");
|
2024-07-26 06:39:27 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public ICodexNodeHooks CreateHooks(string nodeName)
|
|
|
|
|
{
|
2024-07-26 08:11:29 +00:00
|
|
|
|
nodeName = Str.Between(nodeName, "'", "'");
|
2024-07-26 07:14:46 +00:00
|
|
|
|
return new CodexNodeTranscriptWriter(writer, nameIdMap, nodeName);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void IncludeFile(string filepath)
|
|
|
|
|
{
|
|
|
|
|
writer.IncludeArtifact(filepath);
|
2024-07-26 06:39:27 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-07-25 14:00:51 +00:00
|
|
|
|
public void ProcessLogs(IDownloadedLog[] downloadedLogs)
|
|
|
|
|
{
|
2024-08-01 08:39:06 +00:00
|
|
|
|
foreach (var l in downloadedLogs)
|
2024-07-26 07:14:46 +00:00
|
|
|
|
{
|
2024-08-01 08:39:06 +00:00
|
|
|
|
log.Log("Include artifact: " + l.GetFilepath());
|
|
|
|
|
writer.IncludeArtifact(l.GetFilepath());
|
|
|
|
|
|
2024-07-26 08:56:22 +00:00
|
|
|
|
// Not all of these logs are necessarily Codex logs.
|
|
|
|
|
// Check, and process only the Codex ones.
|
2024-08-01 08:39:06 +00:00
|
|
|
|
if (IsCodexLog(l))
|
2024-07-26 08:56:22 +00:00
|
|
|
|
{
|
2024-08-01 08:39:06 +00:00
|
|
|
|
log.Log("Processing Codex log: " + l.GetFilepath());
|
|
|
|
|
converter.ProcessLog(l);
|
2024-07-26 08:56:22 +00:00
|
|
|
|
}
|
2024-07-26 07:14:46 +00:00
|
|
|
|
}
|
2024-07-26 06:39:27 +00:00
|
|
|
|
}
|
2024-07-26 08:11:29 +00:00
|
|
|
|
|
|
|
|
|
public void AddResult(bool success, string result)
|
|
|
|
|
{
|
|
|
|
|
writer.Add(DateTime.UtcNow, new OverwatchCodexEvent
|
|
|
|
|
{
|
2024-07-29 09:02:24 +00:00
|
|
|
|
Name = string.Empty,
|
2024-07-26 08:11:29 +00:00
|
|
|
|
PeerId = string.Empty,
|
|
|
|
|
ScenarioFinished = new ScenarioFinishedEvent
|
|
|
|
|
{
|
|
|
|
|
Success = success,
|
|
|
|
|
Result = result
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
2024-07-26 08:56:22 +00:00
|
|
|
|
|
2024-07-29 08:16:37 +00:00
|
|
|
|
private OverwatchCodexHeader CreateCodexHeader()
|
|
|
|
|
{
|
|
|
|
|
return new OverwatchCodexHeader
|
|
|
|
|
{
|
|
|
|
|
TotalNumberOfNodes = nameIdMap.Size
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
2024-07-26 08:56:22 +00:00
|
|
|
|
private bool IsCodexLog(IDownloadedLog log)
|
|
|
|
|
{
|
|
|
|
|
return log.GetLinesContaining("Run Codex node").Any();
|
|
|
|
|
}
|
2024-07-26 06:39:27 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public class CodexNodeTranscriptWriter : ICodexNodeHooks
|
|
|
|
|
{
|
|
|
|
|
private readonly ITranscriptWriter writer;
|
2024-07-26 07:14:46 +00:00
|
|
|
|
private readonly NameIdMap nameIdMap;
|
2024-07-26 06:39:27 +00:00
|
|
|
|
private readonly string name;
|
|
|
|
|
private string peerId = string.Empty;
|
2024-07-29 12:17:54 +00:00
|
|
|
|
private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>();
|
2024-07-26 06:39:27 +00:00
|
|
|
|
|
2024-07-26 07:14:46 +00:00
|
|
|
|
public CodexNodeTranscriptWriter(ITranscriptWriter writer, NameIdMap nameIdMap, string name)
|
2024-07-26 06:39:27 +00:00
|
|
|
|
{
|
|
|
|
|
this.writer = writer;
|
2024-07-26 07:14:46 +00:00
|
|
|
|
this.nameIdMap = nameIdMap;
|
2024-07-26 06:39:27 +00:00
|
|
|
|
this.name = name;
|
|
|
|
|
}
|
|
|
|
|
|
2024-07-29 08:16:37 +00:00
|
|
|
|
public void OnNodeStarting(DateTime startUtc, string image)
|
|
|
|
|
{
|
|
|
|
|
WriteCodexEvent(startUtc, e =>
|
|
|
|
|
{
|
|
|
|
|
e.NodeStarting = new NodeStartingEvent
|
|
|
|
|
{
|
|
|
|
|
Image = image
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void OnNodeStarted(string peerId)
|
2024-07-26 06:39:27 +00:00
|
|
|
|
{
|
|
|
|
|
this.peerId = peerId;
|
2024-07-26 07:14:46 +00:00
|
|
|
|
nameIdMap.Add(name, peerId);
|
2024-07-26 06:39:27 +00:00
|
|
|
|
WriteCodexEvent(e =>
|
|
|
|
|
{
|
|
|
|
|
e.NodeStarted = new NodeStartedEvent
|
|
|
|
|
{
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void OnNodeStopping()
|
|
|
|
|
{
|
|
|
|
|
WriteCodexEvent(e =>
|
|
|
|
|
{
|
2024-08-01 07:09:30 +00:00
|
|
|
|
e.NodeStopping = new NodeStoppingEvent
|
|
|
|
|
{
|
2024-07-26 06:39:27 +00:00
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2024-08-01 07:09:30 +00:00
|
|
|
|
public void OnFileDownloading(ContentId cid)
|
2024-07-26 06:39:27 +00:00
|
|
|
|
{
|
|
|
|
|
WriteCodexEvent(e =>
|
|
|
|
|
{
|
2024-08-01 07:09:30 +00:00
|
|
|
|
e.FileDownloading = new FileDownloadingEvent
|
2024-07-26 06:39:27 +00:00
|
|
|
|
{
|
|
|
|
|
Cid = cid.Id
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2024-08-01 07:09:30 +00:00
|
|
|
|
public void OnFileDownloaded(ByteSize size, ContentId cid)
|
|
|
|
|
{
|
|
|
|
|
WriteCodexEvent(e =>
|
|
|
|
|
{
|
|
|
|
|
e.FileDownloaded = new FileDownloadedEvent
|
|
|
|
|
{
|
|
|
|
|
Cid = cid.Id,
|
|
|
|
|
ByteSize = size.SizeInBytes
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void OnFileUploading(string uid, ByteSize size)
|
|
|
|
|
{
|
|
|
|
|
WriteCodexEvent(e =>
|
|
|
|
|
{
|
|
|
|
|
e.FileUploading = new FileUploadingEvent
|
|
|
|
|
{
|
|
|
|
|
UniqueId = uid,
|
|
|
|
|
ByteSize = size.SizeInBytes
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
|
2024-07-26 06:39:27 +00:00
|
|
|
|
{
|
|
|
|
|
WriteCodexEvent(e =>
|
|
|
|
|
{
|
|
|
|
|
e.FileUploaded = new FileUploadedEvent
|
|
|
|
|
{
|
2024-08-01 07:09:30 +00:00
|
|
|
|
UniqueId = uid,
|
|
|
|
|
Cid = cid.Id,
|
|
|
|
|
ByteSize = size.SizeInBytes
|
2024-07-26 06:39:27 +00:00
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void WriteCodexEvent(Action<OverwatchCodexEvent> action)
|
|
|
|
|
{
|
2024-07-29 08:16:37 +00:00
|
|
|
|
WriteCodexEvent(DateTime.UtcNow, action);
|
|
|
|
|
}
|
2024-07-26 06:39:27 +00:00
|
|
|
|
|
2024-07-29 08:16:37 +00:00
|
|
|
|
private void WriteCodexEvent(DateTime utc, Action<OverwatchCodexEvent> action)
|
|
|
|
|
{
|
2024-07-26 06:39:27 +00:00
|
|
|
|
var e = new OverwatchCodexEvent
|
|
|
|
|
{
|
2024-07-29 09:02:24 +00:00
|
|
|
|
Name = name,
|
2024-07-26 06:39:27 +00:00
|
|
|
|
PeerId = peerId
|
|
|
|
|
};
|
2024-07-29 12:17:54 +00:00
|
|
|
|
|
2024-07-26 06:39:27 +00:00
|
|
|
|
action(e);
|
|
|
|
|
|
2024-07-29 12:17:54 +00:00
|
|
|
|
if (string.IsNullOrEmpty(peerId))
|
|
|
|
|
{
|
|
|
|
|
// If we don't know our peerId, don't write the events yet.
|
|
|
|
|
AddToCache(utc, e);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
e.Write(utc, writer);
|
|
|
|
|
|
|
|
|
|
// Write any events that we cached when we didn't have our peerId yet.
|
|
|
|
|
WriteAndClearCache();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void AddToCache(DateTime utc, OverwatchCodexEvent e)
|
|
|
|
|
{
|
|
|
|
|
pendingEvents.Add((utc, e));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void WriteAndClearCache()
|
|
|
|
|
{
|
|
|
|
|
if (pendingEvents.Any())
|
|
|
|
|
{
|
|
|
|
|
foreach (var pair in pendingEvents)
|
|
|
|
|
{
|
|
|
|
|
pair.Item2.PeerId = peerId;
|
|
|
|
|
pair.Item2.Write(pair.Item1, writer);
|
|
|
|
|
}
|
|
|
|
|
pendingEvents.Clear();
|
|
|
|
|
}
|
2024-07-25 14:00:51 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|