status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-01-23 23:08:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

test passes

Browse Source
This commit is contained in:
benbierens 2024-08-02 08:56:49 +02:00
parent 7a3a2b558b
commit f6cd9db408
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
9 changed files with 111 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
Framework/OverwatchTranscript/BucketSet.cs
View File

@ -1,20 +1,24 @@
namespace OverwatchTranscript
using Logging;
namespace OverwatchTranscript
{
public class BucketSet
{
private const int numberOfActiveBuckets = 5;
private readonly object _counterLock = new object();
private int pendingAdds = 0;
private readonly ILog log;
private readonly string workingDir;
private readonly object _bucketLock = new object();
private readonly List<EventBucket> fullBuckets = new List<EventBucket>();
private readonly List<EventBucket> activeBuckets = new List<EventBucket>();
private int activeBucketIndex = 0;
private bool closed = false;
private readonly string workingDir;
public BucketSet(string workingDir)
private string internalErrors = string.Empty;
public BucketSet(ILog log, string workingDir)
{
this.log = log;
this.workingDir = workingDir;
for (var i = 0; i < numberOfActiveBuckets;i++)
@ -23,8 +27,6 @@
}
}
public string Error { get; private set; } = string.Empty;
public void Add(DateTime utc, object payload)
{
if (closed) throw new Exception("Buckets already closed!");
@ -32,20 +34,24 @@
Task.Run(() => AddInternal(utc, payload));
}
public bool IsEmpty()
{
return fullBuckets.All(b => b.Count == 0) && activeBuckets.All(b => b.Count == 0);
}
public IFinalizedBucket[] FinalizeBuckets()
{
closed = true;
WaitForZeroPending();
if (IsEmpty()) throw new Exception("No entries have been added.");
if (!string.IsNullOrEmpty(internalErrors)) throw new Exception(internalErrors);
var buckets = fullBuckets.Concat(activeBuckets).ToArray();
log.Debug($"Finalizing {buckets.Length} buckets...");
return buckets.Select(b => b.FinalizeBucket()).ToArray();
}
private bool IsEmpty()
{
return fullBuckets.All(b => b.Count == 0) && activeBuckets.All(b => b.Count == 0);
}
private void AddInternal(DateTime utc, object payload)
{
try
@ -67,7 +73,7 @@
}
catch (Exception ex)
{
Error += ex.ToString();
internalErrors += ex.ToString();
}
}
@ -75,7 +81,7 @@
{
lock (_bucketLock)
{
activeBuckets.Add(new EventBucket(Path.Combine(workingDir, Guid.NewGuid().ToString())));
activeBuckets.Add(new EventBucket(log, Path.Combine(workingDir, Guid.NewGuid().ToString())));
}
}
@ -84,6 +90,7 @@
lock (_counterLock)
{
pendingAdds++;
log.Debug("(+) Pending: " + pendingAdds);
}
}
@ -92,16 +99,19 @@
lock (_counterLock)
{
pendingAdds--;
if (pendingAdds < 0) Error += "Pending less than zero";
if (pendingAdds < 0) internalErrors += "Pending less than zero";
log.Debug("(-) Pending: " + pendingAdds);
}
}
private void WaitForZeroPending()
{
log.Debug("Wait for zero pending.");
while (true)
{
lock (_counterLock)
{
log.Debug("(wait) Pending: " + pendingAdds);
if (pendingAdds == 0) return;
}
Thread.Sleep(10);

34
Framework/OverwatchTranscript/EventBucket.cs
View File

@ -1,4 +1,5 @@
using Newtonsoft.Json;
using Logging;
using Newtonsoft.Json;
namespace OverwatchTranscript
{
@ -9,24 +10,22 @@ namespace OverwatchTranscript
private readonly object _lock = new object();
private bool closed = false;
private readonly ILog log;
private readonly string bucketFile;
private readonly List<EventBucketEntry> buffer = new List<EventBucketEntry>();
private EventBucketEntry? topEntry;
public EventBucket(string bucketFile)
public EventBucket(ILog log, string bucketFile)
{
this.log = log;
this.bucketFile = bucketFile;
if (File.Exists(bucketFile)) throw new Exception("Already exists");
EarliestUtc = DateTime.MaxValue;
LatestUtc = DateTime.MinValue;
log.Debug("Bucket open: " + bucketFile);
}
public int Count { get; private set; }
public bool IsFull { get; private set; }
public DateTime EarliestUtc { get; private set; }
public DateTime LatestUtc { get; private set; }
public void Add(DateTime utc, object payload)
{
@ -34,7 +33,7 @@ namespace OverwatchTranscript
{
if (closed) throw new Exception("Already closed");
AddToBuffer(utc, payload);
BufferToFile();
BufferToFile(emptyBuffer: false);
}
}
@ -43,9 +42,10 @@ namespace OverwatchTranscript
lock (_lock)
{
closed = true;
BufferToFile();
BufferToFile(emptyBuffer: true);
SortFileByTimestamps();
}
log.Debug($"Finalized bucket with {Count} entries");
return this;
}
@ -71,6 +71,11 @@ namespace OverwatchTranscript
}
}
public override string ToString()
{
return $"EventBucket: " + Count;
}
private void AddToBuffer(DateTime utc, object payload)
{
var typeName = payload.GetType().FullName;
@ -87,17 +92,15 @@ namespace OverwatchTranscript
}
};
if (utc < EarliestUtc) EarliestUtc = utc;
if (utc > LatestUtc) LatestUtc = utc;
Count++;
IsFull = Count > MaxCount;
buffer.Add(entry);
}
private void BufferToFile()
private void BufferToFile(bool emptyBuffer)
{
if (buffer.Count > MaxBuffer)
if (emptyBuffer || buffer.Count > MaxBuffer)
{
using var file = File.Open(bucketFile, FileMode.Append);
using var writer = new StreamWriter(file);
@ -105,6 +108,7 @@ namespace OverwatchTranscript
{
writer.WriteLine(JsonConvert.SerializeObject(entry));
}
log.Debug($"Bucket wrote {buffer.Count} entries to file.");
buffer.Clear();
}
}
@ -120,7 +124,7 @@ namespace OverwatchTranscript
File.Delete(bucketFile);
File.WriteAllLines(bucketFile, entries.Select(JsonConvert.SerializeObject));
topEntry = entries.First();
topEntry = entries.FirstOrDefault();
}
}
@ -128,8 +132,6 @@ namespace OverwatchTranscript
{
int Count { get; }
bool IsFull { get; }
DateTime EarliestUtc { get; }
DateTime LatestUtc { get; }
EventBucketEntry? ViewTopEntry();
void PopTopEntry();
}

6
Framework/OverwatchTranscript/MomentReferenceBuilder.cs
View File

@ -15,7 +15,6 @@ namespace OverwatchTranscript
public OverwatchMomentReference[] Build(IFinalizedBucket[] buckets)
{
var result = new List<OverwatchMomentReference>();
var currentBuilder = new Builder(workingDir);
while (EntriesRemaining(buckets))
@ -31,6 +30,11 @@ namespace OverwatchTranscript
}
}
if (currentBuilder.NumberOfMoments > 0)
{
result.Add(currentBuilder.Build());
}
return result.ToArray();
}

4
Framework/OverwatchTranscript/OverwatchTranscript.csproj
View File

@ -10,4 +10,8 @@
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Logging\Logging.csproj" />
</ItemGroup>
</Project>

9
Framework/OverwatchTranscript/Transcript.cs
View File

@ -1,10 +1,13 @@
namespace OverwatchTranscript
using Logging;
namespace OverwatchTranscript
{
public static class Transcript
{
public static ITranscriptWriter NewWriter()
public static ITranscriptWriter NewWriter(ILog log)
{
return new TranscriptWriter(NewWorkDir());
log = new LogPrefixer(log, "(TranscriptWriter) ");
return new TranscriptWriter(log, NewWorkDir());
}
public static ITranscriptReader NewReader(string transcriptFile)

48
Framework/OverwatchTranscript/TranscriptReader.cs
View File

@ -31,6 +31,7 @@ namespace OverwatchTranscript
private long momentCounter;
private readonly object queueLock = new object();
private readonly List<OverwatchMoment> queue = new List<OverwatchMoment>();
private readonly Task queueFiller;
public TranscriptReader(string workingDir, string inputFilename)
{
@ -44,6 +45,8 @@ namespace OverwatchTranscript
model = LoadModel(inputFilename);
reader = new MomentReader(model, workingDir);
queueFiller = Task.Run(FillQueue);
}
public OverwatchCommonHeader Header
@ -98,21 +101,28 @@ namespace OverwatchTranscript
{
CheckClosed();
OverwatchMoment moment = null!;
OverwatchMoment? next = null;
lock (queueLock)
{
if (queue.Count == 0) return;
moment = queue[0];
if (queue.Count > 1) next = queue[1];
queue.RemoveAt(0);
}
ActivateMoment(moment);
var duration = GetMomentDuration(moment, next);
ActivateMoment(moment, duration);
}
public void Close()
{
CheckClosed();
Directory.Delete(workingDir, true);
closed = true;
queueFiller.Wait();
Directory.Delete(workingDir, true);
}
private Action<ActivateMoment, string> CreateEventAction<T>(Action<ActivateEvent<T>> handler)
@ -123,17 +133,37 @@ namespace OverwatchTranscript
};
}
private TimeSpan? GetMomentDuration()
private void FillQueue()
{
if (current == null) return null;
if (next == null) return null;
while (true)
{
if (closed) return;
return next.Utc - current.Utc;
lock (queueLock)
{
while (queue.Count < 100)
{
var moment = reader.Next();
if (moment == null) return;
queue.Add(moment);
}
}
Thread.Sleep(1);
}
}
private void ActivateMoment(OverwatchMoment moment, TimeSpan? duration, long momentIndex)
private TimeSpan? GetMomentDuration(OverwatchMoment moment, OverwatchMoment? next)
{
var m = new ActivateMoment(moment.Utc, duration, momentIndex);
if (moment == null) return null;
if (next == null) return null;
return next.Utc - moment.Utc;
}
private void ActivateMoment(OverwatchMoment moment, TimeSpan? duration)
{
var m = new ActivateMoment(moment.Utc, duration, momentCounter);
lock (handlersLock)
{
@ -144,6 +174,8 @@ namespace OverwatchTranscript
ActivateEventHandlers(m, @event);
}
}
momentCounter++;
}
private void ActivateMomentHandlers(ActivateMoment m)

17
Framework/OverwatchTranscript/TranscriptWriter.cs
View File

@ -1,4 +1,5 @@
using Newtonsoft.Json;
using Logging;
using Newtonsoft.Json;
using System.IO.Compression;
namespace OverwatchTranscript
@ -19,14 +20,16 @@ namespace OverwatchTranscript
private readonly string artifactsFolder;
private readonly Dictionary<string, string> header = new Dictionary<string, string>();
private readonly BucketSet bucketSet;
private readonly ILog log;
private readonly string workingDir;
private bool closed;
public TranscriptWriter(string workingDir)
public TranscriptWriter(ILog log, string workingDir)
{
closed = false;
this.log = log;
this.workingDir = workingDir;
bucketSet = new BucketSet(workingDir);
bucketSet = new BucketSet(log, workingDir);
builder = new MomentReferenceBuilder(workingDir);
transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
@ -61,12 +64,6 @@ namespace OverwatchTranscript
public void Write(string outputFilename)
{
if (bucketSet.IsEmpty()) throw new Exception("No entries added.");
if (!string.IsNullOrEmpty(bucketSet.Error))
{
throw new Exception("Exceptions in BucketSet: " + bucketSet.Error);
}
CheckClosed();
closed = true;
@ -76,8 +73,10 @@ namespace OverwatchTranscript
File.WriteAllText(transcriptFile, JsonConvert.SerializeObject(model, Formatting.Indented));
ZipFile.CreateFromDirectory(workingDir, outputFilename);
log.Debug($"Transcript written to {outputFilename}");
Directory.Delete(workingDir, true);
log.Debug($"Workdir {workingDir} deleted");
}
private OverwatchTranscript CreateModel(OverwatchMomentReference[] momentReferences)

2
Tests/CodexTests/CodexDistTest.cs
View File

@ -139,7 +139,7 @@ namespace CodexTests
if (GetTranscriptAttributeOfCurrentTest() == null) return;
var log = new LogPrefixer(lifecycle.Log, "(Transcript) ");
var writer = new CodexTranscriptWriter(log, Transcript.NewWriter());
var writer = new CodexTranscriptWriter(log, Transcript.NewWriter(log));
Ci.SetCodexHooksProvider(writer);
writers.Add(lifecycle, writer);
}

6
Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs
View File

@ -1,4 +1,5 @@
using Newtonsoft.Json;
using Logging;
using Newtonsoft.Json;
using NUnit.Framework;
using OverwatchTranscript;
@ -31,7 +32,8 @@ namespace FrameworkTests.OverwatchTranscript
private void WriteTranscript(string workdir)
{
var writer = new TranscriptWriter(workdir);
var log = new ConsoleLog();
var writer = new TranscriptWriter(log, workdir);
writer.AddHeader(HeaderKey, new TestHeader
{