diff --git a/Framework/OverwatchTranscript/BucketSet.cs b/Framework/OverwatchTranscript/BucketSet.cs index 544dc5b..0ae1586 100644 --- a/Framework/OverwatchTranscript/BucketSet.cs +++ b/Framework/OverwatchTranscript/BucketSet.cs @@ -1,20 +1,24 @@ -namespace OverwatchTranscript +using Logging; + +namespace OverwatchTranscript { public class BucketSet { private const int numberOfActiveBuckets = 5; private readonly object _counterLock = new object(); private int pendingAdds = 0; - + private readonly ILog log; + private readonly string workingDir; private readonly object _bucketLock = new object(); private readonly List fullBuckets = new List(); private readonly List activeBuckets = new List(); private int activeBucketIndex = 0; private bool closed = false; - private readonly string workingDir; - - public BucketSet(string workingDir) + private string internalErrors = string.Empty; + + public BucketSet(ILog log, string workingDir) { + this.log = log; this.workingDir = workingDir; for (var i = 0; i < numberOfActiveBuckets;i++) @@ -23,8 +27,6 @@ } } - public string Error { get; private set; } = string.Empty; - public void Add(DateTime utc, object payload) { if (closed) throw new Exception("Buckets already closed!"); @@ -32,20 +34,24 @@ Task.Run(() => AddInternal(utc, payload)); } - public bool IsEmpty() - { - return fullBuckets.All(b => b.Count == 0) && activeBuckets.All(b => b.Count == 0); - } - public IFinalizedBucket[] FinalizeBuckets() { closed = true; WaitForZeroPending(); + if (IsEmpty()) throw new Exception("No entries have been added."); + if (!string.IsNullOrEmpty(internalErrors)) throw new Exception(internalErrors); + var buckets = fullBuckets.Concat(activeBuckets).ToArray(); + log.Debug($"Finalizing {buckets.Length} buckets..."); return buckets.Select(b => b.FinalizeBucket()).ToArray(); } + private bool IsEmpty() + { + return fullBuckets.All(b => b.Count == 0) && activeBuckets.All(b => b.Count == 0); + } + private void AddInternal(DateTime utc, object payload) { try @@ -67,7 +73,7 @@ } catch (Exception ex) { - Error += ex.ToString(); + internalErrors += ex.ToString(); } } @@ -75,7 +81,7 @@ { lock (_bucketLock) { - activeBuckets.Add(new EventBucket(Path.Combine(workingDir, Guid.NewGuid().ToString()))); + activeBuckets.Add(new EventBucket(log, Path.Combine(workingDir, Guid.NewGuid().ToString()))); } } @@ -84,6 +90,7 @@ lock (_counterLock) { pendingAdds++; + log.Debug("(+) Pending: " + pendingAdds); } } @@ -92,16 +99,19 @@ lock (_counterLock) { pendingAdds--; - if (pendingAdds < 0) Error += "Pending less than zero"; + if (pendingAdds < 0) internalErrors += "Pending less than zero"; + log.Debug("(-) Pending: " + pendingAdds); } } private void WaitForZeroPending() { + log.Debug("Wait for zero pending."); while (true) { lock (_counterLock) { + log.Debug("(wait) Pending: " + pendingAdds); if (pendingAdds == 0) return; } Thread.Sleep(10); diff --git a/Framework/OverwatchTranscript/EventBucket.cs b/Framework/OverwatchTranscript/EventBucket.cs index a71fa4b..07f3378 100644 --- a/Framework/OverwatchTranscript/EventBucket.cs +++ b/Framework/OverwatchTranscript/EventBucket.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json; +using Logging; +using Newtonsoft.Json; namespace OverwatchTranscript { @@ -9,24 +10,22 @@ namespace OverwatchTranscript private readonly object _lock = new object(); private bool closed = false; - + private readonly ILog log; private readonly string bucketFile; private readonly List buffer = new List(); private EventBucketEntry? topEntry; - public EventBucket(string bucketFile) + public EventBucket(ILog log, string bucketFile) { + this.log = log; this.bucketFile = bucketFile; if (File.Exists(bucketFile)) throw new Exception("Already exists"); - EarliestUtc = DateTime.MaxValue; - LatestUtc = DateTime.MinValue; + log.Debug("Bucket open: " + bucketFile); } public int Count { get; private set; } public bool IsFull { get; private set; } - public DateTime EarliestUtc { get; private set; } - public DateTime LatestUtc { get; private set; } public void Add(DateTime utc, object payload) { @@ -34,7 +33,7 @@ namespace OverwatchTranscript { if (closed) throw new Exception("Already closed"); AddToBuffer(utc, payload); - BufferToFile(); + BufferToFile(emptyBuffer: false); } } @@ -43,9 +42,10 @@ namespace OverwatchTranscript lock (_lock) { closed = true; - BufferToFile(); + BufferToFile(emptyBuffer: true); SortFileByTimestamps(); } + log.Debug($"Finalized bucket with {Count} entries"); return this; } @@ -71,6 +71,11 @@ namespace OverwatchTranscript } } + public override string ToString() + { + return $"EventBucket: " + Count; + } + private void AddToBuffer(DateTime utc, object payload) { var typeName = payload.GetType().FullName; @@ -87,17 +92,15 @@ namespace OverwatchTranscript } }; - if (utc < EarliestUtc) EarliestUtc = utc; - if (utc > LatestUtc) LatestUtc = utc; Count++; IsFull = Count > MaxCount; buffer.Add(entry); } - private void BufferToFile() + private void BufferToFile(bool emptyBuffer) { - if (buffer.Count > MaxBuffer) + if (emptyBuffer || buffer.Count > MaxBuffer) { using var file = File.Open(bucketFile, FileMode.Append); using var writer = new StreamWriter(file); @@ -105,6 +108,7 @@ namespace OverwatchTranscript { writer.WriteLine(JsonConvert.SerializeObject(entry)); } + log.Debug($"Bucket wrote {buffer.Count} entries to file."); buffer.Clear(); } } @@ -120,7 +124,7 @@ namespace OverwatchTranscript File.Delete(bucketFile); File.WriteAllLines(bucketFile, entries.Select(JsonConvert.SerializeObject)); - topEntry = entries.First(); + topEntry = entries.FirstOrDefault(); } } @@ -128,8 +132,6 @@ namespace OverwatchTranscript { int Count { get; } bool IsFull { get; } - DateTime EarliestUtc { get; } - DateTime LatestUtc { get; } EventBucketEntry? ViewTopEntry(); void PopTopEntry(); } diff --git a/Framework/OverwatchTranscript/MomentReferenceBuilder.cs b/Framework/OverwatchTranscript/MomentReferenceBuilder.cs index d94a7da..e0596dd 100644 --- a/Framework/OverwatchTranscript/MomentReferenceBuilder.cs +++ b/Framework/OverwatchTranscript/MomentReferenceBuilder.cs @@ -15,7 +15,6 @@ namespace OverwatchTranscript public OverwatchMomentReference[] Build(IFinalizedBucket[] buckets) { var result = new List(); - var currentBuilder = new Builder(workingDir); while (EntriesRemaining(buckets)) @@ -31,6 +30,11 @@ namespace OverwatchTranscript } } + if (currentBuilder.NumberOfMoments > 0) + { + result.Add(currentBuilder.Build()); + } + return result.ToArray(); } diff --git a/Framework/OverwatchTranscript/OverwatchTranscript.csproj b/Framework/OverwatchTranscript/OverwatchTranscript.csproj index 51eb283..70e6563 100644 --- a/Framework/OverwatchTranscript/OverwatchTranscript.csproj +++ b/Framework/OverwatchTranscript/OverwatchTranscript.csproj @@ -10,4 +10,8 @@ + + + + diff --git a/Framework/OverwatchTranscript/Transcript.cs b/Framework/OverwatchTranscript/Transcript.cs index f540872..5b2a101 100644 --- a/Framework/OverwatchTranscript/Transcript.cs +++ b/Framework/OverwatchTranscript/Transcript.cs @@ -1,10 +1,13 @@ -namespace OverwatchTranscript +using Logging; + +namespace OverwatchTranscript { public static class Transcript { - public static ITranscriptWriter NewWriter() + public static ITranscriptWriter NewWriter(ILog log) { - return new TranscriptWriter(NewWorkDir()); + log = new LogPrefixer(log, "(TranscriptWriter) "); + return new TranscriptWriter(log, NewWorkDir()); } public static ITranscriptReader NewReader(string transcriptFile) diff --git a/Framework/OverwatchTranscript/TranscriptReader.cs b/Framework/OverwatchTranscript/TranscriptReader.cs index 70fa00e..998325b 100644 --- a/Framework/OverwatchTranscript/TranscriptReader.cs +++ b/Framework/OverwatchTranscript/TranscriptReader.cs @@ -31,6 +31,7 @@ namespace OverwatchTranscript private long momentCounter; private readonly object queueLock = new object(); private readonly List queue = new List(); + private readonly Task queueFiller; public TranscriptReader(string workingDir, string inputFilename) { @@ -44,6 +45,8 @@ namespace OverwatchTranscript model = LoadModel(inputFilename); reader = new MomentReader(model, workingDir); + + queueFiller = Task.Run(FillQueue); } public OverwatchCommonHeader Header @@ -98,21 +101,28 @@ namespace OverwatchTranscript { CheckClosed(); OverwatchMoment moment = null!; + OverwatchMoment? next = null; lock (queueLock) { if (queue.Count == 0) return; moment = queue[0]; + if (queue.Count > 1) next = queue[1]; + queue.RemoveAt(0); } - ActivateMoment(moment); + var duration = GetMomentDuration(moment, next); + ActivateMoment(moment, duration); } public void Close() { CheckClosed(); - Directory.Delete(workingDir, true); closed = true; + + queueFiller.Wait(); + + Directory.Delete(workingDir, true); } private Action CreateEventAction(Action> handler) @@ -123,17 +133,37 @@ namespace OverwatchTranscript }; } - private TimeSpan? GetMomentDuration() + private void FillQueue() { - if (current == null) return null; - if (next == null) return null; + while (true) + { + if (closed) return; - return next.Utc - current.Utc; + lock (queueLock) + { + while (queue.Count < 100) + { + var moment = reader.Next(); + if (moment == null) return; + queue.Add(moment); + } + } + + Thread.Sleep(1); + } } - private void ActivateMoment(OverwatchMoment moment, TimeSpan? duration, long momentIndex) + private TimeSpan? GetMomentDuration(OverwatchMoment moment, OverwatchMoment? next) { - var m = new ActivateMoment(moment.Utc, duration, momentIndex); + if (moment == null) return null; + if (next == null) return null; + + return next.Utc - moment.Utc; + } + + private void ActivateMoment(OverwatchMoment moment, TimeSpan? duration) + { + var m = new ActivateMoment(moment.Utc, duration, momentCounter); lock (handlersLock) { @@ -144,6 +174,8 @@ namespace OverwatchTranscript ActivateEventHandlers(m, @event); } } + + momentCounter++; } private void ActivateMomentHandlers(ActivateMoment m) diff --git a/Framework/OverwatchTranscript/TranscriptWriter.cs b/Framework/OverwatchTranscript/TranscriptWriter.cs index f5defa3..dd79909 100644 --- a/Framework/OverwatchTranscript/TranscriptWriter.cs +++ b/Framework/OverwatchTranscript/TranscriptWriter.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json; +using Logging; +using Newtonsoft.Json; using System.IO.Compression; namespace OverwatchTranscript @@ -19,14 +20,16 @@ namespace OverwatchTranscript private readonly string artifactsFolder; private readonly Dictionary header = new Dictionary(); private readonly BucketSet bucketSet; + private readonly ILog log; private readonly string workingDir; private bool closed; - public TranscriptWriter(string workingDir) + public TranscriptWriter(ILog log, string workingDir) { closed = false; + this.log = log; this.workingDir = workingDir; - bucketSet = new BucketSet(workingDir); + bucketSet = new BucketSet(log, workingDir); builder = new MomentReferenceBuilder(workingDir); transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename); artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName); @@ -61,12 +64,6 @@ namespace OverwatchTranscript public void Write(string outputFilename) { - if (bucketSet.IsEmpty()) throw new Exception("No entries added."); - if (!string.IsNullOrEmpty(bucketSet.Error)) - { - throw new Exception("Exceptions in BucketSet: " + bucketSet.Error); - } - CheckClosed(); closed = true; @@ -76,8 +73,10 @@ namespace OverwatchTranscript File.WriteAllText(transcriptFile, JsonConvert.SerializeObject(model, Formatting.Indented)); ZipFile.CreateFromDirectory(workingDir, outputFilename); + log.Debug($"Transcript written to {outputFilename}"); Directory.Delete(workingDir, true); + log.Debug($"Workdir {workingDir} deleted"); } private OverwatchTranscript CreateModel(OverwatchMomentReference[] momentReferences) diff --git a/Tests/CodexTests/CodexDistTest.cs b/Tests/CodexTests/CodexDistTest.cs index 8c25c29..876e035 100644 --- a/Tests/CodexTests/CodexDistTest.cs +++ b/Tests/CodexTests/CodexDistTest.cs @@ -139,7 +139,7 @@ namespace CodexTests if (GetTranscriptAttributeOfCurrentTest() == null) return; var log = new LogPrefixer(lifecycle.Log, "(Transcript) "); - var writer = new CodexTranscriptWriter(log, Transcript.NewWriter()); + var writer = new CodexTranscriptWriter(log, Transcript.NewWriter(log)); Ci.SetCodexHooksProvider(writer); writers.Add(lifecycle, writer); } diff --git a/Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs b/Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs index b0e7912..a66b248 100644 --- a/Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs +++ b/Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json; +using Logging; +using Newtonsoft.Json; using NUnit.Framework; using OverwatchTranscript; @@ -31,7 +32,8 @@ namespace FrameworkTests.OverwatchTranscript private void WriteTranscript(string workdir) { - var writer = new TranscriptWriter(workdir); + var log = new ConsoleLog(); + var writer = new TranscriptWriter(log, workdir); writer.AddHeader(HeaderKey, new TestHeader {