Implements marketinsights api

Framework/Utils/RollingAverage.cs

@@ -15,7 +15,14 @@
             var originalValue = currentAverage;
             var originalValueWeight = ((n - 1.0f) / n);
             var newValueWeight = (1.0f / n);
-            return (originalValue * originalValueWeight) + (newValue * newValueWeight);
+            return GetWeightedAverage(originalValue, originalValueWeight, newValue, newValueWeight);
+        }
+
+        public static float GetWeightedAverage(float value1, float weight1, float value2, float weight2)
+        {
+            float totalWeight = weight1 + weight2;
+            if (totalWeight == 0.0f) return 0.0f;
+            return ((value1 * weight1) + (value2 * weight2)) / totalWeight;
         }
     }
 }

Tools/MarketInsights/AppState.cs

@@ -1,4 +1,6 @@
-namespace MarketInsights
+using Logging;
+
+namespace MarketInsights
 {
     public class AppState
     {
@@ -7,7 +9,9 @@
             Config = config;
         }
 
-        public MarketOverview MarketOverview { get; set; } = new ();
+        public bool Realtime { get; set; }
+        public MarketOverview MarketOverview { get; set; } = new();
         public Configuration Config { get; }
+        public ILog Log { get; } = new ConsoleLog();
     }
 }

Tools/MarketInsights/AverageHistory.cs

@@ -1,71 +1,50 @@
-using CodexContractsPlugin.ChainMonitor;
+using CodexContractsPlugin;
-using GethPlugin;
+using CodexContractsPlugin.ChainMonitor;
-using System.Numerics;
+using Nethereum.Model;
+using TestNetRewarder;
 using Utils;
 
 namespace MarketInsights
 {
-    public class AverageHistory
+    public class AverageHistory : ITimeSegmentHandler
     {
-        public readonly List<MarketTimeSegment> contributions = new List<MarketTimeSegment>();
+        private readonly List<MarketTimeSegment> contributions = new List<MarketTimeSegment>();
+        private readonly ChainStateChangeHandlerMux mux = new ChainStateChangeHandlerMux();
+        private readonly AppState appState;
+        private readonly int maxContributions;
+        private readonly ChainState chainState;
 
-    }
+        public AverageHistory(AppState appState, ICodexContracts contracts, int maxContributions)
-
-    public class ContributionBuilder : IChainStateChangeHandler
-    {
-        private readonly MarketTimeSegment segment = new MarketTimeSegment();
-
-        public void OnNewRequest(RequestEvent requestEvent)
         {
-            AddRequestToAverage(segment.Submitted, requestEvent);
+            this.appState = appState;
+            this.maxContributions = maxContributions;
+            chainState = new ChainState(appState.Log, contracts, mux, appState.Config.HistoryStartUtc);
         }
 
-        public void OnRequestCancelled(RequestEvent requestEvent)
+        public MarketTimeSegment[] Segments { get; private set; } = Array.Empty<MarketTimeSegment>();
+
+        public Task OnNewSegment(TimeRange timeRange)
         {
-            AddRequestToAverage(segment.Expired, requestEvent);
+            var contribution = BuildContribution(timeRange);
+            contributions.Add(contribution);
+
+            while (contributions.Count > maxContributions)
+            {
+                contributions.RemoveAt(0);
+            }
+
+            Segments = contributions.ToArray();
+
+            return Task.CompletedTask;
         }
 
-        public void OnRequestFailed(RequestEvent requestEvent)
+        private MarketTimeSegment BuildContribution(TimeRange timeRange)
         {
-            AddRequestToAverage(segment.Failed, requestEvent);
+            var builder = new ContributionBuilder(timeRange);
-        }
+            mux.Handlers.Add(builder);
-
+            chainState.Update(timeRange.To);
-        public void OnRequestFinished(RequestEvent requestEvent)
+            mux.Handlers.Remove(builder);
-        {
+            return builder.GetSegment();
-            AddRequestToAverage(segment.Finished, requestEvent);
-        }
-
-        public void OnRequestFulfilled(RequestEvent requestEvent)
-        {
-            AddRequestToAverage(segment.Started, requestEvent);
-        }
-
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
-        {
-        }
-
-        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-        }
-
-        private void AddRequestToAverage(ContractAverages average, RequestEvent requestEvent)
-        {
-            average.Number++;
-            average.Price = GetNewAverage(average.Price, average.Number, requestEvent.Request.Request.Ask.Reward);
-            average.Size = GetNewAverage(average.Size, average.Number, requestEvent.Request.Request.Ask.SlotSize);
-            average.Duration = GetNewAverage(average.Duration, average.Number, requestEvent.Request.Request.Ask.Duration);
-            average.Collateral = GetNewAverage(average.Collateral, average.Number, requestEvent.Request.Request.Ask.Collateral);
-            average.ProofProbability = GetNewAverage(average.ProofProbability, average.Number, requestEvent.Request.Request.Ask.ProofProbability);
-        }
-
-        private float GetNewAverage(float currentAverage, int newNumberOfValues, BigInteger newValue)
-        {
-            return GetNewAverage(currentAverage, newNumberOfValues, (float)newValue);
-        }
-
-        private float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue)
-        {
-            return RollingAverage.GetNewAverage(currentAverage, newNumberOfValues, newValue);
         }
     }
 }

Tools/MarketInsights/Configuration.cs

@@ -33,5 +33,13 @@ namespace MarketInsights
                 return DateTimeOffset.FromUnixTimeSeconds(CheckHistoryTimestamp).UtcDateTime;
             }
         }
+
+        public TimeSpan UpdateInterval
+        {
+            get
+            {
+                return TimeSpan.FromMinutes(UpdateIntervalMinutes);
+            }
+        }
     }
 }

Tools/MarketInsights/ContributionBuilder.cs

@@ -0,0 +1,79 @@
+using CodexContractsPlugin.ChainMonitor;
+using GethPlugin;
+using System.Numerics;
+using Utils;
+
+namespace MarketInsights
+{
+    public class ContributionBuilder : IChainStateChangeHandler
+    {
+        private readonly MarketTimeSegment segment = new MarketTimeSegment();
+
+        public ContributionBuilder(TimeRange timeRange)
+        {
+            segment = new MarketTimeSegment
+            {
+                FromUtc = timeRange.From,
+                ToUtc = timeRange.To
+            };
+        }
+
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+            AddRequestToAverage(segment.Submitted, requestEvent);
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+            AddRequestToAverage(segment.Expired, requestEvent);
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+            AddRequestToAverage(segment.Failed, requestEvent);
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+            AddRequestToAverage(segment.Finished, requestEvent);
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+            AddRequestToAverage(segment.Started, requestEvent);
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public MarketTimeSegment GetSegment()
+        {
+            return segment;
+        }
+
+        private void AddRequestToAverage(ContractAverages average, RequestEvent requestEvent)
+        {
+            average.Number++;
+            average.Price = GetNewAverage(average.Price, average.Number, requestEvent.Request.Request.Ask.Reward);
+            average.Size = GetNewAverage(average.Size, average.Number, requestEvent.Request.Request.Ask.SlotSize);
+            average.Duration = GetNewAverage(average.Duration, average.Number, requestEvent.Request.Request.Ask.Duration);
+            average.Collateral = GetNewAverage(average.Collateral, average.Number, requestEvent.Request.Request.Ask.Collateral);
+            average.ProofProbability = GetNewAverage(average.ProofProbability, average.Number, requestEvent.Request.Request.Ask.ProofProbability);
+        }
+
+        private float GetNewAverage(float currentAverage, int newNumberOfValues, BigInteger newValue)
+        {
+            return GetNewAverage(currentAverage, newNumberOfValues, (float)newValue);
+        }
+
+        private float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue)
+        {
+            return RollingAverage.GetNewAverage(currentAverage, newNumberOfValues, newValue);
+        }
+    }
+}

Tools/MarketInsights/MarketInsights.csproj

@@ -19,6 +19,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\TestNetRewarder\TestNetRewarder.csproj" />
   </ItemGroup>
 
 </Project>

Tools/MarketInsights/Model.cs

@@ -7,6 +7,11 @@
         /// </summary>
         public DateTime LastUpdatedUtc { get; set; }
 
+        /// <summary>
+        /// When false, service is busy processing history in order to catch up to the present.
+        /// </summary>
+        public bool IsUpToDate { get; set; }
+
         public MarketTimeSegment[] TimeSegments { get; set; } = Array.Empty<MarketTimeSegment>();
     }
 

Tools/MarketInsights/Program.cs

@@ -1,5 +1,6 @@
 using ArgsUniform;
 using Microsoft.Extensions.Options;
+using Nethereum.Model;
 using System.Reflection;
 
 namespace MarketInsights
@@ -10,9 +11,20 @@ namespace MarketInsights
         {
             var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
             var config = uniformArgs.Parse(true);
-
+            var cts = new CancellationTokenSource();
             var appState = new AppState(config);
-            var updater = new Updater(appState);
+
+            Console.CancelKeyPress += (s, e) =>
+            {
+                appState.Log.Log("Stopping...");
+                cts.Cancel();
+                e.Cancel = true;
+            };
+
+            var connector = GethConnector.GethConnector.Initialize(appState.Log);
+            if (connector == null) throw new Exception("Invalid Geth information");
+
+            var updater = new Updater(appState, connector.CodexContracts, cts.Token);
 
             var builder = WebApplication.CreateBuilder(args);
 
@@ -49,6 +61,14 @@ namespace MarketInsights
         private static void PrintHelp()
         {
             Console.WriteLine("WebAPI for generating market overview for Codex network. Comes with OpenAPI swagger endpoint.");
+
+            var nl = Environment.NewLine;
+            Console.WriteLine($"Required environment variables: {nl}" +
+                $"'GETH_HOST'{nl}",
+                $"'GETH_HTTP_PORT'{nl}",
+                $"'CODEXCONTRACTS_MARKETPLACEADDRESS'{nl}",
+                $"'CODEXCONTRACTS_TOKENADDRESS'{nl}",
+                $"'CODEXCONTRACTS_ABI'{nl}");
         }
     }
 }

Tools/MarketInsights/Tracker.cs

@@ -0,0 +1,69 @@
+using CodexContractsPlugin.ChainMonitor;
+using Utils;
+using YamlDotNet.Core;
+
+namespace MarketInsights
+{
+    public class Tracker
+    {
+        private readonly AverageHistory history;
+
+        public Tracker(int numberOfSegments, AverageHistory history)
+        {
+            NumberOfSegments = numberOfSegments;
+            this.history = history;
+        }
+
+        public int NumberOfSegments { get; }
+
+        public MarketTimeSegment? CreateMarketTimeSegment()
+        {
+            if (history.Segments.Length < NumberOfSegments) return null;
+
+            var mySegments = history.Segments.TakeLast(NumberOfSegments);
+            return AverageSegments(mySegments);
+        }
+
+        private MarketTimeSegment AverageSegments(IEnumerable<MarketTimeSegment> mySegments)
+        {
+            var result = new MarketTimeSegment();
+
+            foreach (var segment in mySegments)
+            {
+                result.FromUtc = Min(result.FromUtc, segment.FromUtc);
+                result.ToUtc = Max(result.ToUtc, segment.ToUtc);
+
+                Combine(result.Submitted, segment.Submitted);
+                Combine(result.Expired, segment.Expired);
+                Combine(result.Started, segment.Started);
+                Combine(result.Finished, segment.Finished);
+                Combine(result.Failed, segment.Failed);
+            }
+            return result;
+        }
+
+        private void Combine(ContractAverages result, ContractAverages toAdd)
+        {
+            float weight1 = result.Number;
+            float weight2 = toAdd.Number;
+
+            result.Price = RollingAverage.GetWeightedAverage(result.Price, weight1, toAdd.Price, weight2);
+            result.Size = RollingAverage.GetWeightedAverage(result.Size, weight1, toAdd.Size, weight2);
+            result.Duration = RollingAverage.GetWeightedAverage(result.Duration, weight1, toAdd.Duration, weight2);
+            result.Collateral = RollingAverage.GetWeightedAverage(result.Collateral, weight1, toAdd.Collateral, weight2);
+            result.ProofProbability = RollingAverage.GetWeightedAverage(result.ProofProbability, weight1, toAdd.ProofProbability, weight2);
+        }
+
+        private DateTime Max(DateTime a, DateTime b)
+        {
+            if (a > b) return a;
+            return b;
+        }
+
+        private DateTime Min(DateTime a, DateTime b)
+        {
+            if (a > b) return b;
+            return a;
+        }
+    }
+}

Tools/MarketInsights/Updater.cs

@@ -1,35 +1,67 @@
-
+using CodexContractsPlugin;
+using TestNetRewarder;
+
 namespace MarketInsights
 {
     public class Updater
     {
+        private readonly Random random = new Random();
         private readonly AppState appState;
+        private readonly CancellationToken ct;
         private readonly Tracker[] trackers;
+        private readonly AverageHistory averageHistory;
 
-        public Updater(AppState appState)
+        public Updater(AppState appState, ICodexContracts contracts, CancellationToken ct)
         {
             this.appState = appState;
+            this.ct = ct;
+
             trackers = CreateTrackers();
+            averageHistory = new AverageHistory(appState, contracts, trackers.Max(t => t.NumberOfSegments));
         }
 
         private Tracker[] CreateTrackers()
         {
             var tokens = appState.Config.TimeSegments.Split(";", StringSplitOptions.RemoveEmptyEntries);
             var nums = tokens.Select(t => Convert.ToInt32(t)).ToArray();
-            return nums.Select(n => new Tracker(n)).ToArray();
+            return nums.Select(n => new Tracker(n, averageHistory)).ToArray();
         }
 
         public void Run()
         {
-
+            Task.Run(Runner);
         }
-    }
 
-    public class Tracker
+        private async Task Runner()
-    {
-        public Tracker(int numberOfSegments)
         {
-            
+            var segmenter = new TimeSegmenter(
+                appState.Log,
+                segmentSize: appState.Config.UpdateInterval,
+                historyStartUtc: appState.Config.HistoryStartUtc,
+                handler: averageHistory
+            );
+
+            while (!ct.IsCancellationRequested)
+            {
+                await segmenter.ProcessNextSegment();
+                await Task.Delay(TimeSpan.FromSeconds(3), ct);
+
+                var marketTimeSegments = trackers
+                    .Select(t => t.CreateMarketTimeSegment())
+                    .Where(t => t != null)
+                    .Cast<MarketTimeSegment>()  
+                    .ToArray();
+
+                appState.MarketOverview = new MarketOverview
+                {
+                    TimeSegments = marketTimeSegments,
+                    IsUpToDate = segmenter.IsRealtime,
+                    LastUpdatedUtc = DateTime.UtcNow
+                };
+
+                var r = random.Next(appState.Config.MaxRandomIntervalSeconds);
+                await Task.Delay(TimeSpan.FromSeconds(r), ct);
+            }
         }
     }
 }

Tools/TestNetRewarder/Program.cs

@@ -44,7 +44,7 @@ namespace TestNetRewarder
             EnsureGethOnline();
 
             Log.Log("Starting TestNet Rewarder...");
-            var segmenter = new TimeSegmenter(Log, Config, processor);
+            var segmenter = new TimeSegmenter(Log, Config.Interval, Config.HistoryStartUtc, processor);
          
             while (!CancellationToken.IsCancellationRequested)
             {

Tools/TestNetRewarder/TimeSegmenter.cs

@@ -15,28 +15,28 @@ namespace TestNetRewarder
         private readonly TimeSpan segmentSize;
         private DateTime latest;
 
-        public TimeSegmenter(ILog log, Configuration configuration, ITimeSegmentHandler handler)
+        public TimeSegmenter(ILog log, TimeSpan segmentSize, DateTime historyStartUtc, ITimeSegmentHandler handler)
         {
             this.log = log;
             this.handler = handler;
-            if (configuration.IntervalMinutes < 0) configuration.IntervalMinutes = 1;
+            this.segmentSize = segmentSize;
-
+            latest = historyStartUtc;
-            segmentSize = configuration.Interval;
-            latest = configuration.HistoryStartUtc;
 
             log.Log("Starting time segments at " + latest);
             log.Log("Segment size: " + Time.FormatDuration(segmentSize));
         }
 
+        public bool IsRealtime { get; private set; } = false;
+
         public async Task ProcessNextSegment()
         {
             var end = latest + segmentSize;
-            var waited = await WaitUntilTimeSegmentInPast(end);
+            IsRealtime = await WaitUntilTimeSegmentInPast(end);
 
             if (Program.CancellationToken.IsCancellationRequested) return;
 
             var postfix = "(Catching up...)";
-            if (waited) postfix = "(Real-time)";
+            if (IsRealtime) postfix = "(Real-time)";
             log.Log($"Time segment [{latest} to {end}] {postfix}");
             
             var range = new TimeRange(latest, end);