From 3d347a936d2f11bfa03725c440b38f5c9636c972 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 21 Aug 2024 15:03:20 +0200 Subject: [PATCH] Implements marketinsights api --- Framework/Utils/RollingAverage.cs | 9 ++- Tools/MarketInsights/AppState.cs | 8 +- Tools/MarketInsights/AverageHistory.cs | 89 ++++++++------------- Tools/MarketInsights/Configuration.cs | 8 ++ Tools/MarketInsights/ContributionBuilder.cs | 79 ++++++++++++++++++ Tools/MarketInsights/MarketInsights.csproj | 1 + Tools/MarketInsights/Model.cs | 5 ++ Tools/MarketInsights/Program.cs | 24 +++++- Tools/MarketInsights/Tracker.cs | 69 ++++++++++++++++ Tools/MarketInsights/Updater.cs | 50 +++++++++--- Tools/TestNetRewarder/Program.cs | 2 +- Tools/TestNetRewarder/TimeSegmenter.cs | 14 ++-- 12 files changed, 281 insertions(+), 77 deletions(-) create mode 100644 Tools/MarketInsights/ContributionBuilder.cs create mode 100644 Tools/MarketInsights/Tracker.cs diff --git a/Framework/Utils/RollingAverage.cs b/Framework/Utils/RollingAverage.cs index 5674684..c053f1a 100644 --- a/Framework/Utils/RollingAverage.cs +++ b/Framework/Utils/RollingAverage.cs @@ -15,7 +15,14 @@ var originalValue = currentAverage; var originalValueWeight = ((n - 1.0f) / n); var newValueWeight = (1.0f / n); - return (originalValue * originalValueWeight) + (newValue * newValueWeight); + return GetWeightedAverage(originalValue, originalValueWeight, newValue, newValueWeight); + } + + public static float GetWeightedAverage(float value1, float weight1, float value2, float weight2) + { + float totalWeight = weight1 + weight2; + if (totalWeight == 0.0f) return 0.0f; + return ((value1 * weight1) + (value2 * weight2)) / totalWeight; } } } diff --git a/Tools/MarketInsights/AppState.cs b/Tools/MarketInsights/AppState.cs index 8a49f25..ad59a81 100644 --- a/Tools/MarketInsights/AppState.cs +++ b/Tools/MarketInsights/AppState.cs @@ -1,4 +1,6 @@ -namespace MarketInsights +using Logging; + +namespace MarketInsights { public class AppState { @@ -7,7 +9,9 @@ Config = config; } - public MarketOverview MarketOverview { get; set; } = new (); + public bool Realtime { get; set; } + public MarketOverview MarketOverview { get; set; } = new(); public Configuration Config { get; } + public ILog Log { get; } = new ConsoleLog(); } } diff --git a/Tools/MarketInsights/AverageHistory.cs b/Tools/MarketInsights/AverageHistory.cs index d0b4350..007a35f 100644 --- a/Tools/MarketInsights/AverageHistory.cs +++ b/Tools/MarketInsights/AverageHistory.cs @@ -1,71 +1,50 @@ -using CodexContractsPlugin.ChainMonitor; -using GethPlugin; -using System.Numerics; +using CodexContractsPlugin; +using CodexContractsPlugin.ChainMonitor; +using Nethereum.Model; +using TestNetRewarder; using Utils; namespace MarketInsights { - public class AverageHistory + public class AverageHistory : ITimeSegmentHandler { - public readonly List contributions = new List(); + private readonly List contributions = new List(); + private readonly ChainStateChangeHandlerMux mux = new ChainStateChangeHandlerMux(); + private readonly AppState appState; + private readonly int maxContributions; + private readonly ChainState chainState; - } - - public class ContributionBuilder : IChainStateChangeHandler - { - private readonly MarketTimeSegment segment = new MarketTimeSegment(); - - public void OnNewRequest(RequestEvent requestEvent) + public AverageHistory(AppState appState, ICodexContracts contracts, int maxContributions) { - AddRequestToAverage(segment.Submitted, requestEvent); + this.appState = appState; + this.maxContributions = maxContributions; + chainState = new ChainState(appState.Log, contracts, mux, appState.Config.HistoryStartUtc); } - public void OnRequestCancelled(RequestEvent requestEvent) + public MarketTimeSegment[] Segments { get; private set; } = Array.Empty(); + + public Task OnNewSegment(TimeRange timeRange) { - AddRequestToAverage(segment.Expired, requestEvent); + var contribution = BuildContribution(timeRange); + contributions.Add(contribution); + + while (contributions.Count > maxContributions) + { + contributions.RemoveAt(0); + } + + Segments = contributions.ToArray(); + + return Task.CompletedTask; } - public void OnRequestFailed(RequestEvent requestEvent) + private MarketTimeSegment BuildContribution(TimeRange timeRange) { - AddRequestToAverage(segment.Failed, requestEvent); - } - - public void OnRequestFinished(RequestEvent requestEvent) - { - AddRequestToAverage(segment.Finished, requestEvent); - } - - public void OnRequestFulfilled(RequestEvent requestEvent) - { - AddRequestToAverage(segment.Started, requestEvent); - } - - public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex) - { - } - - public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex) - { - } - - private void AddRequestToAverage(ContractAverages average, RequestEvent requestEvent) - { - average.Number++; - average.Price = GetNewAverage(average.Price, average.Number, requestEvent.Request.Request.Ask.Reward); - average.Size = GetNewAverage(average.Size, average.Number, requestEvent.Request.Request.Ask.SlotSize); - average.Duration = GetNewAverage(average.Duration, average.Number, requestEvent.Request.Request.Ask.Duration); - average.Collateral = GetNewAverage(average.Collateral, average.Number, requestEvent.Request.Request.Ask.Collateral); - average.ProofProbability = GetNewAverage(average.ProofProbability, average.Number, requestEvent.Request.Request.Ask.ProofProbability); - } - - private float GetNewAverage(float currentAverage, int newNumberOfValues, BigInteger newValue) - { - return GetNewAverage(currentAverage, newNumberOfValues, (float)newValue); - } - - private float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue) - { - return RollingAverage.GetNewAverage(currentAverage, newNumberOfValues, newValue); + var builder = new ContributionBuilder(timeRange); + mux.Handlers.Add(builder); + chainState.Update(timeRange.To); + mux.Handlers.Remove(builder); + return builder.GetSegment(); } } } diff --git a/Tools/MarketInsights/Configuration.cs b/Tools/MarketInsights/Configuration.cs index f8fa574..d4ffde9 100644 --- a/Tools/MarketInsights/Configuration.cs +++ b/Tools/MarketInsights/Configuration.cs @@ -33,5 +33,13 @@ namespace MarketInsights return DateTimeOffset.FromUnixTimeSeconds(CheckHistoryTimestamp).UtcDateTime; } } + + public TimeSpan UpdateInterval + { + get + { + return TimeSpan.FromMinutes(UpdateIntervalMinutes); + } + } } } diff --git a/Tools/MarketInsights/ContributionBuilder.cs b/Tools/MarketInsights/ContributionBuilder.cs new file mode 100644 index 0000000..e38d273 --- /dev/null +++ b/Tools/MarketInsights/ContributionBuilder.cs @@ -0,0 +1,79 @@ +using CodexContractsPlugin.ChainMonitor; +using GethPlugin; +using System.Numerics; +using Utils; + +namespace MarketInsights +{ + public class ContributionBuilder : IChainStateChangeHandler + { + private readonly MarketTimeSegment segment = new MarketTimeSegment(); + + public ContributionBuilder(TimeRange timeRange) + { + segment = new MarketTimeSegment + { + FromUtc = timeRange.From, + ToUtc = timeRange.To + }; + } + + public void OnNewRequest(RequestEvent requestEvent) + { + AddRequestToAverage(segment.Submitted, requestEvent); + } + + public void OnRequestCancelled(RequestEvent requestEvent) + { + AddRequestToAverage(segment.Expired, requestEvent); + } + + public void OnRequestFailed(RequestEvent requestEvent) + { + AddRequestToAverage(segment.Failed, requestEvent); + } + + public void OnRequestFinished(RequestEvent requestEvent) + { + AddRequestToAverage(segment.Finished, requestEvent); + } + + public void OnRequestFulfilled(RequestEvent requestEvent) + { + AddRequestToAverage(segment.Started, requestEvent); + } + + public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex) + { + } + + public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex) + { + } + + public MarketTimeSegment GetSegment() + { + return segment; + } + + private void AddRequestToAverage(ContractAverages average, RequestEvent requestEvent) + { + average.Number++; + average.Price = GetNewAverage(average.Price, average.Number, requestEvent.Request.Request.Ask.Reward); + average.Size = GetNewAverage(average.Size, average.Number, requestEvent.Request.Request.Ask.SlotSize); + average.Duration = GetNewAverage(average.Duration, average.Number, requestEvent.Request.Request.Ask.Duration); + average.Collateral = GetNewAverage(average.Collateral, average.Number, requestEvent.Request.Request.Ask.Collateral); + average.ProofProbability = GetNewAverage(average.ProofProbability, average.Number, requestEvent.Request.Request.Ask.ProofProbability); + } + + private float GetNewAverage(float currentAverage, int newNumberOfValues, BigInteger newValue) + { + return GetNewAverage(currentAverage, newNumberOfValues, (float)newValue); + } + + private float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue) + { + return RollingAverage.GetNewAverage(currentAverage, newNumberOfValues, newValue); + } + } +} diff --git a/Tools/MarketInsights/MarketInsights.csproj b/Tools/MarketInsights/MarketInsights.csproj index 08ec3d7..42b33c4 100644 --- a/Tools/MarketInsights/MarketInsights.csproj +++ b/Tools/MarketInsights/MarketInsights.csproj @@ -19,6 +19,7 @@ + diff --git a/Tools/MarketInsights/Model.cs b/Tools/MarketInsights/Model.cs index 06057ca..65a82f7 100644 --- a/Tools/MarketInsights/Model.cs +++ b/Tools/MarketInsights/Model.cs @@ -7,6 +7,11 @@ /// public DateTime LastUpdatedUtc { get; set; } + /// + /// When false, service is busy processing history in order to catch up to the present. + /// + public bool IsUpToDate { get; set; } + public MarketTimeSegment[] TimeSegments { get; set; } = Array.Empty(); } diff --git a/Tools/MarketInsights/Program.cs b/Tools/MarketInsights/Program.cs index 07f5853..2a10e5e 100644 --- a/Tools/MarketInsights/Program.cs +++ b/Tools/MarketInsights/Program.cs @@ -1,5 +1,6 @@ using ArgsUniform; using Microsoft.Extensions.Options; +using Nethereum.Model; using System.Reflection; namespace MarketInsights @@ -10,9 +11,20 @@ namespace MarketInsights { var uniformArgs = new ArgsUniform(PrintHelp, args); var config = uniformArgs.Parse(true); - + var cts = new CancellationTokenSource(); var appState = new AppState(config); - var updater = new Updater(appState); + + Console.CancelKeyPress += (s, e) => + { + appState.Log.Log("Stopping..."); + cts.Cancel(); + e.Cancel = true; + }; + + var connector = GethConnector.GethConnector.Initialize(appState.Log); + if (connector == null) throw new Exception("Invalid Geth information"); + + var updater = new Updater(appState, connector.CodexContracts, cts.Token); var builder = WebApplication.CreateBuilder(args); @@ -49,6 +61,14 @@ namespace MarketInsights private static void PrintHelp() { Console.WriteLine("WebAPI for generating market overview for Codex network. Comes with OpenAPI swagger endpoint."); + + var nl = Environment.NewLine; + Console.WriteLine($"Required environment variables: {nl}" + + $"'GETH_HOST'{nl}", + $"'GETH_HTTP_PORT'{nl}", + $"'CODEXCONTRACTS_MARKETPLACEADDRESS'{nl}", + $"'CODEXCONTRACTS_TOKENADDRESS'{nl}", + $"'CODEXCONTRACTS_ABI'{nl}"); } } } diff --git a/Tools/MarketInsights/Tracker.cs b/Tools/MarketInsights/Tracker.cs new file mode 100644 index 0000000..3e4354d --- /dev/null +++ b/Tools/MarketInsights/Tracker.cs @@ -0,0 +1,69 @@ +using CodexContractsPlugin.ChainMonitor; +using Utils; +using YamlDotNet.Core; + +namespace MarketInsights +{ + public class Tracker + { + private readonly AverageHistory history; + + public Tracker(int numberOfSegments, AverageHistory history) + { + NumberOfSegments = numberOfSegments; + this.history = history; + } + + public int NumberOfSegments { get; } + + public MarketTimeSegment? CreateMarketTimeSegment() + { + if (history.Segments.Length < NumberOfSegments) return null; + + var mySegments = history.Segments.TakeLast(NumberOfSegments); + return AverageSegments(mySegments); + } + + private MarketTimeSegment AverageSegments(IEnumerable mySegments) + { + var result = new MarketTimeSegment(); + + foreach (var segment in mySegments) + { + result.FromUtc = Min(result.FromUtc, segment.FromUtc); + result.ToUtc = Max(result.ToUtc, segment.ToUtc); + + Combine(result.Submitted, segment.Submitted); + Combine(result.Expired, segment.Expired); + Combine(result.Started, segment.Started); + Combine(result.Finished, segment.Finished); + Combine(result.Failed, segment.Failed); + } + return result; + } + + private void Combine(ContractAverages result, ContractAverages toAdd) + { + float weight1 = result.Number; + float weight2 = toAdd.Number; + + result.Price = RollingAverage.GetWeightedAverage(result.Price, weight1, toAdd.Price, weight2); + result.Size = RollingAverage.GetWeightedAverage(result.Size, weight1, toAdd.Size, weight2); + result.Duration = RollingAverage.GetWeightedAverage(result.Duration, weight1, toAdd.Duration, weight2); + result.Collateral = RollingAverage.GetWeightedAverage(result.Collateral, weight1, toAdd.Collateral, weight2); + result.ProofProbability = RollingAverage.GetWeightedAverage(result.ProofProbability, weight1, toAdd.ProofProbability, weight2); + } + + private DateTime Max(DateTime a, DateTime b) + { + if (a > b) return a; + return b; + } + + private DateTime Min(DateTime a, DateTime b) + { + if (a > b) return b; + return a; + } + } +} diff --git a/Tools/MarketInsights/Updater.cs b/Tools/MarketInsights/Updater.cs index 11419c9..87743f6 100644 --- a/Tools/MarketInsights/Updater.cs +++ b/Tools/MarketInsights/Updater.cs @@ -1,35 +1,67 @@ - +using CodexContractsPlugin; +using TestNetRewarder; + namespace MarketInsights { public class Updater { + private readonly Random random = new Random(); private readonly AppState appState; + private readonly CancellationToken ct; private readonly Tracker[] trackers; + private readonly AverageHistory averageHistory; - public Updater(AppState appState) + public Updater(AppState appState, ICodexContracts contracts, CancellationToken ct) { this.appState = appState; + this.ct = ct; + trackers = CreateTrackers(); + averageHistory = new AverageHistory(appState, contracts, trackers.Max(t => t.NumberOfSegments)); } private Tracker[] CreateTrackers() { var tokens = appState.Config.TimeSegments.Split(";", StringSplitOptions.RemoveEmptyEntries); var nums = tokens.Select(t => Convert.ToInt32(t)).ToArray(); - return nums.Select(n => new Tracker(n)).ToArray(); + return nums.Select(n => new Tracker(n, averageHistory)).ToArray(); } public void Run() { - + Task.Run(Runner); } - } - public class Tracker - { - public Tracker(int numberOfSegments) + private async Task Runner() { - + var segmenter = new TimeSegmenter( + appState.Log, + segmentSize: appState.Config.UpdateInterval, + historyStartUtc: appState.Config.HistoryStartUtc, + handler: averageHistory + ); + + while (!ct.IsCancellationRequested) + { + await segmenter.ProcessNextSegment(); + await Task.Delay(TimeSpan.FromSeconds(3), ct); + + var marketTimeSegments = trackers + .Select(t => t.CreateMarketTimeSegment()) + .Where(t => t != null) + .Cast() + .ToArray(); + + appState.MarketOverview = new MarketOverview + { + TimeSegments = marketTimeSegments, + IsUpToDate = segmenter.IsRealtime, + LastUpdatedUtc = DateTime.UtcNow + }; + + var r = random.Next(appState.Config.MaxRandomIntervalSeconds); + await Task.Delay(TimeSpan.FromSeconds(r), ct); + } } } } diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index 0056ffe..8c504e0 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -44,7 +44,7 @@ namespace TestNetRewarder EnsureGethOnline(); Log.Log("Starting TestNet Rewarder..."); - var segmenter = new TimeSegmenter(Log, Config, processor); + var segmenter = new TimeSegmenter(Log, Config.Interval, Config.HistoryStartUtc, processor); while (!CancellationToken.IsCancellationRequested) { diff --git a/Tools/TestNetRewarder/TimeSegmenter.cs b/Tools/TestNetRewarder/TimeSegmenter.cs index 24899ed..39a9bbb 100644 --- a/Tools/TestNetRewarder/TimeSegmenter.cs +++ b/Tools/TestNetRewarder/TimeSegmenter.cs @@ -15,28 +15,28 @@ namespace TestNetRewarder private readonly TimeSpan segmentSize; private DateTime latest; - public TimeSegmenter(ILog log, Configuration configuration, ITimeSegmentHandler handler) + public TimeSegmenter(ILog log, TimeSpan segmentSize, DateTime historyStartUtc, ITimeSegmentHandler handler) { this.log = log; this.handler = handler; - if (configuration.IntervalMinutes < 0) configuration.IntervalMinutes = 1; - - segmentSize = configuration.Interval; - latest = configuration.HistoryStartUtc; + this.segmentSize = segmentSize; + latest = historyStartUtc; log.Log("Starting time segments at " + latest); log.Log("Segment size: " + Time.FormatDuration(segmentSize)); } + public bool IsRealtime { get; private set; } = false; + public async Task ProcessNextSegment() { var end = latest + segmentSize; - var waited = await WaitUntilTimeSegmentInPast(end); + IsRealtime = await WaitUntilTimeSegmentInPast(end); if (Program.CancellationToken.IsCancellationRequested) return; var postfix = "(Catching up...)"; - if (waited) postfix = "(Real-time)"; + if (IsRealtime) postfix = "(Real-time)"; log.Log($"Time segment [{latest} to {end}] {postfix}"); var range = new TimeRange(latest, end);