status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implements marketinsights api

This commit is contained in:
Ben 2024-08-21 15:03:20 +02:00
parent 87bda475df
commit 3d347a936d
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
12 changed files with 281 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Framework/Utils/RollingAverage.cs
View File

@ -15,7 +15,14 @@
var originalValue = currentAverage;
var originalValueWeight = ((n - 1.0f) / n);
var newValueWeight = (1.0f / n);
return (originalValue * originalValueWeight) + (newValue * newValueWeight);
return GetWeightedAverage(originalValue, originalValueWeight, newValue, newValueWeight);
}
public static float GetWeightedAverage(float value1, float weight1, float value2, float weight2)
{
float totalWeight = weight1 + weight2;
if (totalWeight == 0.0f) return 0.0f;
return ((value1 * weight1) + (value2 * weight2)) / totalWeight;
}
}
}

8
Tools/MarketInsights/AppState.cs
View File

@ -1,4 +1,6 @@
namespace MarketInsights
using Logging;
namespace MarketInsights
{
public class AppState
{
@ -7,7 +9,9 @@
Config = config;
}
public MarketOverview MarketOverview { get; set; } = new ();
public bool Realtime { get; set; }
public MarketOverview MarketOverview { get; set; } = new();
public Configuration Config { get; }
public ILog Log { get; } = new ConsoleLog();
}
}

89
Tools/MarketInsights/AverageHistory.cs
View File

@ -1,71 +1,50 @@
using CodexContractsPlugin.ChainMonitor;
using GethPlugin;
using System.Numerics;
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using Nethereum.Model;
using TestNetRewarder;
using Utils;
namespace MarketInsights
{
public class AverageHistory
public class AverageHistory : ITimeSegmentHandler
{
public readonly List<MarketTimeSegment> contributions = new List<MarketTimeSegment>();
private readonly List<MarketTimeSegment> contributions = new List<MarketTimeSegment>();
private readonly ChainStateChangeHandlerMux mux = new ChainStateChangeHandlerMux();
private readonly AppState appState;
private readonly int maxContributions;
private readonly ChainState chainState;
}
public class ContributionBuilder : IChainStateChangeHandler
{
private readonly MarketTimeSegment segment = new MarketTimeSegment();
public void OnNewRequest(RequestEvent requestEvent)
public AverageHistory(AppState appState, ICodexContracts contracts, int maxContributions)
{
AddRequestToAverage(segment.Submitted, requestEvent);
this.appState = appState;
this.maxContributions = maxContributions;
chainState = new ChainState(appState.Log, contracts, mux, appState.Config.HistoryStartUtc);
}
public void OnRequestCancelled(RequestEvent requestEvent)
public MarketTimeSegment[] Segments { get; private set; } = Array.Empty<MarketTimeSegment>();
public Task OnNewSegment(TimeRange timeRange)
{
AddRequestToAverage(segment.Expired, requestEvent);
var contribution = BuildContribution(timeRange);
contributions.Add(contribution);
while (contributions.Count > maxContributions)
{
contributions.RemoveAt(0);
}
Segments = contributions.ToArray();
return Task.CompletedTask;
}
public void OnRequestFailed(RequestEvent requestEvent)
private MarketTimeSegment BuildContribution(TimeRange timeRange)
{
AddRequestToAverage(segment.Failed, requestEvent);
}
public void OnRequestFinished(RequestEvent requestEvent)
{
AddRequestToAverage(segment.Finished, requestEvent);
}
public void OnRequestFulfilled(RequestEvent requestEvent)
{
AddRequestToAverage(segment.Started, requestEvent);
}
public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
{
}
public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
{
}
private void AddRequestToAverage(ContractAverages average, RequestEvent requestEvent)
{
average.Number++;
average.Price = GetNewAverage(average.Price, average.Number, requestEvent.Request.Request.Ask.Reward);
average.Size = GetNewAverage(average.Size, average.Number, requestEvent.Request.Request.Ask.SlotSize);
average.Duration = GetNewAverage(average.Duration, average.Number, requestEvent.Request.Request.Ask.Duration);
average.Collateral = GetNewAverage(average.Collateral, average.Number, requestEvent.Request.Request.Ask.Collateral);
average.ProofProbability = GetNewAverage(average.ProofProbability, average.Number, requestEvent.Request.Request.Ask.ProofProbability);
}
private float GetNewAverage(float currentAverage, int newNumberOfValues, BigInteger newValue)
{
return GetNewAverage(currentAverage, newNumberOfValues, (float)newValue);
}
private float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue)
{
return RollingAverage.GetNewAverage(currentAverage, newNumberOfValues, newValue);
var builder = new ContributionBuilder(timeRange);
mux.Handlers.Add(builder);
chainState.Update(timeRange.To);
mux.Handlers.Remove(builder);
return builder.GetSegment();
}
}
}

8
Tools/MarketInsights/Configuration.cs
View File

@ -33,5 +33,13 @@ namespace MarketInsights
return DateTimeOffset.FromUnixTimeSeconds(CheckHistoryTimestamp).UtcDateTime;
}
}
public TimeSpan UpdateInterval
{
get
{
return TimeSpan.FromMinutes(UpdateIntervalMinutes);
}
}
}
}

79
Tools/MarketInsights/ContributionBuilder.cs Normal file
View File

@ -0,0 +1,79 @@
using CodexContractsPlugin.ChainMonitor;
using GethPlugin;
using System.Numerics;
using Utils;
namespace MarketInsights
{
public class ContributionBuilder : IChainStateChangeHandler
{
private readonly MarketTimeSegment segment = new MarketTimeSegment();
public ContributionBuilder(TimeRange timeRange)
{
segment = new MarketTimeSegment
{
FromUtc = timeRange.From,
ToUtc = timeRange.To
};
}
public void OnNewRequest(RequestEvent requestEvent)
{
AddRequestToAverage(segment.Submitted, requestEvent);
}
public void OnRequestCancelled(RequestEvent requestEvent)
{
AddRequestToAverage(segment.Expired, requestEvent);
}
public void OnRequestFailed(RequestEvent requestEvent)
{
AddRequestToAverage(segment.Failed, requestEvent);
}
public void OnRequestFinished(RequestEvent requestEvent)
{
AddRequestToAverage(segment.Finished, requestEvent);
}
public void OnRequestFulfilled(RequestEvent requestEvent)
{
AddRequestToAverage(segment.Started, requestEvent);
}
public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
{
}
public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
{
}
public MarketTimeSegment GetSegment()
{
return segment;
}
private void AddRequestToAverage(ContractAverages average, RequestEvent requestEvent)
{
average.Number++;
average.Price = GetNewAverage(average.Price, average.Number, requestEvent.Request.Request.Ask.Reward);
average.Size = GetNewAverage(average.Size, average.Number, requestEvent.Request.Request.Ask.SlotSize);
average.Duration = GetNewAverage(average.Duration, average.Number, requestEvent.Request.Request.Ask.Duration);
average.Collateral = GetNewAverage(average.Collateral, average.Number, requestEvent.Request.Request.Ask.Collateral);
average.ProofProbability = GetNewAverage(average.ProofProbability, average.Number, requestEvent.Request.Request.Ask.ProofProbability);
}
private float GetNewAverage(float currentAverage, int newNumberOfValues, BigInteger newValue)
{
return GetNewAverage(currentAverage, newNumberOfValues, (float)newValue);
}
private float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue)
{
return RollingAverage.GetNewAverage(currentAverage, newNumberOfValues, newValue);
}
}
}

1
Tools/MarketInsights/MarketInsights.csproj
View File

@ -19,6 +19,7 @@
<ItemGroup>
<ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
<ProjectReference Include="..\TestNetRewarder\TestNetRewarder.csproj" />
</ItemGroup>
</Project>

5
Tools/MarketInsights/Model.cs
View File

@ -7,6 +7,11 @@
/// </summary>
public DateTime LastUpdatedUtc { get; set; }
/// <summary>
/// When false, service is busy processing history in order to catch up to the present.
/// </summary>
public bool IsUpToDate { get; set; }
public MarketTimeSegment[] TimeSegments { get; set; } = Array.Empty<MarketTimeSegment>();
}

24
Tools/MarketInsights/Program.cs
View File

@ -1,5 +1,6 @@
using ArgsUniform;
using Microsoft.Extensions.Options;
using Nethereum.Model;
using System.Reflection;
namespace MarketInsights
@ -10,9 +11,20 @@ namespace MarketInsights
{
var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
var config = uniformArgs.Parse(true);
var cts = new CancellationTokenSource();
var appState = new AppState(config);
var updater = new Updater(appState);
Console.CancelKeyPress += (s, e) =>
{
appState.Log.Log("Stopping...");
cts.Cancel();
e.Cancel = true;
};
var connector = GethConnector.GethConnector.Initialize(appState.Log);
if (connector == null) throw new Exception("Invalid Geth information");
var updater = new Updater(appState, connector.CodexContracts, cts.Token);
var builder = WebApplication.CreateBuilder(args);
@ -49,6 +61,14 @@ namespace MarketInsights
private static void PrintHelp()
{
Console.WriteLine("WebAPI for generating market overview for Codex network. Comes with OpenAPI swagger endpoint.");
var nl = Environment.NewLine;
Console.WriteLine($"Required environment variables: {nl}" +
$"'GETH_HOST'{nl}",
$"'GETH_HTTP_PORT'{nl}",
$"'CODEXCONTRACTS_MARKETPLACEADDRESS'{nl}",
$"'CODEXCONTRACTS_TOKENADDRESS'{nl}",
$"'CODEXCONTRACTS_ABI'{nl}");
}
}
}

69
Tools/MarketInsights/Tracker.cs Normal file
View File

@ -0,0 +1,69 @@
using CodexContractsPlugin.ChainMonitor;
using Utils;
using YamlDotNet.Core;
namespace MarketInsights
{
public class Tracker
{
private readonly AverageHistory history;
public Tracker(int numberOfSegments, AverageHistory history)
{
NumberOfSegments = numberOfSegments;
this.history = history;
}
public int NumberOfSegments { get; }
public MarketTimeSegment? CreateMarketTimeSegment()
{
if (history.Segments.Length < NumberOfSegments) return null;
var mySegments = history.Segments.TakeLast(NumberOfSegments);
return AverageSegments(mySegments);
}
private MarketTimeSegment AverageSegments(IEnumerable<MarketTimeSegment> mySegments)
{
var result = new MarketTimeSegment();
foreach (var segment in mySegments)
{
result.FromUtc = Min(result.FromUtc, segment.FromUtc);
result.ToUtc = Max(result.ToUtc, segment.ToUtc);
Combine(result.Submitted, segment.Submitted);
Combine(result.Expired, segment.Expired);
Combine(result.Started, segment.Started);
Combine(result.Finished, segment.Finished);
Combine(result.Failed, segment.Failed);
}
return result;
}
private void Combine(ContractAverages result, ContractAverages toAdd)
{
float weight1 = result.Number;
float weight2 = toAdd.Number;
result.Price = RollingAverage.GetWeightedAverage(result.Price, weight1, toAdd.Price, weight2);
result.Size = RollingAverage.GetWeightedAverage(result.Size, weight1, toAdd.Size, weight2);
result.Duration = RollingAverage.GetWeightedAverage(result.Duration, weight1, toAdd.Duration, weight2);
result.Collateral = RollingAverage.GetWeightedAverage(result.Collateral, weight1, toAdd.Collateral, weight2);
result.ProofProbability = RollingAverage.GetWeightedAverage(result.ProofProbability, weight1, toAdd.ProofProbability, weight2);
}
private DateTime Max(DateTime a, DateTime b)
{
if (a > b) return a;
return b;
}
private DateTime Min(DateTime a, DateTime b)
{
if (a > b) return b;
return a;
}
}
}

48
Tools/MarketInsights/Updater.cs
View File

@ -1,35 +1,67 @@

using CodexContractsPlugin;
using TestNetRewarder;
namespace MarketInsights
{
public class Updater
{
private readonly Random random = new Random();
private readonly AppState appState;
private readonly CancellationToken ct;
private readonly Tracker[] trackers;
private readonly AverageHistory averageHistory;
public Updater(AppState appState)
public Updater(AppState appState, ICodexContracts contracts, CancellationToken ct)
{
this.appState = appState;
this.ct = ct;
trackers = CreateTrackers();
averageHistory = new AverageHistory(appState, contracts, trackers.Max(t => t.NumberOfSegments));
}
private Tracker[] CreateTrackers()
{
var tokens = appState.Config.TimeSegments.Split(";", StringSplitOptions.RemoveEmptyEntries);
var nums = tokens.Select(t => Convert.ToInt32(t)).ToArray();
return nums.Select(n => new Tracker(n)).ToArray();
return nums.Select(n => new Tracker(n, averageHistory)).ToArray();
}
public void Run()
{
Task.Run(Runner);
}
}
public class Tracker
{
public Tracker(int numberOfSegments)
private async Task Runner()
{
var segmenter = new TimeSegmenter(
appState.Log,
segmentSize: appState.Config.UpdateInterval,
historyStartUtc: appState.Config.HistoryStartUtc,
handler: averageHistory
);
while (!ct.IsCancellationRequested)
{
await segmenter.ProcessNextSegment();
await Task.Delay(TimeSpan.FromSeconds(3), ct);
var marketTimeSegments = trackers
.Select(t => t.CreateMarketTimeSegment())
.Where(t => t != null)
.Cast<MarketTimeSegment>()
.ToArray();
appState.MarketOverview = new MarketOverview
{
TimeSegments = marketTimeSegments,
IsUpToDate = segmenter.IsRealtime,
LastUpdatedUtc = DateTime.UtcNow
};
var r = random.Next(appState.Config.MaxRandomIntervalSeconds);
await Task.Delay(TimeSpan.FromSeconds(r), ct);
}
}
}
}

2
Tools/TestNetRewarder/Program.cs
View File

@ -44,7 +44,7 @@ namespace TestNetRewarder
EnsureGethOnline();
Log.Log("Starting TestNet Rewarder...");
var segmenter = new TimeSegmenter(Log, Config, processor);
var segmenter = new TimeSegmenter(Log, Config.Interval, Config.HistoryStartUtc, processor);
while (!CancellationToken.IsCancellationRequested)
{

14
Tools/TestNetRewarder/TimeSegmenter.cs
View File

@ -15,28 +15,28 @@ namespace TestNetRewarder
private readonly TimeSpan segmentSize;
private DateTime latest;
public TimeSegmenter(ILog log, Configuration configuration, ITimeSegmentHandler handler)
public TimeSegmenter(ILog log, TimeSpan segmentSize, DateTime historyStartUtc, ITimeSegmentHandler handler)
{
this.log = log;
this.handler = handler;
if (configuration.IntervalMinutes < 0) configuration.IntervalMinutes = 1;
segmentSize = configuration.Interval;
latest = configuration.HistoryStartUtc;
this.segmentSize = segmentSize;
latest = historyStartUtc;
log.Log("Starting time segments at " + latest);
log.Log("Segment size: " + Time.FormatDuration(segmentSize));
}
public bool IsRealtime { get; private set; } = false;
public async Task ProcessNextSegment()
{
var end = latest + segmentSize;
var waited = await WaitUntilTimeSegmentInPast(end);
IsRealtime = await WaitUntilTimeSegmentInPast(end);
if (Program.CancellationToken.IsCancellationRequested) return;
var postfix = "(Catching up...)";
if (waited) postfix = "(Real-time)";
if (IsRealtime) postfix = "(Real-time)";
log.Log($"Time segment [{latest} to {end}] {postfix}");
var range = new TimeRange(latest, end);