Compare commits

...
17 changed files with 264 additions and 159 deletions
@@ -15,8 +15,7 @@
public class MarketAverage
{
public int NumberOfFinished { get; set; }
public TimeSpan TimeRange { get; set; }
public string Title { get; set; } = string.Empty;
public float Price { get; set; }
public float Size { get; set; }
public float Duration { get; set; }
@@ -18,6 +18,13 @@ namespace NethereumWorkflow.BlockUtils
bounds = new BlockchainBounds(cache, web3);
}
public BlockTimeEntry Get(ulong blockNumber)
{
var b = cache.Get(blockNumber);
if (b != null) return b;
return GetBlock(blockNumber);
}
public ulong? GetHighestBlockNumberBefore(DateTime moment)
{
bounds.Initialize();
@@ -121,5 +121,12 @@ namespace NethereumWorkflow
to: toBlock.Value
);
}
public BlockTimeEntry GetBlockForNumber(ulong number)
{
var wrapper = new Web3Wrapper(web3, log);
var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
return blockTimeFinder.Get(number);
}
}
}
@@ -5,6 +5,9 @@ using Nethereum.ABI;
using Nethereum.Hex.HexTypes;
using Nethereum.Util;
using NethereumWorkflow;
using NethereumWorkflow.BlockUtils;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Utils;
namespace CodexContractsPlugin
@@ -28,6 +31,7 @@ namespace CodexContractsPlugin
SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange);
}
[JsonConverter(typeof(StringEnumConverter))]
public enum RequestState
{
New,
@@ -86,7 +90,7 @@ namespace CodexContractsPlugin
{
var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
var result = requestEvent.ReturnValue1;
result.BlockNumber = e.Log.BlockNumber.ToUlong();
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
result.RequestId = e.Event.RequestId;
return result;
})
@@ -99,7 +103,7 @@ namespace CodexContractsPlugin
return events.Select(e =>
{
var result = e.Event;
result.BlockNumber = e.Log.BlockNumber.ToUlong();
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
return result;
}).ToArray();
}
@@ -110,7 +114,7 @@ namespace CodexContractsPlugin
return events.Select(e =>
{
var result = e.Event;
result.BlockNumber = e.Log.BlockNumber.ToUlong();
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
return result;
}).ToArray();
}
@@ -121,7 +125,7 @@ namespace CodexContractsPlugin
return events.Select(e =>
{
var result = e.Event;
result.BlockNumber = e.Log.BlockNumber.ToUlong();
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
return result;
}).ToArray();
@@ -133,7 +137,7 @@ namespace CodexContractsPlugin
return events.Select(e =>
{
var result = e.Event;
result.BlockNumber = e.Log.BlockNumber.ToUlong();
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
return result;
}).ToArray();
}
@@ -166,6 +170,11 @@ namespace CodexContractsPlugin
return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
}
private BlockTimeEntry GetBlock(ulong number)
{
return gethNode.GetBlockForNumber(number);
}
private EthAddress GetEthAddressFromTransaction(string transactionHash)
{
var transaction = gethNode.GetTransaction(transactionHash);
@@ -1,4 +1,5 @@
using KubernetesWorkflow;
using CodexContractsPlugin.Marketplace;
using KubernetesWorkflow;
using KubernetesWorkflow.Types;
using Logging;
using Newtonsoft.Json;
@@ -53,7 +54,18 @@ namespace CodexContractsPlugin
var artifact = JObject.Parse(json);
var abi = artifact["abi"];
return abi!.ToString(Formatting.None);
var byteCode = artifact["bytecode"];
var abiResult = abi!.ToString(Formatting.None);
var byteCodeResult = byteCode!.ToString(Formatting.None);
if (byteCodeResult
.ToLowerInvariant()
.Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
{
throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
}
return abiResult;
}
private static string Retry(Func<string> fetch)
@@ -1,5 +1,6 @@
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
using GethPlugin;
using NethereumWorkflow.BlockUtils;
using Newtonsoft.Json;
namespace CodexContractsPlugin.Marketplace
@@ -7,7 +8,7 @@ namespace CodexContractsPlugin.Marketplace
public partial class Request : RequestBase
{
[JsonIgnore]
public ulong BlockNumber { get; set; }
public BlockTimeEntry Block { get; set; }
public byte[] RequestId { get; set; }
public EthAddress ClientAddress { get { return new EthAddress(Client); } }
@@ -16,26 +17,26 @@ namespace CodexContractsPlugin.Marketplace
public partial class RequestFulfilledEventDTO
{
[JsonIgnore]
public ulong BlockNumber { get; set; }
public BlockTimeEntry Block { get; set; }
}
public partial class RequestCancelledEventDTO
{
[JsonIgnore]
public ulong BlockNumber { get; set; }
public BlockTimeEntry Block { get; set; }
}
public partial class SlotFilledEventDTO
{
[JsonIgnore]
public ulong BlockNumber { get; set; }
public BlockTimeEntry Block { get; set; }
public EthAddress Host { get; set; }
}
public partial class SlotFreedEventDTO
{
[JsonIgnore]
public ulong BlockNumber { get; set; }
public BlockTimeEntry Block { get; set; }
}
}
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
File diff suppressed because one or more lines are too long
+7
View File
@@ -5,6 +5,7 @@ using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.Contracts;
using Nethereum.RPC.Eth.DTOs;
using NethereumWorkflow;
using NethereumWorkflow.BlockUtils;
using Utils;
namespace GethPlugin
@@ -27,6 +28,7 @@ namespace GethPlugin
List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new();
List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
BlockTimeEntry GetBlockForNumber(ulong number);
}
public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -160,6 +162,11 @@ namespace GethPlugin
return StartInteraction().ConvertTimeRangeToBlockRange(timeRange);
}
public BlockTimeEntry GetBlockForNumber(ulong number)
{
return StartInteraction().GetBlockForNumber(number);
}
protected abstract NethereumInteraction StartInteraction();
}
}
+1 -7
View File
@@ -1,7 +1,6 @@
using BiblioTech.Options;
using DiscordRewards;
using System.Globalization;
using Utils;
namespace BiblioTech.Commands
{
@@ -31,12 +30,7 @@ namespace BiblioTech.Commands
private string[] GetInsight(MarketAverage avg)
{
var headerLine = $"[Last {Time.FormatDuration(avg.TimeRange)}] ({avg.NumberOfFinished} Contracts finished)";
if (avg.NumberOfFinished == 0)
{
return new[] { headerLine };
}
var headerLine = $"[{avg.Title}]";
return new[]
{
+4 -1
View File
@@ -23,7 +23,10 @@ namespace BiblioTech.Rewards
{
try
{
Program.Averages = cmd.Averages;
if (cmd.Averages != null && cmd.Averages.Any())
{
Program.Averages = cmd.Averages;
}
await Program.RoleDriver.GiveRewards(cmd);
}
catch (Exception ex)
+20 -4
View File
@@ -24,6 +24,16 @@ namespace BiblioTech.Rewards
{
Program.Log.Log($"Processing rewards command: '{JsonConvert.SerializeObject(rewards)}'");
if (rewards.Rewards.Any())
{
await ProcessRewards(rewards);
}
await ProcessChainEvents(rewards.EventsOverview);
}
private async Task ProcessRewards(GiveRewardsCommand rewards)
{
var guild = GetGuild();
// We load all role and user information first,
// so we don't ask the server for the same info multiple times.
@@ -33,7 +43,6 @@ namespace BiblioTech.Rewards
rewardsChannel);
await context.ProcessGiveRewardsCommand(LookUpUsers(rewards));
await ProcessChainEvents(rewards.EventsOverview);
}
private SocketTextChannel? GetChannel(string name)
@@ -45,10 +54,17 @@ namespace BiblioTech.Rewards
private async Task ProcessChainEvents(string[] eventsOverview)
{
if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
foreach (var e in eventsOverview)
await Task.Run(async () =>
{
await eventsChannel.SendMessageAsync(e);
}
foreach (var e in eventsOverview)
{
if (!string.IsNullOrEmpty(e))
{
await eventsChannel.SendMessageAsync(e);
await Task.Delay(3000);
}
}
});
}
private async Task<Dictionary<ulong, IGuildUser>> LoadAllUsers(SocketGuild guild)
+2 -4
View File
@@ -1,7 +1,5 @@
using CodexContractsPlugin.Marketplace;
using DiscordRewards;
using DiscordRewards;
using Logging;
using Newtonsoft.Json;
using System.Net.Http.Json;
namespace TestNetRewarder
@@ -26,7 +24,7 @@ namespace TestNetRewarder
public async Task<bool> SendRewards(GiveRewardsCommand command)
{
if (command == null || command.Rewards == null || !command.Rewards.Any()) return false;
if (command == null) return false;
var result = await HttpPostJson(command);
log.Log("Reward response: " + result);
return result == "OK";
+59 -15
View File
@@ -1,5 +1,6 @@
using CodexContractsPlugin;
using CodexContractsPlugin.Marketplace;
using NethereumWorkflow.BlockUtils;
using Newtonsoft.Json;
using Utils;
@@ -8,9 +9,34 @@ namespace TestNetRewarder
public class ChainState
{
private readonly HistoricState historicState;
private readonly string[] colorIcons = new[]
{
"🔴",
"🟠",
"🟡",
"🟢",
"🔵",
"🟣",
"🟤",
"⚫",
"⚪",
"🟥",
"🟧",
"🟨",
"🟩",
"🟦",
"🟪",
"🟫",
"⬛",
"⬜",
"🔶",
"🔷"
};
public ChainState(HistoricState historicState, ICodexContracts contracts, BlockInterval blockRange)
{
this.historicState = historicState;
NewRequests = contracts.GetStorageRequests(blockRange);
historicState.CleanUpOldRequests();
historicState.ProcessNewRequests(NewRequests);
@@ -18,19 +44,16 @@ namespace TestNetRewarder
StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray();
FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFinished).ToArray();
ChangedRequests = historicState.StorageRequests.Where(r => r.RecentlyChanged).ToArray();
RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange);
RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange);
SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange);
SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange);
this.historicState = historicState;
}
public Request[] NewRequests { get; }
public StorageRequest[] AllRequests => historicState.StorageRequests;
public StorageRequest[] StartedRequests { get; private set; }
public StorageRequest[] FinishedRequests { get; private set; }
public StorageRequest[] ChangedRequests { get; private set; }
public RequestFulfilledEventDTO[] RequestFulfilledEvents { get; }
public RequestCancelledEventDTO[] RequestCancelledEvents { get; }
public SlotFilledEventDTO[] SlotFilledEvents { get; }
@@ -40,57 +63,78 @@ namespace TestNetRewarder
{
var entries = new List<StringBlockNumberPair>();
entries.AddRange(ChangedRequests.Select(ToPair));
entries.AddRange(NewRequests.Select(ToPair));
entries.AddRange(RequestFulfilledEvents.Select(ToPair));
entries.AddRange(RequestCancelledEvents.Select(ToPair));
entries.AddRange(SlotFilledEvents.Select(ToPair));
entries.AddRange(SlotFreedEvents.Select(ToPair));
entries.AddRange(FinishedRequests.Select(ToPair));
entries.Sort(new StringUtcComparer());
return entries.Select(ToLine).ToArray();
}
private StringBlockNumberPair ToPair(StorageRequest r)
private StringBlockNumberPair ToPair(Request r)
{
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.Request.BlockNumber);
return new StringBlockNumberPair("NewRequest", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
}
public StringBlockNumberPair ToPair(StorageRequest r)
{
return new StringBlockNumberPair("FinishedRequest", JsonConvert.SerializeObject(r), r.Request.Block, r.Request.RequestId);
}
private StringBlockNumberPair ToPair(RequestFulfilledEventDTO r)
{
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
return new StringBlockNumberPair("Fulfilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
}
private StringBlockNumberPair ToPair(RequestCancelledEventDTO r)
{
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
return new StringBlockNumberPair("Cancelled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
}
private StringBlockNumberPair ToPair(SlotFilledEventDTO r)
{
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
return new StringBlockNumberPair("SlotFilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
}
private StringBlockNumberPair ToPair(SlotFreedEventDTO r)
{
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
return new StringBlockNumberPair("SlotFreed", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
}
private string ToLine(StringBlockNumberPair pair)
{
return $"[{pair.Number}] {pair.Str}";
var nl = Environment.NewLine;
var colorIcon = GetColorIcon(pair.RequestId);
return $"{colorIcon} {pair.Block} ({pair.Name}){nl}" +
$"```json{nl}" +
$"{pair.Str}{nl}" +
$"```";
}
private string GetColorIcon(byte[] requestId)
{
var index = requestId[0] % colorIcons.Length;
return colorIcons[index];
}
public class StringBlockNumberPair
{
public StringBlockNumberPair(string str, ulong number)
public StringBlockNumberPair(string name, string str, BlockTimeEntry block, byte[] requestId)
{
Name = name;
Str = str;
Number = number;
Block = block;
RequestId = requestId;
}
public string Name { get; }
public string Str { get; }
public ulong Number { get; }
public BlockTimeEntry Block { get; }
public byte[] RequestId { get; }
}
public class StringUtcComparer : IComparer<StringBlockNumberPair>
@@ -100,7 +144,7 @@ namespace TestNetRewarder
if (x == null && y == null) return 0;
if (x == null) return 1;
if (y == null) return -1;
return x.Number.CompareTo(y.Number);
return x.Block.BlockNumber.CompareTo(y.Block.BlockNumber);
}
}
}
-3
View File
@@ -19,9 +19,6 @@ namespace TestNetRewarder
[Uniform("check-history", "ch", "CHECKHISTORY", true, "Unix epoc timestamp of a moment in history on which processing begins. Required for hosting rewards. Should be 'launch of the testnet'.")]
public int CheckHistoryTimestamp { get; set; } = 0;
[Uniform("market-insights", "mi", "MARKETINSIGHTS", false, "Semi-colon separated integers. Each represents a multiple of intervals, for which a market insights average will be generated.")]
public string MarketInsights { get; set; } = "1;96";
[Uniform("events-overview", "eo", "EVENTSOVERVIEW", false, "When greater than zero, chain event summary will be generated. (default 1)")]
public int CreateChainEventsOverview { get; set; } = 1;
-24
View File
@@ -28,8 +28,6 @@ namespace TestNetRewarder
r.State == RequestState.Finished ||
r.State == RequestState.Failed
);
foreach (var r in storageRequests) r.IsNew = false;
}
}
@@ -39,13 +37,11 @@ namespace TestNetRewarder
{
Request = request;
Hosts = Array.Empty<EthAddress>();
IsNew = true;
}
public Request Request { get; }
public EthAddress[] Hosts { get; private set; }
public RequestState State { get; private set; }
public bool IsNew { get; set; }
[JsonIgnore]
public bool RecentlyStarted { get; private set; }
@@ -53,9 +49,6 @@ namespace TestNetRewarder
[JsonIgnore]
public bool RecentlyFinished { get; private set; }
[JsonIgnore]
public bool RecentlyChanged { get; private set; }
public void Update(ICodexContracts contracts)
{
var newHosts = GetHosts(contracts);
@@ -70,27 +63,10 @@ namespace TestNetRewarder
State == RequestState.Started &&
newState == RequestState.Finished;
RecentlyChanged =
IsNew ||
State != newState ||
HostsChanged(newHosts);
State = newState;
Hosts = newHosts;
}
private bool HostsChanged(EthAddress[] newHosts)
{
if (newHosts.Length != Hosts.Length) return true;
foreach (var newHost in newHosts)
{
if (!Hosts.Contains(newHost)) return true;
}
return false;
}
private EthAddress[] GetHosts(ICodexContracts contracts)
{
var result = new List<EthAddress>();
+48 -75
View File
@@ -6,65 +6,47 @@ namespace TestNetRewarder
{
public class MarketTracker
{
private readonly List<ChainState> buffer = new List<ChainState>();
private readonly MarketAverage MostRecent = new MarketAverage
{
Title = "Most recent"
};
private readonly MarketAverage Irf = new MarketAverage
{
Title = "Recent average"
};
public MarketAverage[] ProcessChainState(ChainState chainState)
{
var intervalCounts = GetInsightCounts();
if (!intervalCounts.Any()) return Array.Empty<MarketAverage>();
UpdateMostRecent(chainState);
UpdateIrf(chainState);
UpdateBuffer(chainState, intervalCounts.Max());
var result = intervalCounts
.Select(GenerateMarketAverage)
.Where(a => a != null)
.Cast<MarketAverage>()
.ToArray();
if (!result.Any()) result = Array.Empty<MarketAverage>();
return result;
}
private void UpdateBuffer(ChainState chainState, int maxNumberOfIntervals)
{
buffer.Add(chainState);
while (buffer.Count > maxNumberOfIntervals)
return new[]
{
buffer.RemoveAt(0);
}
MostRecent,
Irf
};
}
private MarketAverage? GenerateMarketAverage(int numberOfIntervals)
private void UpdateIrf(ChainState chainState)
{
var states = SelectStates(numberOfIntervals);
return CreateAverage(states);
if (!chainState.FinishedRequests.Any()) return;
MostRecent.Price = GetIrf(MostRecent.Price, chainState, s => s.Request.Ask.Reward);
MostRecent.Duration = GetIrf(MostRecent.Duration, chainState, s => s.Request.Ask.Duration);
MostRecent.Size = GetIrf(MostRecent.Size, chainState, s => GetTotalSize(s.Request.Ask));
MostRecent.Collateral = GetIrf(MostRecent.Collateral, chainState, s => s.Request.Ask.Collateral);
MostRecent.ProofProbability = GetIrf(MostRecent.ProofProbability, chainState, s => s.Request.Ask.ProofProbability);
}
private ChainState[] SelectStates(int numberOfIntervals)
private void UpdateMostRecent(ChainState chainState)
{
if (numberOfIntervals < 1) return Array.Empty<ChainState>();
return buffer.TakeLast(numberOfIntervals).ToArray();
}
if (!chainState.FinishedRequests.Any()) return;
private MarketAverage? CreateAverage(ChainState[] states)
{
try
{
return new MarketAverage
{
NumberOfFinished = CountNumberOfFinishedRequests(states),
TimeRange = GetTotalTimeRange(states),
Price = Average(states, s => s.Request.Ask.Reward),
Duration = Average(states, s => s.Request.Ask.Duration),
Size = Average(states, s => GetTotalSize(s.Request.Ask)),
Collateral = Average(states, s => s.Request.Ask.Collateral),
ProofProbability = Average(states, s => s.Request.Ask.ProofProbability)
};
}
catch (Exception ex)
{
Program.Log.Error($"Exception in CreateAverage: {ex}");
return null;
}
MostRecent.Price = Average(chainState, s => s.Request.Ask.Reward);
MostRecent.Duration = Average(chainState, s => s.Request.Ask.Duration);
MostRecent.Size = Average(chainState, s => GetTotalSize(s.Request.Ask));
MostRecent.Collateral = Average(chainState, s => s.Request.Ask.Collateral);
MostRecent.ProofProbability = Average(chainState, s => s.Request.Ask.ProofProbability);
}
private int GetTotalSize(Ask ask)
@@ -74,49 +56,40 @@ namespace TestNetRewarder
return nSlots * slotSize;
}
private float Average(ChainState[] states, Func<StorageRequest, BigInteger> getValue)
private float Average(ChainState state, Func<StorageRequest, BigInteger> getValue)
{
return Average(states, s => Convert.ToInt32(getValue(s)));
return Average(state, s => Convert.ToInt32(getValue(s)));
}
private float Average(ChainState[] states, Func<StorageRequest, int> getValue)
private float GetIrf(float current, ChainState state, Func<StorageRequest, BigInteger> getValue)
{
return GetIrf(current, state, s => Convert.ToInt32(getValue(s)));
}
private float Average(ChainState state, Func<StorageRequest, int> getValue)
{
var sum = 0.0f;
var count = 0.0f;
foreach (var state in states)
foreach (var finishedRequest in state.FinishedRequests)
{
foreach (var finishedRequest in state.FinishedRequests)
{
sum += getValue(finishedRequest);
count++;
}
sum += getValue(finishedRequest);
count++;
}
if (count < 1.0f) return 0.0f;
return sum / count;
}
private TimeSpan GetTotalTimeRange(ChainState[] states)
private float GetIrf(float current, ChainState state, Func<StorageRequest, int> getValue)
{
return Program.Config.Interval * states.Length;
}
private int CountNumberOfFinishedRequests(ChainState[] states)
{
return states.Sum(s => s.FinishedRequests.Length);
}
private int[] GetInsightCounts()
{
try
var result = current;
foreach (var finishedRequest in state.FinishedRequests)
{
var tokens = Program.Config.MarketInsights.Split(';').ToArray();
return tokens.Select(t => Convert.ToInt32(t)).ToArray();
float v = getValue(finishedRequest);
result = (result + v) / 2.0f;
}
catch (Exception ex)
{
Program.Log.Error($"Exception when parsing MarketInsights config parameters: {ex}");
}
return Array.Empty<int>();
return result;
}
}
}
+4 -2
View File
@@ -67,9 +67,11 @@ namespace TestNetRewarder
var marketAverages = GetMarketAverages(chainState);
var eventsOverview = GenerateEventsOverview(chainState);
log.Log($"Found {outgoingRewards.Count} rewards to send. Found {marketAverages.Length} market averages.");
log.Log($"Found {outgoingRewards.Count} rewards. " +
$"Found {marketAverages.Length} market averages. " +
$"Found {eventsOverview.Length} events.");
if (outgoingRewards.Any())
if (outgoingRewards.Any() || marketAverages.Any() || eventsOverview.Any())
{
if (!await SendRewardsCommand(outgoingRewards, marketAverages, eventsOverview))
{