codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Debugging time segmenter

This commit is contained in:
benbierens 2024-02-19 14:56:49 +01:00
parent 3c210f96fc
commit da4101a042
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
14 changed files with 161 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
Framework/NethereumWorkflow/BlockTimeFinder.cs
View File

@ -52,7 +52,13 @@ namespace NethereumWorkflow
return closestBefore.BlockNumber;
}
FetchBlocksAround(moment);
var newBlocks = FetchBlocksAround(moment);
if (newBlocks == 0)
{
log.Log("Didn't find any new blocks.");
if (closestBefore != null) return closestBefore.BlockNumber;
throw new Exception("Failed to find highest before.");
}
return GetHighestBlockBefore(moment);
}
@ -71,11 +77,17 @@ namespace NethereumWorkflow
return closestAfter.BlockNumber;
}
FetchBlocksAround(moment);
var newBlocks = FetchBlocksAround(moment);
if (newBlocks == 0)
{
log.Log("Didn't find any new blocks.");
if (closestAfter != null) return closestAfter.BlockNumber;
throw new Exception("Failed to find lowest before.");
}
return GetLowestBlockAfter(moment);
}
private void FetchBlocksAround(DateTime moment)
private int FetchBlocksAround(DateTime moment)
{
var timePerBlock = EstimateTimePerBlock();
log.Debug("Fetching blocks around " + moment.ToString("o") + " timePerBlock: " + timePerBlock.TotalSeconds);
@ -85,42 +97,55 @@ namespace NethereumWorkflow
var max = entries.Keys.Max();
var blockDifference = CalculateBlockDifference(moment, timePerBlock, max);
FetchUp(max, blockDifference);
FetchDown(max, blockDifference);
return
FetchUp(max, blockDifference) +
FetchDown(max, blockDifference);
}
private void FetchDown(ulong max, ulong blockDifference)
private int FetchDown(ulong max, ulong blockDifference)
{
var target = max - blockDifference - 1;
var target = GetTarget(max, blockDifference);
var fetchDown = FetchRange;
var newBlocks = 0;
while (fetchDown > 0)
{
if (!entries.ContainsKey(target))
{
var newBlock = AddBlockNumber(target);
if (newBlock == null) return;
var newBlock = AddBlockNumber("FD" + fetchDown, target);
if (newBlock == null) return newBlocks;
newBlocks++;
fetchDown--;
}
target--;
if (target <= 0) return;
if (target <= 0) return newBlocks;
}
return newBlocks;
}
private void FetchUp(ulong max, ulong blockDifference)
private int FetchUp(ulong max, ulong blockDifference)
{
var target = max - blockDifference;
var target = GetTarget(max, blockDifference);
var fetchUp = FetchRange;
var newBlocks = 0;
while (fetchUp > 0)
{
if (!entries.ContainsKey(target))
{
var newBlock = AddBlockNumber(target);
if (newBlock == null) return;
var newBlock = AddBlockNumber("FU" + fetchUp, target);
if (newBlock == null) return newBlocks;
newBlocks++;
fetchUp--;
}
target++;
if (target >= max) return;
if (target >= max) return newBlocks;
}
return newBlocks;
}
private ulong GetTarget(ulong max, ulong blockDifference)
{
if (max <= blockDifference) return 1;
return max - blockDifference;
}
private ulong CalculateBlockDifference(DateTime moment, TimeSpan timePerBlock, ulong max)
@ -155,13 +180,14 @@ namespace NethereumWorkflow
}
}
private BlockTimeEntry? AddBlockNumber(decimal blockNumber)
private BlockTimeEntry? AddBlockNumber(string a, decimal blockNumber)
{
return AddBlockNumber(Convert.ToUInt64(blockNumber));
return AddBlockNumber(a, Convert.ToUInt64(blockNumber));
}
private BlockTimeEntry? AddBlockNumber(ulong blockNumber)
private BlockTimeEntry? AddBlockNumber(string a, ulong blockNumber)
{
log.Log(a + " - Adding blockNumber: " + blockNumber);
if (entries.ContainsKey(blockNumber))
{
return entries[blockNumber];
@ -190,9 +216,10 @@ namespace NethereumWorkflow
{
var min = entries.Keys.Min();
var max = entries.Keys.Max();
log.Log("min/max: " + min + " / " + max);
var clippedMin = Math.Max(max - 100, min);
var minTime = entries[min].Utc;
var clippedMinBlock = AddBlockNumber(clippedMin);
var clippedMinBlock = AddBlockNumber("EST", clippedMin);
if (clippedMinBlock != null) minTime = clippedMinBlock.Utc;
var maxTime = entries[max].Utc;
@ -212,7 +239,7 @@ namespace NethereumWorkflow
if (!entries.Any())
{
AddCurrentBlock();
AddBlockNumber(entries.Single().Key - 1);
AddBlockNumber("INIT", entries.Single().Key - 1);
}
}
@ -225,7 +252,7 @@ namespace NethereumWorkflow
{
var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
var blockNumber = number.ToDecimal();
return AddBlockNumber(blockNumber);
return AddBlockNumber("CUR", blockNumber);
}
private DateTime? GetTimestampFromBlock(ulong blockNumber)
@ -238,7 +265,7 @@ namespace NethereumWorkflow
}
catch (Exception ex)
{
int i = 0;
log.Error(nameof(GetTimestampFromBlock) + " Exception: " + ex);
throw;
}
}

9
Framework/NethereumWorkflow/NethereumInteraction.cs
View File

@ -3,7 +3,6 @@ using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.Contracts;
using Nethereum.RPC.Eth.DTOs;
using Nethereum.Web3;
using System.Runtime.CompilerServices;
using Utils;
namespace NethereumWorkflow
@ -90,14 +89,18 @@ namespace NethereumWorkflow
{
var blockTimeFinder = new BlockTimeFinder(web3, log);
var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
var lowest = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
var highest = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
var fromBlock = Math.Min(lowest, highest);
var toBlock = Math.Max(lowest, highest);
return GetEvents<TEvent>(address, fromBlock, toBlock);
}
public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
{
log.Debug($"Getting events of type [{typeof(TEvent).Name}] in block range [{fromBlockNumber} - {toBlockNumber}]");
var eventHandler = web3.Eth.GetEvent<TEvent>(address);
var from = new BlockParameter(fromBlockNumber);
var to = new BlockParameter(toBlockNumber);

2
Tools/BiblioTech/BiblioTech.csproj
View File

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<OutputType>Exe</OutputType>

6
Tools/BiblioTech/CommandHandler.cs
View File

@ -26,12 +26,10 @@ namespace BiblioTech
Program.AdminChecker.SetGuild(guild);
Program.Log.Log($"Initializing for guild: '{guild.Name}'");
var roleController = new RoleController(client);
var rewardsApi = new RewardsApi(roleController);
var adminChannels = guild.TextChannels.Where(Program.AdminChecker.IsAdminChannel).ToArray();
if (adminChannels == null || !adminChannels.Any()) throw new Exception("No admin message channel");
Program.AdminChecker.SetAdminChannel(adminChannels.First());
Program.RoleDriver = new RoleDriver(client);
var builders = commands.Select(c =>
{
@ -62,8 +60,6 @@ namespace BiblioTech
var json = JsonConvert.SerializeObject(exception.Errors, Formatting.Indented);
Program.Log.Error(json);
}
rewardsApi.Start();
}
private async Task SlashCommandHandler(SocketSlashCommand command)

17
Tools/BiblioTech/Program.cs
View File

@ -1,5 +1,6 @@
using ArgsUniform;
using BiblioTech.Commands;
using BiblioTech.Rewards;
using Discord;
using Discord.WebSocket;
using Logging;
@ -13,6 +14,7 @@ namespace BiblioTech
public static Configuration Config { get; private set; } = null!;
public static UserRepo UserRepo { get; } = new UserRepo();
public static AdminChecker AdminChecker { get; private set; } = null!;
public static IDiscordRoleDriver RoleDriver { get; set; } = null!;
public static ILog Log { get; private set; } = null!;
public static Task Main(string[] args)
@ -29,10 +31,10 @@ namespace BiblioTech
EnsurePath(Config.UserDataPath);
EnsurePath(Config.EndpointsPath);
return new Program().MainAsync();
return new Program().MainAsync(args);
}
public async Task MainAsync()
public async Task MainAsync(string[] args)
{
Log.Log("Starting Codex Discord Bot...");
client = new DiscordSocketClient();
@ -52,10 +54,19 @@ namespace BiblioTech
await client.LoginAsync(TokenType.Bot, Config.ApplicationToken);
await client.StartAsync();
AdminChecker = new AdminChecker();
var builder = WebApplication.CreateBuilder(args);
builder.WebHost.ConfigureKestrel((context, options) =>
{
options.ListenAnyIP(Config.RewardApiPort);
});
builder.Services.AddControllers();
var app = builder.Build();
app.MapControllers();
Log.Log("Running...");
await app.RunAsync();
await Task.Delay(-1);
}

12
Tools/BiblioTech/Properties/launchSettings.json Normal file
View File

@ -0,0 +1,12 @@
{
"profiles": {
"BiblioTech": {
"commandName": "Project",
"launchBrowser": true,
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"applicationUrl": "https://localhost:52960;http://localhost:52961"
}
}
}

35
Tools/BiblioTech/Rewards/RewardController.cs Normal file
View File

@ -0,0 +1,35 @@
using DiscordRewards;
using Microsoft.AspNetCore.Mvc;
namespace BiblioTech.Rewards
{
public interface IDiscordRoleDriver
{
Task GiveRewards(GiveRewardsCommand rewards);
}
[Route("api/[controller]")]
[ApiController]
public class RewardController : ControllerBase
{
[HttpGet]
public string Ping()
{
return "Pong";
}
[HttpPost]
public async Task<string> Give(GiveRewardsCommand cmd)
{
try
{
await Program.RoleDriver.GiveRewards(cmd);
}
catch (Exception ex)
{
Program.Log.Error("Exception: " + ex);
}
return "OK";
}
}
}

91
Tools/BiblioTech/Rewards/RewardsApi.cs
View File

@ -1,91 +0,0 @@
using DiscordRewards;
using Newtonsoft.Json;
using System.Net;
using TaskFactory = Utils.TaskFactory;
namespace BiblioTech.Rewards
{
public interface IDiscordRoleController
{
Task GiveRewards(GiveRewardsCommand rewards);
}
public class RewardsApi
{
private readonly HttpListener listener = new HttpListener();
private readonly TaskFactory taskFactory = new TaskFactory();
private readonly IDiscordRoleController roleController;
private CancellationTokenSource cts = new CancellationTokenSource();
public RewardsApi(IDiscordRoleController roleController)
{
this.roleController = roleController;
}
public void Start()
{
cts = new CancellationTokenSource();
var uri = $"http://*:{Program.Config.RewardApiPort}/";
listener.Prefixes.Add(uri);
listener.Start();
taskFactory.Run(ConnectionDispatcher, nameof(ConnectionDispatcher));
Program.Log.Log($"Reward API listening on '{uri}'");
}
public void Stop()
{
listener.Stop();
cts.Cancel();
taskFactory.WaitAll();
}
private void ConnectionDispatcher()
{
while (!cts.Token.IsCancellationRequested)
{
var wait = listener.GetContextAsync();
wait.Wait(cts.Token);
if (wait.IsCompletedSuccessfully)
{
taskFactory.Run(() =>
{
var context = wait.Result;
try
{
HandleConnection(context).Wait();
}
catch (Exception ex)
{
Program.Log.Error("Exception during HTTP handler: " + ex);
}
}, nameof(HandleConnection));
}
}
}
private async Task HandleConnection(HttpListenerContext context)
{
using var reader = new StreamReader(context.Request.InputStream);
var content = reader.ReadToEnd();
if (content == "Ping")
{
using var writer = new StreamWriter(context.Response.OutputStream);
writer.Write("Pong");
return;
}
if (!content.StartsWith("{")) return;
var rewards = JsonConvert.DeserializeObject<GiveRewardsCommand>(content);
if (rewards != null)
{
await ProcessRewards(rewards);
}
}
private async Task ProcessRewards(GiveRewardsCommand rewards)
{
await roleController.GiveRewards(rewards);
}
}
}

12
Tools/BiblioTech/Rewards/RoleController.cs → Tools/BiblioTech/Rewards/RoleDriver.cs
View File

@ -4,13 +4,13 @@ using DiscordRewards;
namespace BiblioTech.Rewards
{
public class RoleController : IDiscordRoleController
public class RoleDriver : IDiscordRoleDriver
{
private readonly DiscordSocketClient client;
private readonly SocketTextChannel? rewardsChannel;
private readonly RewardRepo repo = new RewardRepo();
public RoleController(DiscordSocketClient client)
public RoleDriver(DiscordSocketClient client)
{
this.client = client;
@ -107,7 +107,13 @@ namespace BiblioTech.Rewards
private SocketGuild GetGuild()
{
return client.Guilds.Single(g => g.Name == Program.Config.ServerName);
var guild = client.Guilds.SingleOrDefault(g => g.Name == Program.Config.ServerName);
if (guild == null)
{
throw new Exception($"Unable to find guild by name: '{Program.Config.ServerName}'. " +
$"Known guilds: [{string.Join(",", client.Guilds.Select(g => g.Name))}]");
}
return guild;
}
}

30
Tools/TestNetRewarder/BotClient.cs
View File

@ -1,4 +1,5 @@
using DiscordRewards;
using CodexContractsPlugin.Marketplace;
using DiscordRewards;
using Logging;
using Newtonsoft.Json;
@ -17,13 +18,30 @@ namespace TestNetRewarder
public async Task<bool> IsOnline()
{
return await HttpPost("Ping") == "Ping";
var result = await HttpGet();
log.Log("Is DiscordBot online: " + result);
return result == "Pong";
}
public async Task SendRewards(GiveRewardsCommand command)
public async Task<bool> SendRewards(GiveRewardsCommand command)
{
if (command == null || command.Rewards == null || !command.Rewards.Any()) return;
await HttpPost(JsonConvert.SerializeObject(command));
if (command == null || command.Rewards == null || !command.Rewards.Any()) return false;
return await HttpPost(JsonConvert.SerializeObject(command)) == "OK";
}
private async Task<string> HttpGet()
{
try
{
var client = new HttpClient();
var response = await client.GetAsync(GetUrl());
return await response.Content.ReadAsStringAsync();
}
catch (Exception ex)
{
log.Error(ex.ToString());
return string.Empty;
}
}
private async Task<string> HttpPost(string content)
@ -43,7 +61,7 @@ namespace TestNetRewarder
private string GetUrl()
{
return $"{configuration.DiscordHost}:{configuration.DiscordPort}";
return $"{configuration.DiscordHost}:{configuration.DiscordPort}/api/reward";
}
}
}

9
Tools/TestNetRewarder/Processor.cs
View File

@ -44,11 +44,14 @@ namespace TestNetRewarder
if (outgoingRewards.Any())
{
await SendRewardsCommand(outgoingRewards);
if (!await SendRewardsCommand(outgoingRewards))
{
log.Error("Failed to send reward command.");
}
}
}
private async Task SendRewardsCommand(List<RewardUsersCommand> outgoingRewards)
private async Task<bool> SendRewardsCommand(List<RewardUsersCommand> outgoingRewards)
{
var cmd = new GiveRewardsCommand
{
@ -56,7 +59,7 @@ namespace TestNetRewarder
};
log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd));
await Program.BotClient.SendRewards(cmd);
return await Program.BotClient.SendRewards(cmd);
}
private void ProcessReward(List<RewardUsersCommand> outgoingRewards, RewardConfig reward, ChainState chainState)

1
Tools/TestNetRewarder/Program.cs
View File

@ -59,6 +59,7 @@ namespace TestNetRewarder
var blockNumber = gc.GethNode.GetSyncedBlockNumber();
if (blockNumber == null || blockNumber < 1) throw new Exception("Geth connection failed.");
Log.Log("Geth OK. Block number: " + blockNumber);
}
private static async Task EnsureBotOnline()

1
Tools/TestNetRewarder/TimeSegmenter.cs
View File

@ -33,6 +33,7 @@ namespace TestNetRewarder
// Wait for the entire time segment to be in the past.
var delay = (end - now).Add(TimeSpan.FromSeconds(3));
waited = true;
log.Log($"Waiting till time segment is in the past... {Time.FormatDuration(delay)}");
await Task.Delay(delay, Program.CancellationToken);
}

2
Tools/TestNetRewarder/build-docker.bat Normal file
View File

@ -0,0 +1,2 @@
docker build -f docker/Dockerfile -t thatbenbierens/codex-rewardbot:initial ../..
docker push thatbenbierens/codex-rewardbot:initial