Compare commits

..
Author SHA1 Message Date
benbierens 77cdd3e2d8 Merge branch 'master' into feature/waku-plugin
# Conflicts:
#	Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs
#	cs-codex-dist-testing.sln
2024-04-09 08:19:34 +02:00
benbierens 3776f46c02 Setting up basic test for waku 2023-09-25 15:43:16 +02:00
benbierens 12f6710a56 Bootstrapping waku nodes 2023-09-25 15:14:51 +02:00
benbierens 30ba382db7 Can start waku node 2023-09-25 13:02:44 +02:00
benbierens ab4f4695cb Setup waku plugin and test 2023-09-25 10:16:34 +02:00
41 changed files with 565 additions and 335 deletions
@@ -15,7 +15,8 @@
public class MarketAverage
{
public string Title { get; set; } = string.Empty;
public int NumberOfFinished { get; set; }
public TimeSpan TimeRange { get; set; }
public float Price { get; set; }
public float Size { get; set; }
public float Duration { get; set; }
@@ -43,11 +43,6 @@ namespace KubernetesWorkflow
return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
}
public void WaitUntilOnline(RunningContainer container)
{
WaitUntilDeploymentOnline(container.Recipe.Name);
}
public PodInfo GetPodInfo(RunningDeployment deployment)
{
var pod = GetPodForDeployment(deployment);
@@ -377,6 +372,7 @@ namespace KubernetesWorkflow
};
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
var name = deploymentSpec.Metadata.Name;
return new RunningDeployment(name, podLabel);
@@ -73,6 +73,13 @@ namespace KubernetesWorkflow.Recipe
return p;
}
protected Port AddInternalPort(int number, string tag = "", PortProtocol protocol = PortProtocol.TCP)
{
var p = factory.CreateInternalPort(number, tag, protocol);
internalPorts.Add(p);
return p;
}
protected void AddExposedPortAndVar(string name, string tag, PortProtocol protocol = PortProtocol.TCP)
{
AddEnvVar(name, AddExposedPort(tag, protocol));
@@ -16,7 +16,12 @@ namespace KubernetesWorkflow.Recipe
public Port CreateInternalPort(string tag, PortProtocol protocol)
{
return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
return CreateInternalPort(internalNumberSource.GetNextNumber(), tag, protocol);
}
public Port CreateInternalPort(int number, string tag, PortProtocol protocol)
{
return new Port(number, tag, protocol);
}
public Port CreateExternalPort(int number, string tag, PortProtocol protocol)
@@ -9,8 +9,8 @@ namespace KubernetesWorkflow
public interface IStartupWorkflow
{
IKnownLocations GetAvailableLocations();
FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
PodInfo GetPodInfo(RunningContainer container);
PodInfo GetPodInfo(RunningContainers containers);
CrashWatcher CreateCrashWatcher(RunningContainer container);
@@ -45,12 +45,12 @@ namespace KubernetesWorkflow
return locationProvider.GetAvailableLocations();
}
public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
{
return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
}
public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
{
return K8s(controller =>
{
@@ -67,18 +67,7 @@ namespace KubernetesWorkflow
{
componentFactory.Update(controller);
}
return new FutureContainers(rc, this);
});
}
public void WaitUntilOnline(RunningContainers rc)
{
K8s(controller =>
{
foreach (var c in rc.Containers)
{
controller.WaitUntilOnline(c);
}
return rc;
});
}
@@ -1,20 +0,0 @@
namespace KubernetesWorkflow.Types
{
public class FutureContainers
{
private readonly RunningContainers runningContainers;
private readonly StartupWorkflow workflow;
public FutureContainers(RunningContainers runningContainers, StartupWorkflow workflow)
{
this.runningContainers = runningContainers;
this.workflow = workflow;
}
public RunningContainers WaitForOnline()
{
workflow.WaitUntilOnline(runningContainers);
return runningContainers;
}
}
}
@@ -18,13 +18,6 @@ namespace NethereumWorkflow.BlockUtils
bounds = new BlockchainBounds(cache, web3);
}
public BlockTimeEntry Get(ulong blockNumber)
{
var b = cache.Get(blockNumber);
if (b != null) return b;
return GetBlock(blockNumber);
}
public ulong? GetHighestBlockNumberBefore(DateTime moment)
{
bounds.Initialize();
@@ -121,12 +121,5 @@ namespace NethereumWorkflow
to: toBlock.Value
);
}
public BlockTimeEntry GetBlockForNumber(ulong number)
{
var wrapper = new Web3Wrapper(web3, log);
var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
return blockTimeFinder.Get(number);
}
}
}
@@ -5,9 +5,6 @@ using Nethereum.ABI;
using Nethereum.Hex.HexTypes;
using Nethereum.Util;
using NethereumWorkflow;
using NethereumWorkflow.BlockUtils;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Utils;
namespace CodexContractsPlugin
@@ -31,7 +28,6 @@ namespace CodexContractsPlugin
SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange);
}
[JsonConverter(typeof(StringEnumConverter))]
public enum RequestState
{
New,
@@ -90,7 +86,7 @@ namespace CodexContractsPlugin
{
var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
var result = requestEvent.ReturnValue1;
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
result.BlockNumber = e.Log.BlockNumber.ToUlong();
result.RequestId = e.Event.RequestId;
return result;
})
@@ -103,7 +99,7 @@ namespace CodexContractsPlugin
return events.Select(e =>
{
var result = e.Event;
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
result.BlockNumber = e.Log.BlockNumber.ToUlong();
return result;
}).ToArray();
}
@@ -114,7 +110,7 @@ namespace CodexContractsPlugin
return events.Select(e =>
{
var result = e.Event;
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
result.BlockNumber = e.Log.BlockNumber.ToUlong();
return result;
}).ToArray();
}
@@ -125,7 +121,7 @@ namespace CodexContractsPlugin
return events.Select(e =>
{
var result = e.Event;
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
result.BlockNumber = e.Log.BlockNumber.ToUlong();
result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
return result;
}).ToArray();
@@ -137,7 +133,7 @@ namespace CodexContractsPlugin
return events.Select(e =>
{
var result = e.Event;
result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
result.BlockNumber = e.Log.BlockNumber.ToUlong();
return result;
}).ToArray();
}
@@ -170,11 +166,6 @@ namespace CodexContractsPlugin
return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
}
private BlockTimeEntry GetBlock(ulong number)
{
return gethNode.GetBlockForNumber(number);
}
private EthAddress GetEthAddressFromTransaction(string transactionHash)
{
var transaction = gethNode.GetTransaction(transactionHash);
@@ -24,7 +24,7 @@ namespace CodexContractsPlugin
var startupConfig = CreateStartupConfig(gethNode);
startupConfig.NameOverride = "codex-contracts";
var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline();
var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig);
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
var container = containers.Containers[0];
@@ -1,5 +1,4 @@
using CodexContractsPlugin.Marketplace;
using KubernetesWorkflow;
using KubernetesWorkflow;
using KubernetesWorkflow.Types;
using Logging;
using Newtonsoft.Json;
@@ -54,18 +53,7 @@ namespace CodexContractsPlugin
var artifact = JObject.Parse(json);
var abi = artifact["abi"];
var byteCode = artifact["bytecode"];
var abiResult = abi!.ToString(Formatting.None);
var byteCodeResult = byteCode!.ToString(Formatting.None);
if (byteCodeResult
.ToLowerInvariant()
.Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
{
throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
}
return abiResult;
return abi!.ToString(Formatting.None);
}
private static string Retry(Func<string> fetch)
@@ -1,6 +1,5 @@
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
using GethPlugin;
using NethereumWorkflow.BlockUtils;
using Newtonsoft.Json;
namespace CodexContractsPlugin.Marketplace
@@ -8,7 +7,7 @@ namespace CodexContractsPlugin.Marketplace
public partial class Request : RequestBase
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
public ulong BlockNumber { get; set; }
public byte[] RequestId { get; set; }
public EthAddress ClientAddress { get { return new EthAddress(Client); } }
@@ -17,26 +16,26 @@ namespace CodexContractsPlugin.Marketplace
public partial class RequestFulfilledEventDTO
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
public ulong BlockNumber { get; set; }
}
public partial class RequestCancelledEventDTO
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
public ulong BlockNumber { get; set; }
}
public partial class SlotFilledEventDTO
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
public ulong BlockNumber { get; set; }
public EthAddress Host { get; set; }
}
public partial class SlotFreedEventDTO
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
public ulong BlockNumber { get; set; }
}
}
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
File diff suppressed because one or more lines are too long
@@ -46,14 +46,14 @@ namespace CodexDiscordBotPlugin
var startupConfig = new StartupConfig();
startupConfig.NameOverride = config.Name;
startupConfig.Add(config);
return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
}
private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
{
var startupConfig = new StartupConfig();
startupConfig.Add(config);
return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig);
}
}
}
+3 -6
View File
@@ -87,16 +87,13 @@ namespace CodexPlugin
private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
{
var futureContainers = new List<FutureContainers>();
var result = new List<RunningContainers>();
for (var i = 0; i < numberOfNodes; i++)
{
var workflow = pluginTools.CreateWorkflow();
futureContainers.Add(workflow.Start(1, location, recipe, startupConfig));
result.Add(workflow.Start(1, location, recipe, startupConfig));
}
return futureContainers
.Select(f => f.WaitForOnline())
.ToArray();
return result.ToArray();
}
private PodInfo GetPodInfo(RunningContainers rc)
@@ -30,7 +30,7 @@ namespace DeployAndRunPlugin
startupConfig.Add(config);
var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd");
var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline();
var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig);
return containers.Containers.Single();
}
}
-7
View File
@@ -5,7 +5,6 @@ using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.Contracts;
using Nethereum.RPC.Eth.DTOs;
using NethereumWorkflow;
using NethereumWorkflow.BlockUtils;
using Utils;
namespace GethPlugin
@@ -28,7 +27,6 @@ namespace GethPlugin
List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new();
List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
BlockTimeEntry GetBlockForNumber(ulong number);
}
public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -162,11 +160,6 @@ namespace GethPlugin
return StartInteraction().ConvertTimeRangeToBlockRange(timeRange);
}
public BlockTimeEntry GetBlockForNumber(ulong number)
{
return StartInteraction().GetBlockForNumber(number);
}
protected abstract NethereumInteraction StartInteraction();
}
}
+1 -1
View File
@@ -21,7 +21,7 @@ namespace GethPlugin
startupConfig.NameOverride = gethStartupConfig.NameOverride;
var workflow = tools.CreateWorkflow();
var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline();
var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig);
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
var container = containers.Containers[0];
@@ -25,7 +25,7 @@ namespace MetricsPlugin
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
var workflow = tools.CreateWorkflow();
var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
var runningContainers = workflow.Start(1, recipe, startupConfig);
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
Log("Metrics server started.");
@@ -0,0 +1,34 @@
using Core;
using KubernetesWorkflow.Types;
namespace WakuPlugin
{
public static class CoreInterfaceExtensions
{
public static RunningContainers[] DeployWakuNodes(this CoreInterface ci, int number, Action<IWakuSetup> setup)
{
return Plugin(ci).DeployWakuNodes(number, setup);
}
public static IWakuNode WrapWakuContainer(this CoreInterface ci, RunningContainer container)
{
return Plugin(ci).WrapWakuContainer(container);
}
public static IWakuNode StartWakuNode(this CoreInterface ci)
{
return ci.StartWakuNode(s => { });
}
public static IWakuNode StartWakuNode(this CoreInterface ci, Action<IWakuSetup> setup)
{
var rc = ci.DeployWakuNodes(1, setup);
return ci.WrapWakuContainer(rc.First().Containers.First());
}
private static WakuPlugin Plugin(CoreInterface ci)
{
return ci.GetPlugin<WakuPlugin>();
}
}
}
@@ -0,0 +1,8 @@
namespace WakuPlugin
{
public class DebugInfoResponse
{
public string[] listenAddresses { get; set; } = Array.Empty<string>();
public string enrUri { get; set; } = string.Empty;
}
}
@@ -0,0 +1,38 @@
using KubernetesWorkflow;
using KubernetesWorkflow.Recipe;
using Utils;
namespace WakuPlugin
{
public class WakuContainerRecipe : ContainerRecipeFactory
{
public override string AppName => "waku";
//public override string Image => "statusteam/nim-waku:deploy-wakuv2-test";
public override string Image => "thatbenbierens/nim-waku:try";
public static string RestPortTag = "REST_PORT";
protected override void Initialize(StartupConfig startupConfig)
{
var config = startupConfig.Get<WakuSetup>();
SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
AddEnvVar("WAKUNODE2_LOG_LEVEL", "TRACE");
AddEnvVar("WAKUNODE2_REST", "1");
AddExposedPortAndVar("WAKUNODE2_REST_PORT", RestPortTag);
AddEnvVar("WAKUNODE2_REST_ADDRESS", "0.0.0.0");
AddInternalPortAndVar("WAKUNODE2_TCP_PORT");
AddEnvVar("WAKUNODE2_RPC_ADDRESS", "0.0.0.0");
AddEnvVar("WAKUNODE2_DISCV5_DISCOVERY", "1");
AddInternalPortAndVar("WAKUNODE2_DISCV5_UDP_PORT");
AddEnvVar("WAKUNODE2_DISCV5_ENR_AUTO_UPDATEY", "1");
if (!string.IsNullOrEmpty(config.BootstrapEnr))
{
AddEnvVar("WAKUNODE2_DISCV5_BOOTSTRAP_NODE", config.BootstrapEnr);
}
}
}
}
+53
View File
@@ -0,0 +1,53 @@
using Core;
using KubernetesWorkflow.Types;
namespace WakuPlugin
{
public interface IWakuNode : IHasContainer
{
DebugInfoResponse DebugInfo();
void SubscribeToTopic(string topic);
void SendMessage(string topic, string message);
string[] GetMessages(string topic);
}
public class WakuNode : IWakuNode
{
private readonly IPluginTools tools;
public WakuNode(IPluginTools tools, RunningContainer container)
{
this.tools = tools;
Container = container;
}
public RunningContainer Container { get; }
public DebugInfoResponse DebugInfo()
{
return Api().HttpGetJson<DebugInfoResponse>("debug/v1/info");
}
public void SubscribeToTopic(string topic)
{
var response = Api().HttpPostString<string>(route: "relay/v1/subscriptions", body: topic);
}
public void SendMessage(string topic, string message)
{
var response = Api().HttpPostString<string>($"relay/v1/messages/{topic}", message);
}
public string[] GetMessages(string topic)
{
var response = Api().HttpGetString($"relay/v1/messages/{topic}");
return new[] { "" };
}
private IEndpoint Api()
{
var address = Container.GetAddress(tools.GetLog(), WakuContainerRecipe.RestPortTag);
return tools.CreateHttp().CreateEndpoint(address, "", logAlias: "waku");
}
}
}
+71
View File
@@ -0,0 +1,71 @@
using Core;
using KubernetesWorkflow.Types;
namespace WakuPlugin
{
public class WakuPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
{
private readonly IPluginTools tools;
private readonly WakuStarter starter;
public WakuPlugin(IPluginTools tools)
{
this.tools = tools;
starter = new WakuStarter(tools);
}
public string LogPrefix => "(Waku) ";
public void Announce()
{
tools.GetLog().Log($"Loaded with Waku plugin.");
}
public void AddMetadata(IAddMetadata metadata)
{
//metadata.Add("codexid", codexStarter.GetCodexId());
}
public void Decommission()
{
}
public RunningContainers[] DeployWakuNodes(int numberOfNodes, Action<IWakuSetup> setup)
{
return starter.Start(numberOfNodes, setup);
}
public IWakuNode WrapWakuContainer(RunningContainer container)
{
container = SerializeGate.Gate(container);
return starter.Wrap(container);
}
//public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
//{
// containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
// return codexStarter.WrapCodexContainers(coreInterface, containers);
//}
//public void WireUpMarketplace(ICodexNodeGroup result, Action<ICodexSetup> setup)
//{
// var codexSetup = GetSetup(1, setup);
// if (codexSetup.MarketplaceConfig == null) return;
// var mconfig = codexSetup.MarketplaceConfig;
// foreach (var node in result)
// {
// mconfig.GethNode.SendEth(node, mconfig.InitialEth);
// mconfig.CodexContracts.MintTestTokens(mconfig.GethNode, node, mconfig.InitialTokens);
// }
//}
//private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)
//{
// var codexSetup = new CodexSetup(numberOfNodes);
// codexSetup.LogLevel = defaultLogLevel;
// setup(codexSetup);
// return codexSetup;
//}
}
}
@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Framework\Core\Core.csproj" />
</ItemGroup>
</Project>
+26
View File
@@ -0,0 +1,26 @@
namespace WakuPlugin
{
public interface IWakuSetup
{
IWakuSetup WithName(string name);
IWakuSetup WithBootstrapNode(IWakuNode node);
}
public class WakuSetup : IWakuSetup
{
internal string? Name { get; private set; }
internal string? BootstrapEnr { get; private set; }
public IWakuSetup WithName(string name)
{
Name = name;
return this;
}
public IWakuSetup WithBootstrapNode(IWakuNode node)
{
BootstrapEnr = node.DebugInfo().enrUri;
return this;
}
}
}
+45
View File
@@ -0,0 +1,45 @@
using Core;
using KubernetesWorkflow;
using KubernetesWorkflow.Types;
namespace WakuPlugin
{
public class WakuStarter
{
private readonly IPluginTools tools;
public WakuStarter(IPluginTools tools)
{
this.tools = tools;
}
public RunningContainers[] Start(int numberOfNodes, Action<IWakuSetup> setup)
{
var result = new List<RunningContainers>();
var workflow = tools.CreateWorkflow();
var startupConfig = CreateStartupConfig(setup);
for (var i = 0; i < numberOfNodes; i++)
{
result.Add(workflow.Start(1, new WakuContainerRecipe(), startupConfig));
}
return result.ToArray();
}
public IWakuNode Wrap(RunningContainer container)
{
return new WakuNode(tools, container);
}
private StartupConfig CreateStartupConfig(Action<IWakuSetup> setup)
{
var config = new WakuSetup();
setup(config);
var startupConfig = new StartupConfig();
startupConfig.Add(config);
startupConfig.NameOverride = config.Name;
return startupConfig;
}
}
}
+15 -16
View File
@@ -19,24 +19,23 @@ namespace CodexTests.BasicTests
var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
var contracts = Ci.StartCodexContracts(geth);
var numberOfHosts = 5;
var hosts = AddCodex(numberOfHosts, s => s
.WithName("Host")
.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
{
ContractClock = CodexLogLevel.Trace,
})
.WithStorageQuota(11.GB())
.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), hostInitialBalance)
.AsStorageNode()
.AsValidator()));
var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens();
foreach (var host in hosts)
var numberOfHosts = 3;
for (var i = 0; i < numberOfHosts; i++)
{
AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance));
var host = AddCodex(s => s
.WithName("Host")
.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
{
ContractClock = CodexLogLevel.Trace,
})
.WithStorageQuota(11.GB())
.EnableMarketplace(geth, contracts, m => m
.WithInitial(10.Eth(), hostInitialBalance)
.AsStorageNode()
.AsValidator()));
AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
var availability = new StorageAvailability(
totalSpace: 10.GB(),
+27
View File
@@ -0,0 +1,27 @@
using NUnit.Framework;
using WakuPlugin;
namespace WakuTests
{
public class BasicTests : WakuDistTest
{
[Test]
public void Hi()
{
var bootNode = Ci.StartWakuNode(s => s.WithName("BootstrapNode"));
var node = Ci.StartWakuNode(s => s.WithName("Waku1").WithBootstrapNode(bootNode));
var topic = "cheeseWheels";
var message = "hmm, cheese...";
bootNode.SubscribeToTopic(topic);
node.SubscribeToTopic(topic);
node.SendMessage(topic, message);
var received = bootNode.GetMessages(topic);
CollectionAssert.Contains(received, message);
}
}
}
+13
View File
@@ -0,0 +1,13 @@
using Core;
using DistTestCore;
namespace WakuTests
{
public class WakuDistTest : DistTest
{
public WakuDistTest()
{
ProjectPlugin.Load<WakuPlugin.WakuPlugin>();
}
}
}
+20
View File
@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="nunit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\ProjectPlugins\WakuPlugin\WakuPlugin.csproj" />
<ProjectReference Include="..\..\Tests\DistTestCore\DistTestCore.csproj" />
</ItemGroup>
</Project>
+7 -1
View File
@@ -1,6 +1,7 @@
using BiblioTech.Options;
using DiscordRewards;
using System.Globalization;
using Utils;
namespace BiblioTech.Commands
{
@@ -30,7 +31,12 @@ namespace BiblioTech.Commands
private string[] GetInsight(MarketAverage avg)
{
var headerLine = $"[{avg.Title}]";
var headerLine = $"[Last {Time.FormatDuration(avg.TimeRange)}] ({avg.NumberOfFinished} Contracts finished)";
if (avg.NumberOfFinished == 0)
{
return new[] { headerLine };
}
return new[]
{
+1 -4
View File
@@ -23,10 +23,7 @@ namespace BiblioTech.Rewards
{
try
{
if (cmd.Averages != null && cmd.Averages.Any())
{
Program.Averages = cmd.Averages;
}
Program.Averages = cmd.Averages;
await Program.RoleDriver.GiveRewards(cmd);
}
catch (Exception ex)
+4 -20
View File
@@ -24,16 +24,6 @@ namespace BiblioTech.Rewards
{
Program.Log.Log($"Processing rewards command: '{JsonConvert.SerializeObject(rewards)}'");
if (rewards.Rewards.Any())
{
await ProcessRewards(rewards);
}
await ProcessChainEvents(rewards.EventsOverview);
}
private async Task ProcessRewards(GiveRewardsCommand rewards)
{
var guild = GetGuild();
// We load all role and user information first,
// so we don't ask the server for the same info multiple times.
@@ -43,6 +33,7 @@ namespace BiblioTech.Rewards
rewardsChannel);
await context.ProcessGiveRewardsCommand(LookUpUsers(rewards));
await ProcessChainEvents(rewards.EventsOverview);
}
private SocketTextChannel? GetChannel(string name)
@@ -54,17 +45,10 @@ namespace BiblioTech.Rewards
private async Task ProcessChainEvents(string[] eventsOverview)
{
if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
await Task.Run(async () =>
foreach (var e in eventsOverview)
{
foreach (var e in eventsOverview)
{
if (!string.IsNullOrEmpty(e))
{
await eventsChannel.SendMessageAsync(e);
await Task.Delay(3000);
}
}
});
await eventsChannel.SendMessageAsync(e);
}
}
private async Task<Dictionary<ulong, IGuildUser>> LoadAllUsers(SocketGuild guild)
+4 -2
View File
@@ -1,5 +1,7 @@
using DiscordRewards;
using CodexContractsPlugin.Marketplace;
using DiscordRewards;
using Logging;
using Newtonsoft.Json;
using System.Net.Http.Json;
namespace TestNetRewarder
@@ -24,7 +26,7 @@ namespace TestNetRewarder
public async Task<bool> SendRewards(GiveRewardsCommand command)
{
if (command == null) return false;
if (command == null || command.Rewards == null || !command.Rewards.Any()) return false;
var result = await HttpPostJson(command);
log.Log("Reward response: " + result);
return result == "OK";
+15 -59
View File
@@ -1,6 +1,5 @@
using CodexContractsPlugin;
using CodexContractsPlugin.Marketplace;
using NethereumWorkflow.BlockUtils;
using Newtonsoft.Json;
using Utils;
@@ -9,34 +8,9 @@ namespace TestNetRewarder
public class ChainState
{
private readonly HistoricState historicState;
private readonly string[] colorIcons = new[]
{
"🔴",
"🟠",
"🟡",
"🟢",
"🔵",
"🟣",
"🟤",
"⚫",
"⚪",
"🟥",
"🟧",
"🟨",
"🟩",
"🟦",
"🟪",
"🟫",
"⬛",
"⬜",
"🔶",
"🔷"
};
public ChainState(HistoricState historicState, ICodexContracts contracts, BlockInterval blockRange)
{
this.historicState = historicState;
NewRequests = contracts.GetStorageRequests(blockRange);
historicState.CleanUpOldRequests();
historicState.ProcessNewRequests(NewRequests);
@@ -44,16 +18,19 @@ namespace TestNetRewarder
StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray();
FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFinished).ToArray();
ChangedRequests = historicState.StorageRequests.Where(r => r.RecentlyChanged).ToArray();
RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange);
RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange);
SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange);
SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange);
this.historicState = historicState;
}
public Request[] NewRequests { get; }
public StorageRequest[] AllRequests => historicState.StorageRequests;
public StorageRequest[] StartedRequests { get; private set; }
public StorageRequest[] FinishedRequests { get; private set; }
public StorageRequest[] ChangedRequests { get; private set; }
public RequestFulfilledEventDTO[] RequestFulfilledEvents { get; }
public RequestCancelledEventDTO[] RequestCancelledEvents { get; }
public SlotFilledEventDTO[] SlotFilledEvents { get; }
@@ -63,78 +40,57 @@ namespace TestNetRewarder
{
var entries = new List<StringBlockNumberPair>();
entries.AddRange(NewRequests.Select(ToPair));
entries.AddRange(ChangedRequests.Select(ToPair));
entries.AddRange(RequestFulfilledEvents.Select(ToPair));
entries.AddRange(RequestCancelledEvents.Select(ToPair));
entries.AddRange(SlotFilledEvents.Select(ToPair));
entries.AddRange(SlotFreedEvents.Select(ToPair));
entries.AddRange(FinishedRequests.Select(ToPair));
entries.Sort(new StringUtcComparer());
return entries.Select(ToLine).ToArray();
}
private StringBlockNumberPair ToPair(Request r)
private StringBlockNumberPair ToPair(StorageRequest r)
{
return new StringBlockNumberPair("NewRequest", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
}
public StringBlockNumberPair ToPair(StorageRequest r)
{
return new StringBlockNumberPair("FinishedRequest", JsonConvert.SerializeObject(r), r.Request.Block, r.Request.RequestId);
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.Request.BlockNumber);
}
private StringBlockNumberPair ToPair(RequestFulfilledEventDTO r)
{
return new StringBlockNumberPair("Fulfilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
}
private StringBlockNumberPair ToPair(RequestCancelledEventDTO r)
{
return new StringBlockNumberPair("Cancelled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
}
private StringBlockNumberPair ToPair(SlotFilledEventDTO r)
{
return new StringBlockNumberPair("SlotFilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
}
private StringBlockNumberPair ToPair(SlotFreedEventDTO r)
{
return new StringBlockNumberPair("SlotFreed", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
}
private string ToLine(StringBlockNumberPair pair)
{
var nl = Environment.NewLine;
var colorIcon = GetColorIcon(pair.RequestId);
return $"{colorIcon} {pair.Block} ({pair.Name}){nl}" +
$"```json{nl}" +
$"{pair.Str}{nl}" +
$"```";
}
private string GetColorIcon(byte[] requestId)
{
var index = requestId[0] % colorIcons.Length;
return colorIcons[index];
return $"[{pair.Number}] {pair.Str}";
}
public class StringBlockNumberPair
{
public StringBlockNumberPair(string name, string str, BlockTimeEntry block, byte[] requestId)
public StringBlockNumberPair(string str, ulong number)
{
Name = name;
Str = str;
Block = block;
RequestId = requestId;
Number = number;
}
public string Name { get; }
public string Str { get; }
public BlockTimeEntry Block { get; }
public byte[] RequestId { get; }
public ulong Number { get; }
}
public class StringUtcComparer : IComparer<StringBlockNumberPair>
@@ -144,7 +100,7 @@ namespace TestNetRewarder
if (x == null && y == null) return 0;
if (x == null) return 1;
if (y == null) return -1;
return x.Block.BlockNumber.CompareTo(y.Block.BlockNumber);
return x.Number.CompareTo(y.Number);
}
}
}
+3
View File
@@ -19,6 +19,9 @@ namespace TestNetRewarder
[Uniform("check-history", "ch", "CHECKHISTORY", true, "Unix epoc timestamp of a moment in history on which processing begins. Required for hosting rewards. Should be 'launch of the testnet'.")]
public int CheckHistoryTimestamp { get; set; } = 0;
[Uniform("market-insights", "mi", "MARKETINSIGHTS", false, "Semi-colon separated integers. Each represents a multiple of intervals, for which a market insights average will be generated.")]
public string MarketInsights { get; set; } = "1;96";
[Uniform("events-overview", "eo", "EVENTSOVERVIEW", false, "When greater than zero, chain event summary will be generated. (default 1)")]
public int CreateChainEventsOverview { get; set; } = 1;
+24
View File
@@ -28,6 +28,8 @@ namespace TestNetRewarder
r.State == RequestState.Finished ||
r.State == RequestState.Failed
);
foreach (var r in storageRequests) r.IsNew = false;
}
}
@@ -37,11 +39,13 @@ namespace TestNetRewarder
{
Request = request;
Hosts = Array.Empty<EthAddress>();
IsNew = true;
}
public Request Request { get; }
public EthAddress[] Hosts { get; private set; }
public RequestState State { get; private set; }
public bool IsNew { get; set; }
[JsonIgnore]
public bool RecentlyStarted { get; private set; }
@@ -49,6 +53,9 @@ namespace TestNetRewarder
[JsonIgnore]
public bool RecentlyFinished { get; private set; }
[JsonIgnore]
public bool RecentlyChanged { get; private set; }
public void Update(ICodexContracts contracts)
{
var newHosts = GetHosts(contracts);
@@ -63,10 +70,27 @@ namespace TestNetRewarder
State == RequestState.Started &&
newState == RequestState.Finished;
RecentlyChanged =
IsNew ||
State != newState ||
HostsChanged(newHosts);
State = newState;
Hosts = newHosts;
}
private bool HostsChanged(EthAddress[] newHosts)
{
if (newHosts.Length != Hosts.Length) return true;
foreach (var newHost in newHosts)
{
if (!Hosts.Contains(newHost)) return true;
}
return false;
}
private EthAddress[] GetHosts(ICodexContracts contracts)
{
var result = new List<EthAddress>();
+76 -49
View File
@@ -6,47 +6,65 @@ namespace TestNetRewarder
{
public class MarketTracker
{
private readonly MarketAverage MostRecent = new MarketAverage
{
Title = "Most recent"
};
private readonly MarketAverage Irf = new MarketAverage
{
Title = "Recent average"
};
private readonly List<ChainState> buffer = new List<ChainState>();
public MarketAverage[] ProcessChainState(ChainState chainState)
{
UpdateMostRecent(chainState);
UpdateIrf(chainState);
var intervalCounts = GetInsightCounts();
if (!intervalCounts.Any()) return Array.Empty<MarketAverage>();
return new[]
UpdateBuffer(chainState, intervalCounts.Max());
var result = intervalCounts
.Select(GenerateMarketAverage)
.Where(a => a != null)
.Cast<MarketAverage>()
.ToArray();
if (!result.Any()) result = Array.Empty<MarketAverage>();
return result;
}
private void UpdateBuffer(ChainState chainState, int maxNumberOfIntervals)
{
buffer.Add(chainState);
while (buffer.Count > maxNumberOfIntervals)
{
MostRecent,
Irf
};
buffer.RemoveAt(0);
}
}
private void UpdateIrf(ChainState chainState)
private MarketAverage? GenerateMarketAverage(int numberOfIntervals)
{
if (!chainState.FinishedRequests.Any()) return;
MostRecent.Price = GetIrf(MostRecent.Price, chainState, s => s.Request.Ask.Reward);
MostRecent.Duration = GetIrf(MostRecent.Duration, chainState, s => s.Request.Ask.Duration);
MostRecent.Size = GetIrf(MostRecent.Size, chainState, s => GetTotalSize(s.Request.Ask));
MostRecent.Collateral = GetIrf(MostRecent.Collateral, chainState, s => s.Request.Ask.Collateral);
MostRecent.ProofProbability = GetIrf(MostRecent.ProofProbability, chainState, s => s.Request.Ask.ProofProbability);
var states = SelectStates(numberOfIntervals);
return CreateAverage(states);
}
private void UpdateMostRecent(ChainState chainState)
private ChainState[] SelectStates(int numberOfIntervals)
{
if (!chainState.FinishedRequests.Any()) return;
if (numberOfIntervals < 1) return Array.Empty<ChainState>();
return buffer.TakeLast(numberOfIntervals).ToArray();
}
MostRecent.Price = Average(chainState, s => s.Request.Ask.Reward);
MostRecent.Duration = Average(chainState, s => s.Request.Ask.Duration);
MostRecent.Size = Average(chainState, s => GetTotalSize(s.Request.Ask));
MostRecent.Collateral = Average(chainState, s => s.Request.Ask.Collateral);
MostRecent.ProofProbability = Average(chainState, s => s.Request.Ask.ProofProbability);
private MarketAverage? CreateAverage(ChainState[] states)
{
try
{
return new MarketAverage
{
NumberOfFinished = CountNumberOfFinishedRequests(states),
TimeRange = GetTotalTimeRange(states),
Price = Average(states, s => s.Request.Ask.Reward),
Duration = Average(states, s => s.Request.Ask.Duration),
Size = Average(states, s => GetTotalSize(s.Request.Ask)),
Collateral = Average(states, s => s.Request.Ask.Collateral),
ProofProbability = Average(states, s => s.Request.Ask.ProofProbability)
};
}
catch (Exception ex)
{
Program.Log.Error($"Exception in CreateAverage: {ex}");
return null;
}
}
private int GetTotalSize(Ask ask)
@@ -56,40 +74,49 @@ namespace TestNetRewarder
return nSlots * slotSize;
}
private float Average(ChainState state, Func<StorageRequest, BigInteger> getValue)
private float Average(ChainState[] states, Func<StorageRequest, BigInteger> getValue)
{
return Average(state, s => Convert.ToInt32(getValue(s)));
return Average(states, s => Convert.ToInt32(getValue(s)));
}
private float GetIrf(float current, ChainState state, Func<StorageRequest, BigInteger> getValue)
{
return GetIrf(current, state, s => Convert.ToInt32(getValue(s)));
}
private float Average(ChainState state, Func<StorageRequest, int> getValue)
private float Average(ChainState[] states, Func<StorageRequest, int> getValue)
{
var sum = 0.0f;
var count = 0.0f;
foreach (var finishedRequest in state.FinishedRequests)
foreach (var state in states)
{
sum += getValue(finishedRequest);
count++;
foreach (var finishedRequest in state.FinishedRequests)
{
sum += getValue(finishedRequest);
count++;
}
}
if (count < 1.0f) return 0.0f;
return sum / count;
}
private float GetIrf(float current, ChainState state, Func<StorageRequest, int> getValue)
private TimeSpan GetTotalTimeRange(ChainState[] states)
{
var result = current;
foreach (var finishedRequest in state.FinishedRequests)
{
float v = getValue(finishedRequest);
result = (result + v) / 2.0f;
}
return Program.Config.Interval * states.Length;
}
return result;
private int CountNumberOfFinishedRequests(ChainState[] states)
{
return states.Sum(s => s.FinishedRequests.Length);
}
private int[] GetInsightCounts()
{
try
{
var tokens = Program.Config.MarketInsights.Split(';').ToArray();
return tokens.Select(t => Convert.ToInt32(t)).ToArray();
}
catch (Exception ex)
{
Program.Log.Error($"Exception when parsing MarketInsights config parameters: {ex}");
}
return Array.Empty<int>();
}
}
}
+2 -4
View File
@@ -67,11 +67,9 @@ namespace TestNetRewarder
var marketAverages = GetMarketAverages(chainState);
var eventsOverview = GenerateEventsOverview(chainState);
log.Log($"Found {outgoingRewards.Count} rewards. " +
$"Found {marketAverages.Length} market averages. " +
$"Found {eventsOverview.Length} events.");
log.Log($"Found {outgoingRewards.Count} rewards to send. Found {marketAverages.Length} market averages.");
if (outgoingRewards.Any() || marketAverages.Any() || eventsOverview.Any())
if (outgoingRewards.Any())
{
if (!await SendRewardsCommand(outgoingRewards, marketAverages, eventsOverview))
{
+14
View File
@@ -66,6 +66,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
.editorconfig = .editorconfig
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WakuTests", "Tests\WakuTests\WakuTests.csproj", "{DF69D56E-854E-45CD-B130-76386B5BB959}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WakuPlugin", "ProjectPlugins\WakuPlugin\WakuPlugin.csproj", "{2DB199E1-78D3-4A69-9773-C522F7D2FE69}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -172,6 +176,14 @@ Global
{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.Build.0 = Release|Any CPU
{DF69D56E-854E-45CD-B130-76386B5BB959}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DF69D56E-854E-45CD-B130-76386B5BB959}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DF69D56E-854E-45CD-B130-76386B5BB959}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DF69D56E-854E-45CD-B130-76386B5BB959}.Release|Any CPU.Build.0 = Release|Any CPU
{2DB199E1-78D3-4A69-9773-C522F7D2FE69}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2DB199E1-78D3-4A69-9773-C522F7D2FE69}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2DB199E1-78D3-4A69-9773-C522F7D2FE69}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2DB199E1-78D3-4A69-9773-C522F7D2FE69}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -202,6 +214,8 @@ Global
{F730DA73-1C92-4107-BCFB-D33759DAB0C3} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
{B07820C4-309F-4454-BCC1-1D4902C9C67B} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
{88C212E9-308A-46A4-BAAD-468E8EBD8EDF} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124}
{DF69D56E-854E-45CD-B130-76386B5BB959} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
{2DB199E1-78D3-4A69-9773-C522F7D2FE69} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}