Better logging for json errors and node actions.
This commit is contained in:
benbierens
parent
e4e7afd580
commit
e9d84a5cf7
|
|
@ -9,6 +9,16 @@
|
|||
|
||||
public long SizeInBytes { get; }
|
||||
|
||||
public override bool Equals(object? obj)
|
||||
{
|
||||
return obj is ByteSize size && SizeInBytes == size.SizeInBytes;
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return HashCode.Combine(SizeInBytes);
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{SizeInBytes} bytes";
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
using KubernetesWorkflow;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace DistTestCore.Codex
|
||||
{
|
||||
|
|
|
|||
|
|
@ -40,9 +40,9 @@ namespace DistTestCore
|
|||
}
|
||||
|
||||
fixtureLog.Log("Global setup cleanup successful");
|
||||
fixtureLog.Log($"Codex image: {CodexContainerRecipe.DockerImage}");
|
||||
fixtureLog.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}");
|
||||
fixtureLog.Log($"Geth image: {GethContainerRecipe.DockerImage}");
|
||||
fixtureLog.Log($"Codex image: '{CodexContainerRecipe.DockerImage}'");
|
||||
fixtureLog.Log($"Prometheus image: '{PrometheusContainerRecipe.DockerImage}'");
|
||||
fixtureLog.Log($"Geth image: '{GethContainerRecipe.DockerImage}'");
|
||||
}
|
||||
|
||||
[SetUp]
|
||||
|
|
|
|||
|
|
@ -35,20 +35,22 @@ namespace DistTestCore
|
|||
|
||||
public T HttpGetJson<T>(string route)
|
||||
{
|
||||
return JsonConvert.DeserializeObject<T>(HttpGetString(route))!;
|
||||
var json = HttpGetString(route);
|
||||
return TryJsonDeserialize<T>(json);
|
||||
}
|
||||
|
||||
public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
|
||||
{
|
||||
return Retry(() =>
|
||||
var json = Retry(() =>
|
||||
{
|
||||
using var client = GetClient();
|
||||
var url = GetUrl() + route;
|
||||
using var content = JsonContent.Create(body);
|
||||
var result = Time.Wait(client.PostAsync(url, content));
|
||||
var json = Time.Wait(result.Content.ReadAsStringAsync());
|
||||
return JsonConvert.DeserializeObject<TResponse>(json)!;
|
||||
return Time.Wait(result.Content.ReadAsStringAsync());
|
||||
});
|
||||
|
||||
return TryJsonDeserialize<TResponse>(json);
|
||||
}
|
||||
|
||||
public string HttpPostStream(string route, Stream stream)
|
||||
|
|
@ -98,13 +100,27 @@ namespace DistTestCore
|
|||
retryCounter++;
|
||||
if (retryCounter > Timing.HttpCallRetryCount())
|
||||
{
|
||||
Assert.Fail(exception.Message);
|
||||
Assert.Fail(exception.ToString());
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static T TryJsonDeserialize<T>(string json)
|
||||
{
|
||||
try
|
||||
{
|
||||
return JsonConvert.DeserializeObject<T>(json)!;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
|
||||
Assert.Fail(msg);
|
||||
throw new InvalidOperationException(msg, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpClient GetClient()
|
||||
{
|
||||
var client = new HttpClient();
|
||||
|
|
|
|||
|
|
@ -11,10 +11,12 @@ namespace DistTestCore.Logs
|
|||
public class CodexNodeLog : ICodexNodeLog
|
||||
{
|
||||
private readonly LogFile logFile;
|
||||
private readonly OnlineCodexNode owner;
|
||||
|
||||
public CodexNodeLog(LogFile logFile)
|
||||
public CodexNodeLog(LogFile logFile, OnlineCodexNode owner)
|
||||
{
|
||||
this.logFile = logFile;
|
||||
this.owner = owner;
|
||||
}
|
||||
|
||||
public void AssertLogContains(string expectedString)
|
||||
|
|
@ -29,7 +31,7 @@ namespace DistTestCore.Logs
|
|||
line = streamReader.ReadLine();
|
||||
}
|
||||
|
||||
Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
|
||||
Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@ namespace DistTestCore.Logs
|
|||
{
|
||||
public class LogDownloadHandler : LogHandler, ILogHandler
|
||||
{
|
||||
private readonly OnlineCodexNode node;
|
||||
private readonly LogFile log;
|
||||
|
||||
public LogDownloadHandler(string description, LogFile log)
|
||||
public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log)
|
||||
{
|
||||
this.node = node;
|
||||
this.log = log;
|
||||
|
||||
log.Write($"{description} -->> {log.FullFilename}");
|
||||
|
|
@ -17,7 +19,7 @@ namespace DistTestCore.Logs
|
|||
|
||||
public CodexNodeLog CreateCodexNodeLog()
|
||||
{
|
||||
return new CodexNodeLog(log);
|
||||
return new CodexNodeLog(log, node);
|
||||
}
|
||||
|
||||
protected override void ProcessLine(string line)
|
||||
|
|
|
|||
|
|
@ -44,10 +44,10 @@ namespace DistTestCore.Marketplace
|
|||
};
|
||||
|
||||
Log($"Requesting storage for: {contentId.Id}... (" +
|
||||
$"pricePerBytePerSecond: {pricePerBytePerSecond}," +
|
||||
$"requiredCollateral: {requiredCollateral}," +
|
||||
$"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}," +
|
||||
$"proofProbability: {proofProbability}," +
|
||||
$"pricePerBytePerSecond: {pricePerBytePerSecond}, " +
|
||||
$"requiredCollateral: {requiredCollateral}, " +
|
||||
$"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
|
||||
$"proofProbability: {proofProbability}, " +
|
||||
$"duration: {Time.FormatDuration(duration)})");
|
||||
|
||||
var response = codexAccess.RequestStorage(request, contentId.Id);
|
||||
|
|
@ -68,10 +68,10 @@ namespace DistTestCore.Marketplace
|
|||
};
|
||||
|
||||
Log($"Making storage available... (" +
|
||||
$"size: {size}," +
|
||||
$"minPricePerBytePerSecond: {minPricePerBytePerSecond}," +
|
||||
$"maxCollateral: {maxCollateral}," +
|
||||
$"maxDuration: {Time.FormatDuration(maxDuration)}");
|
||||
$"size: {size}, " +
|
||||
$"minPricePerBytePerSecond: {minPricePerBytePerSecond}, " +
|
||||
$"maxCollateral: {maxCollateral}, " +
|
||||
$"maxDuration: {Time.FormatDuration(maxDuration)})");
|
||||
|
||||
var response = codexAccess.SalesAvailability(request);
|
||||
|
||||
|
|
@ -103,14 +103,14 @@ namespace DistTestCore.Marketplace
|
|||
var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account);
|
||||
var balance = new TestToken(amount);
|
||||
|
||||
Log($"Balance of {codexAccess.Container.GetName()}({account}) is {balance}.");
|
||||
Log($"Balance of {account} is {balance}.");
|
||||
|
||||
return balance;
|
||||
}
|
||||
|
||||
private void Log(string msg)
|
||||
{
|
||||
log.Log(msg);
|
||||
log.Log($"{codexAccess.Container.GetName()} {msg}");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using NUnit.Framework;
|
||||
using NUnit.Framework.Constraints;
|
||||
using Utils;
|
||||
|
|
@ -10,22 +11,15 @@ namespace DistTestCore.Metrics
|
|||
void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
|
||||
}
|
||||
|
||||
public class MetricsUnavailable : IMetricsAccess
|
||||
{
|
||||
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
|
||||
{
|
||||
Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
public class MetricsAccess : IMetricsAccess
|
||||
{
|
||||
private readonly TestLog log;
|
||||
private readonly MetricsQuery query;
|
||||
private readonly RunningContainer node;
|
||||
|
||||
public MetricsAccess(MetricsQuery query, RunningContainer node)
|
||||
public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node)
|
||||
{
|
||||
this.log = log;
|
||||
this.query = query;
|
||||
this.node = node;
|
||||
}
|
||||
|
|
@ -34,6 +28,9 @@ namespace DistTestCore.Metrics
|
|||
{
|
||||
var metricSet = GetMetricWithTimeout(metricName);
|
||||
var metricValue = metricSet.Values[0].Value;
|
||||
|
||||
log.Log($"{node.GetName()} metric '{metricName}' = {metricValue}");
|
||||
|
||||
Assert.That(metricValue, constraint, message);
|
||||
}
|
||||
|
||||
|
|
@ -67,4 +64,13 @@ namespace DistTestCore.Metrics
|
|||
return result.Sets.LastOrDefault();
|
||||
}
|
||||
}
|
||||
|
||||
public class MetricsUnavailable : IMetricsAccess
|
||||
{
|
||||
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
|
||||
{
|
||||
Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,17 +17,19 @@ namespace DistTestCore.Metrics
|
|||
|
||||
public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
|
||||
{
|
||||
private readonly TestLifecycle lifecycle;
|
||||
private readonly RunningContainers prometheusContainer;
|
||||
|
||||
public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer)
|
||||
public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
this.prometheusContainer = prometheusContainer;
|
||||
}
|
||||
|
||||
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
|
||||
{
|
||||
var query = new MetricsQuery(prometheusContainer);
|
||||
return new MetricsAccess(query, codexContainer);
|
||||
return new MetricsAccess(lifecycle.Log, query, codexContainer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ namespace DistTestCore
|
|||
|
||||
public string Describe()
|
||||
{
|
||||
return $"({Group.Describe()} contains {GetName()})";
|
||||
return $"({GetName()} in {Group.Describe()})";
|
||||
}
|
||||
|
||||
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ namespace DistTestCore
|
|||
|
||||
LogEnd("Metrics server started.");
|
||||
|
||||
return new CodexNodeMetricsAccessFactory(runningContainers);
|
||||
return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
|
||||
}
|
||||
|
||||
private string GeneratePrometheusConfig(RunningContainer[] nodes)
|
||||
|
|
|
|||
|
|
@ -35,12 +35,12 @@ namespace DistTestCore
|
|||
{
|
||||
var subFile = Log.CreateSubfile();
|
||||
var description = node.Describe();
|
||||
var handler = new LogDownloadHandler(description, subFile);
|
||||
var handler = new LogDownloadHandler(node, description, subFile);
|
||||
|
||||
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
|
||||
CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
|
||||
|
||||
return new CodexNodeLog(subFile);
|
||||
return new CodexNodeLog(subFile, node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,21 @@
|
|||
}
|
||||
|
||||
public decimal Wei { get; }
|
||||
|
||||
public override bool Equals(object? obj)
|
||||
{
|
||||
return obj is Ether ether && Wei == ether.Wei;
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return HashCode.Combine(Wei);
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{Wei} Wei";
|
||||
}
|
||||
}
|
||||
|
||||
public class TestToken
|
||||
|
|
@ -19,6 +34,16 @@
|
|||
|
||||
public decimal Amount { get; }
|
||||
|
||||
public override bool Equals(object? obj)
|
||||
{
|
||||
return obj is TestToken token && Amount == token.Amount;
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return HashCode.Combine(Amount);
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{Amount} TestTokens";
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ namespace Logging
|
|||
methodName = GetMethodName();
|
||||
fullName = Path.Combine(folder, methodName);
|
||||
|
||||
Log($"Begin: {methodName}");
|
||||
Log($"*** Begin: {methodName}");
|
||||
}
|
||||
|
||||
public LogFile CreateSubfile(string ext = "log")
|
||||
|
|
@ -26,7 +26,7 @@ namespace Logging
|
|||
{
|
||||
var result = TestContext.CurrentContext.Result;
|
||||
|
||||
Log($"Finished: {methodName} = {result.Outcome.Status}");
|
||||
Log($"*** Finished: {methodName} = {result.Outcome.Status}");
|
||||
if (!string.IsNullOrEmpty(result.Message))
|
||||
{
|
||||
Log(result.Message);
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ namespace Tests.BasicTests
|
|||
primary.ConnectToPeer(secondary);
|
||||
primary2.ConnectToPeer(secondary2);
|
||||
|
||||
Thread.Sleep(TimeSpan.FromMinutes(5));
|
||||
Thread.Sleep(TimeSpan.FromMinutes(2));
|
||||
|
||||
primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
|
||||
primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
|
||||
|
|
@ -55,7 +55,7 @@ namespace Tests.BasicTests
|
|||
.EnableMarketplace(initialBalance: 234.TestTokens())
|
||||
.BringOnline()[0];
|
||||
|
||||
Assert.That(primary.Marketplace.GetBalance(), Is.EqualTo(234));
|
||||
primary.Marketplace.AssertThatBalance(Is.EqualTo(234.TestTokens()));
|
||||
|
||||
var secondary = SetupCodexNodes(1)
|
||||
.EnableMarketplace(initialBalance: 1000.TestTokens())
|
||||
|
|
|
|||
Loading…
Reference in New Issue