This commit is contained in:
Ben 2024-12-19 12:10:37 +01:00
parent 4801e33986
commit 596011c962
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
11 changed files with 368 additions and 16 deletions

View File

@ -22,6 +22,8 @@ namespace CodexPlugin
ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
(TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "");
(TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
LocalDataset DownloadStreamless(ContentId cid);
/// <summary>
/// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
@ -189,10 +191,20 @@ namespace CodexPlugin
public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
{
return DownloadContent(contentId, DoNothing, fileLabel);
return DownloadContentT(contentId, fileLabel).Item1;
}
public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
{
return DownloadContentT(contentId, onFailure, fileLabel).Item1;
}
public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "")
{
return DownloadContentT(contentId, DoNothing, fileLabel);
}
public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
{
var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
hooks.OnFileDownloading(contentId);
@ -205,7 +217,7 @@ namespace CodexPlugin
transferSpeeds.AddDownloadSample(size, measurement);
hooks.OnFileDownloaded(size, contentId);
return file;
return (file, measurement);
}
public LocalDataset DownloadStreamless(ContentId cid)

9
SeeTimeline/App.xaml Normal file
View File

@ -0,0 +1,9 @@
<Application x:Class="SeeTimeline.App"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="clr-namespace:SeeTimeline"
StartupUri="MainWindow.xaml">
<Application.Resources>
</Application.Resources>
</Application>

14
SeeTimeline/App.xaml.cs Normal file
View File

@ -0,0 +1,14 @@
using System.Configuration;
using System.Data;
using System.Windows;
namespace SeeTimeline
{
/// <summary>
/// Interaction logic for App.xaml
/// </summary>
public partial class App : Application
{
}
}

View File

@ -0,0 +1,10 @@
using System.Windows;
[assembly: ThemeInfo(
ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located
//(used if a resource is not found in the page,
// or application resource dictionaries)
ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located
//(used if a resource is not found in the page,
// app, or any theme specific resource dictionaries)
)]

View File

@ -0,0 +1,29 @@
<Window x:Class="SeeTimeline.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:tl="clr-namespace:TimelinerNet;assembly=TimelinerNet"
xmlns:local="clr-namespace:SeeTimeline"
mc:Ignorable="d"
Title="SeeTimeline" Height="450" Width="800">
<Grid>
<Grid.RowDefinitions>
<RowDefinition Height="Auto"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="*"/>
<RowDefinition Height="Auto"/>
<RowDefinition Height="*"/>
</Grid.RowDefinitions>
<Grid.LayoutTransform>
<ScaleTransform
ScaleX="1.5"
ScaleY="1.5"/>
</Grid.LayoutTransform>
<Button Grid.Row="0" Content="Load Logs" Click="Button_Click"/>
<TextBlock Grid.Row="1" Name="Line1Name" HorizontalAlignment="Center"/>
<tl:Timeliner Name="Timeliner1" Grid.Row="2"/>
<TextBlock Grid.Row="3" Name="Line2Name" HorizontalAlignment="Center"/>
<tl:Timeliner Name="Timeliner2" Grid.Row="4"/>
</Grid>
</Window>

View File

@ -0,0 +1,199 @@
using CodexPlugin;
using Microsoft.Win32;
using System.IO;
using System.Text;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;
using TimelinerNet;
namespace SeeTimeline
{
/// <summary>
/// Interaction logic for MainWindow.xaml
/// </summary>
public partial class MainWindow : Window
{
public MainWindow()
{
InitializeComponent();
Timeliner1.PropertyChanged += Timeliner1_PropertyChanged;
}
private void Timeliner1_PropertyChanged(object? sender, System.ComponentModel.PropertyChangedEventArgs e)
{
Timeliner2.LeftEdge = Timeliner1.LeftEdge;
Timeliner2.RightEdge = Timeliner1.RightEdge;
}
private TimelinerData CreateTimelineData(string logfile, DateTime now)
{
var lines = File.ReadAllLines(logfile);
var handler = new LogLineHandler(now);
foreach (var line in lines)
{
var cline = CodexLogLine.Parse(line);
if (cline != null)
{
handler.Handle(cline);
if (handler.Size > 20) break;
}
}
return handler.GetTimeline();
}
public class LogLineHandler
{
public class BlockReq
{
public string Address { get; set; } = string.Empty;
public DateTime Created { get; set; }
public DateTime[] WantHaveSent { get; set; } = Array.Empty<DateTime>();
public DateTime[] PresenceRecv{ get; set; } = Array.Empty<DateTime>();
public DateTime[] WantBlkSent { get; set; } = Array.Empty<DateTime>();
public DateTime[] BlkRecv { get; set; } = Array.Empty<DateTime>();
public DateTime[] CancelSent { get; set; } = Array.Empty<DateTime>();
public DateTime[] Resolve { get; set; } = Array.Empty<DateTime>();
}
private readonly List<BlockReq> requests = new List<BlockReq>();
private readonly DateTime now;
private long? zero;
public int Size => requests.Count;
public LogLineHandler(DateTime now)
{
this.now = now;
}
public void Handle(CodexLogLine line)
{
if (line.Message != "times for") return;
var addr = line.Attributes["addrs"];
if (!addr.Contains("index")) return;
// addrs="treeCid: zDz*qNvVWp, index: 0"
// reqCreatedTime=4784825696831
// wantHaveSentTimes=@[4784825884225]
// presenceRecvTimes=@[4784826770921]
// wantBlkSentTimes=@[4784826954293]
// blkRecvTimes=@[4784829724756]
// cancelSentTimes=@[4784830399255]
// resolveTimes=@[4784830322141]
var req = new BlockReq
{
Address = line.Attributes["addrs"],
Created = ToUtc(line.Attributes["reqCreatedTime"]),
WantHaveSent = ToUtcs(line.Attributes["wantHaveSentTimes"]),
PresenceRecv = ToUtcs(line.Attributes["presenceRecvTimes"]),
WantBlkSent = ToUtcs(line.Attributes["wantBlkSentTimes"]),
BlkRecv = ToUtcs(line.Attributes["blkRecvTimes"]),
CancelSent = ToUtcs(line.Attributes["cancelSentTimes"]),
Resolve = ToUtcs(line.Attributes["resolveTimes"])
};
requests.Add(req);
}
private DateTime[] ToUtcs(string str)
{
var tokens = str.Split(",");
return tokens
.Select(t => t
.Replace(" ", "")
.Replace("@[", "")
.Replace("]", "")
).Select(t => ToUtc(t)).ToArray();
}
private DateTime ToUtc(string str)
{
// is nanoseconds from arbitrary time point.
var monotime = Convert.ToInt64(str);
if (!zero.HasValue) zero = monotime;
double deltaNanoseconds = (monotime - zero.Value);
var delta = deltaNanoseconds / (1000 * 1000);
return now + TimeSpan.FromSeconds(delta * 5.0);
}
public TimelinerData GetTimeline()
{
return new TimelinerData
{
Items = requests.Select(ToTimelineItem).ToList()
};
}
private TimelinerItem ToTimelineItem(BlockReq req)
{
return new TimelinerItem
{
Name = req.Address,
Jobs = CreateTimelineJobs(req)
};
}
private List<TimelinerJob> CreateTimelineJobs(BlockReq req)
{
var result = new List<TimelinerJob>();
AddJobs(result, "Created", Colors.Red, req.Created);
AddJobs(result, "WantHaveSent", Colors.Orange, req.WantHaveSent);
AddJobs(result, "PresenceRecv", Colors.Yellow, req.PresenceRecv);
AddJobs(result, "WantBlkSent", Colors.Green, req.WantBlkSent);
AddJobs(result, "BlkRecv", Colors.Blue, req.BlkRecv);
AddJobs(result, "CancelSent", Colors.Purple, req.CancelSent);
AddJobs(result, "Resolve", Colors.Pink, req.Resolve);
return result;
}
private void AddJobs(List<TimelinerJob> result, string name, Color color, params DateTime[] moments)
{
var i = 0;
foreach (var dt in moments)
{
result.Add(new TimelinerJob
{
Name = name + i.ToString(),
Begin = dt,
End = dt,
Color = new SolidColorBrush(color),
});
i++;
}
}
}
private void Button_Click(object sender, RoutedEventArgs e)
{
var now = DateTime.Now;
var dlg = new OpenFileDialog();
if (dlg.ShowDialog() != true) return;
var file1 = dlg.FileName;
Line1Name.Text = file1;
if (dlg.ShowDialog() != true) return;
var file2 = dlg.FileName;
Line2Name.Text = file2;
var data1 = CreateTimelineData(file1, now);
var data2 = CreateTimelineData(file2, now);
Timeliner1.Data = data1;
Timeliner2.Data = data2;
}
}
}

View File

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>WinExe</OutputType>
<TargetFramework>net8.0-windows</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<UseWPF>true</UseWPF>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="TimelinerNet" Version="0.1.6" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup />
<ItemGroup>
<ApplicationDefinition Update="App.xaml">
<SubType>Designer</SubType>
</ApplicationDefinition>
</ItemGroup>
<ItemGroup>
<Page Update="MainWindow.xaml">
<SubType>Designer</SubType>
</Page>
</ItemGroup>
</Project>

View File

@ -14,8 +14,28 @@ namespace CodexReleaseTests.DataTests
public class TwoClientTests : CodexDistTest
{
[Test]
public void TwoClientTest()
[Combinatorial]
public void TwoClientTest(
[Values(
"thatbenbierens/nim-codex:blkex-cancelpresence-2", // S don't send cancel-presence messages
"thatbenbierens/nim-codex:blkex-cancelpresence-1", // F ignore cancel-presence messages
"codexstorage/nim-codex:sha-4b5c355-dist-tests", // F unmodified
"thatbenbierens/nim-codex:blkex-cancelpresence-3", // F same as 1 but logging
"thatbenbierens/nim-codex:blkex-cancelpresence-4", // S no cancel-presence-msg, no fromCancel field
"thatbenbierens/nim-codex:blkex-cancelpresence-5", // F all-presence = cancel? return from handler
"thatbenbierens/nim-codex:blkex-cancelpresence-6", // F no cancel-presence-msg, but if any cancel send empty presence msg
"thatbenbierens/nim-codex:blkex-cancelpresence-7", // F same but logs outgoing empty presence message. (msg is empty structure)
"thatbenbierens/nim-codex:blkex-cancelpresence-8", // crashes F? eventtimelogging
"thatbenbierens/nim-codex:blkex-cancelpresence-9", // crashes S? eventtimelogging + no cancel-presence-msg (should be slow)
"thatbenbierens/nim-codex:blkex-cancelpresence-10", // F eventtimelogging (should be fast)
"thatbenbierens/nim-codex:blkex-cancelpresence-11" // S eventtimelogging + no cancel-presence-msg (should be slow)
)] string img
)
{
CodexContainerRecipe.DockerImageOverride = img;
var uploader = StartCodex(s => s.WithName("Uploader"));
var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
@ -23,25 +43,43 @@ namespace CodexReleaseTests.DataTests
}
[Test]
[Ignore("Location selection is currently unavailable.")]
public void TwoClientsTwoLocationsTest()
public void ParseLogs()
{
var locations = Ci.GetKnownLocations();
if (locations.NumberOfLocations < 2)
var path = "";
var lines = File.ReadAllLines(path);
foreach (var line in lines)
{
Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster.");
return;
var cline = CodexLogLine.Parse(line);
if (cline == null) continue;
if (cline.Message == "times for")
{
ProcessTimes(cline);
}
}
}
var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0)));
var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1)));
private void ProcessTimes(CodexLogLine cline)
{
// reqCreatedTime
// wantHaveSentTimes
// presenceRecvTimes
// wantBlkSentTimes
// blkRecvTimes
// cancelSentTimes
// resolveTimes
}
public class BlockReqTimes
{
public TimeSpan CreateToWantHaveSent { get; set; }
PerformTwoClientTest(uploader, downloader);
}
private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader)
{
PerformTwoClientTest(uploader, downloader, 10.MB());
PerformTwoClientTest(uploader, downloader, 100.MB());
}
private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size)
@ -51,11 +89,12 @@ namespace CodexReleaseTests.DataTests
var contentId = uploader.UploadFile(testFile);
AssertNodesContainFile(contentId, uploader);
var downloadedFile = downloader.DownloadContent(contentId);
var (downloadedFile, timeTaken) = downloader.DownloadContentT(contentId);
AssertNodesContainFile(contentId, uploader, downloader);
Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(15.0)), "Too slow!");
testFile.AssertIsEqual(downloadedFile);
CheckLogForErrors(uploader, downloader);
}
}
}

View File

@ -14,7 +14,7 @@ namespace DistTestCore
kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
AlwaysDownloadContainerLogs = !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", ""));
AlwaysDownloadContainerLogs = true; // !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", ""));
}
public Configuration(string? kubeConfigFile, string logPath, string dataFilesPath)

View File

@ -82,6 +82,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExperimentalTests", "Tests\
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BlockchainUtils", "Framework\BlockchainUtils\BlockchainUtils.csproj", "{4648B5AA-A0A7-44BA-89BC-2FD57370943C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SeeTimeline", "SeeTimeline\SeeTimeline.csproj", "{24702B12-0448-4770-9AD2-8BDE5B04EF94}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -220,6 +222,10 @@ Global
{4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Release|Any CPU.Build.0 = Release|Any CPU
{24702B12-0448-4770-9AD2-8BDE5B04EF94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{24702B12-0448-4770-9AD2-8BDE5B04EF94}.Debug|Any CPU.Build.0 = Debug|Any CPU
{24702B12-0448-4770-9AD2-8BDE5B04EF94}.Release|Any CPU.ActiveCfg = Release|Any CPU
{24702B12-0448-4770-9AD2-8BDE5B04EF94}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -258,6 +264,7 @@ Global
{639A0603-4E80-465B-BB59-AB02F1DEEF5A} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
{BA7369CD-7C2F-4075-8E35-98BCC19EE203} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
{4648B5AA-A0A7-44BA-89BC-2FD57370943C} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
{24702B12-0448-4770-9AD2-8BDE5B04EF94} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}