status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improvements

This commit is contained in:
Ben 2024-09-18 11:54:23 +02:00
parent 9d5abd8955
commit d03dd9f954
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
16 changed files with 218 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Framework/FileUtils/FileManager.cs
View File

@ -31,9 +31,16 @@ namespace FileUtils
public const int ChunkSize = 1024 * 1024 * 100;
public FileManager(ILog log, string rootFolder)
public FileManager(ILog log, string rootFolder, bool numberSubfolders = true)
{
folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
if (numberSubfolders)
{
folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
}
else
{
folder = rootFolder;
}
this.log = log;
}

2
Framework/KubernetesWorkflow/StartupWorkflow.cs
View File

@ -170,7 +170,7 @@ namespace KubernetesWorkflow
var addresses = CreateContainerAddresses(startResult, r);
log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
return new RunningContainer(log, Guid.NewGuid().ToString(), name, r, addresses);
}).ToArray();
}

7
Framework/KubernetesWorkflow/Types/RunningContainer.cs
View File

@ -7,8 +7,11 @@ namespace KubernetesWorkflow.Types
{
public class RunningContainer
{
public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
private readonly ILog log;
public RunningContainer(ILog log, string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
{
this.log = log;
Id = id;
Name = name;
Recipe = recipe;
@ -24,7 +27,7 @@ namespace KubernetesWorkflow.Types
[JsonIgnore]
public RunningPod RunningPod { get; internal set; } = null!;
public Address GetAddress(ILog log, string portTag)
public Address GetAddress(string portTag)
{
var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);

2
ProjectPlugins/BittorrentPlugin/BittorrentContainerRecipe.cs
View File

@ -6,7 +6,7 @@ namespace BittorrentPlugin
public class BittorrentContainerRecipe : ContainerRecipeFactory
{
public override string AppName => "bittorrent";
public override string Image => "thatbenbierens/bittorrentdriver:init6";
public override string Image => "thatbenbierens/bittorrentdriver:init11";
public static string ApiPortTag = "API_PORT";
public static string TrackerPortTag = "TRACKER_PORT";

106
ProjectPlugins/BittorrentPlugin/BittorrentNode.cs
View File

@ -1,9 +1,11 @@
using Core;
using KubernetesWorkflow.Types;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using Utils;
@ -13,34 +15,52 @@ namespace BittorrentPlugin
public interface IBittorrentNode
{
string StartAsTracker();
string CreateTorrent(ByteSize size, IBittorrentNode tracker);
string AddTracker(IBittorrentNode tracker, string localFile);
string PutFile(string base64);
string GetTrackerStats();
CreateTorrentResult CreateTorrent(ByteSize size, IBittorrentNode tracker);
string StartDaemon();
string DownloadTorrent(string torrent);
string DownloadTorrent(string LocalFile);
}
public class BittorrentNode : IBittorrentNode
{
private readonly IPluginTools tools;
private readonly RunningContainer container;
private readonly PodInfo podInfo;
public BittorrentNode(IPluginTools tools, RunningContainer container)
{
this.tools = tools;
this.container = container;
podInfo = tools.CreateWorkflow().GetPodInfo(container);
}
public string CreateTorrent(ByteSize size, IBittorrentNode tracker)
public string StartAsTracker()
{
var trackerUrl = ((BittorrentNode)tracker).TrackerAddress;
//TrackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
var endpoint = GetEndpoint();
return endpoint.HttpPutString("starttracker", GetTrackerAddress().Port.ToString());
}
var torrent = endpoint.HttpPostJson("create", new CreateTorrentRequest
public string AddTracker(IBittorrentNode tracker, string localFile)
{
var endpoint = GetEndpoint();
var trackerUrl = ((BittorrentNode)tracker).GetTrackerAddress();
return endpoint.HttpPostJson("addtracker", new AddTrackerRequest
{
Size = Convert.ToInt32(size.SizeInBytes),
LocalFile = localFile,
TrackerUrl = $"{trackerUrl}/announce"
});
}
return torrent;
public string PutFile(string base64)
{
var endpoint = GetEndpoint();
return endpoint.HttpPostJson("postfile", new PostFileRequest
{
Base64Content = base64
});
}
public string StartDaemon()
@ -50,43 +70,79 @@ namespace BittorrentPlugin
return endpoint.HttpPutString("daemon", peerPortAddress.Port.ToString());
}
public string DownloadTorrent(string torrent)
public CreateTorrentResult CreateTorrent(ByteSize size, IBittorrentNode tracker)
{
var trackerUrl = ((BittorrentNode)tracker).GetTrackerAddress();
var endpoint = GetEndpoint();
var json = endpoint.HttpPostJson("create", new CreateTorrentRequest
{
Size = Convert.ToInt32(size.SizeInBytes),
TrackerUrl = $"{trackerUrl}/announce"
});
return JsonConvert.DeserializeObject<CreateTorrentResult>(json)!;
}
public string DownloadTorrent(string localFile)
{
var endpoint = GetEndpoint();
return endpoint.HttpPostJson("download", new DownloadTorrentRequest
{
TorrentBase64 = torrent
LocalFile = localFile
});
}
public string StartAsTracker()
public string GetTrackerStats()
{
TrackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
var endpoint = GetEndpoint();
var trackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
return endpoint.HttpPutString("tracker", trackerAddress.Port.ToString());
//var http = tools.CreateHttp(TrackerAddress.ToString(), c => { });
//var endpoint = http.CreateEndpoint(TrackerAddress, "/", container.Name);
//return endpoint.HttpGetString("stats");
return "no";
}
public Address TrackerAddress { get; private set; } = new Address("", 0);
//public Address TrackerAddress { get; private set; } = new Address("", 0);
public class CreateTorrentRequest
public Address GetTrackerAddress()
{
public int Size { get; set; }
public string TrackerUrl { get; set; } = string.Empty;
}
public class DownloadTorrentRequest
{
public string TorrentBase64 { get; set; } = string.Empty;
var address = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
return new Address("http://" + podInfo.Ip, address.Port);
}
private IEndpoint GetEndpoint()
{
var address = container.GetAddress(tools.GetLog(), BittorrentContainerRecipe.ApiPortTag);
var address = container.GetAddress(BittorrentContainerRecipe.ApiPortTag);
var http = tools.CreateHttp(address.ToString(), c => { });
return http.CreateEndpoint(address, "/torrent/", container.Name);
}
}
public class CreateTorrentRequest
{
public int Size { get; set; }
public string TrackerUrl { get; set; } = string.Empty;
}
public class CreateTorrentResult
{
public string LocalFilePath { get; set; } = string.Empty;
public string TorrentBase64 { get; set; } = string.Empty;
}
public class DownloadTorrentRequest
{
public string LocalFile { get; set; } = string.Empty;
}
public class AddTrackerRequest
{
public string TrackerUrl { get; set; } = string.Empty;
public string LocalFile { get; set; } = string.Empty;
}
public class PostFileRequest
{
public string Base64Content { get; set; } = string.Empty;
}
}

2
ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs
View File

@ -19,7 +19,7 @@ namespace CodexContractsPlugin
{
var config = startupConfig.Get<CodexContractsContainerConfig>();
var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
SetSchedulingAffinity(notIn: "false");

2
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -189,7 +189,7 @@ namespace CodexPlugin
private Address GetAddress()
{
return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
}
private string GetHttpId()

2
ProjectPlugins/GethPlugin/GethNode.cs
View File

@ -57,7 +57,7 @@ namespace GethPlugin
protected override NethereumInteraction StartInteraction()
{
var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
var account = StartResult.Account;
var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);

4
ProjectPlugins/MetricsPlugin/MetricsQuery.cs
View File

@ -15,7 +15,7 @@ namespace MetricsPlugin
{
RunningContainer = runningContainer;
log = tools.GetLog();
var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag);
var address = RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag);
endpoint = tools
.CreateHttp(address.ToString())
.CreateEndpoint(address, "/api/v1/");
@ -126,7 +126,7 @@ namespace MetricsPlugin
private string GetInstanceNameForNode(IMetricsScrapeTarget target)
{
return ScrapeTargetHelper.FormatTarget(log, target);
return ScrapeTargetHelper.FormatTarget(target);
}
private string GetInstanceStringForNode(IMetricsScrapeTarget target)

6
ProjectPlugins/MetricsPlugin/PrometheusStarter.cs
View File

@ -72,15 +72,15 @@ namespace MetricsPlugin
private string FormatTarget(IMetricsScrapeTarget target)
{
return ScrapeTargetHelper.FormatTarget(tools.GetLog(), target);
return ScrapeTargetHelper.FormatTarget(target);
}
}
public static class ScrapeTargetHelper
{
public static string FormatTarget(ILog log, IMetricsScrapeTarget target)
public static string FormatTarget(IMetricsScrapeTarget target)
{
var a = target.Container.GetAddress(log, target.MetricsPortTag);
var a = target.Container.GetAddress(target.MetricsPortTag);
var host = a.Host.Replace("http://", "").Replace("https://", "");
return $"{host}:{a.Port}";
}

2
Tests/CodexContinuousTests/StartupChecker.cs
View File

@ -98,7 +98,7 @@ namespace ContinuousTests
{
cancelToken.ThrowIfCancellationRequested();
var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag);
var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
log.Log($"Checking {n.Container.Name} @ '{address}'...");
if (EnsureOnline(log, n))

10
Tests/CodexTests/BasicTests/ExampleTests.cs
View File

@ -73,13 +73,21 @@ namespace CodexTests.BasicTests
{
var tracker = Ci.StartBittorrentNode();
var msg = tracker.StartAsTracker();
msg = tracker.GetTrackerStats();
var seeder = Ci.StartBittorrentNode();
var torrent = seeder.CreateTorrent(10.MB(), tracker);
msg = seeder.AddTracker(tracker, torrent.LocalFilePath);
msg = seeder.StartDaemon();
Thread.Sleep(5000);
msg = tracker.GetTrackerStats();
var leecher = Ci.StartBittorrentNode();
msg = leecher.DownloadTorrent(torrent);
var local = leecher.PutFile(torrent.TorrentBase64);
leecher.AddTracker(tracker, local);
msg = leecher.DownloadTorrent(local);
var yay = 0;
}

2
Tools/AutoClient/Purchaser.cs
View File

@ -151,7 +151,7 @@ namespace AutoClient
{
try
{
var sp = await GetStoragePurchase(pid)!;
var sp = (await GetStoragePurchase(pid))!;
return sp.Request.Content.Cid;
}
catch (Exception ex)

45
Tools/BittorrentDriver/Controllers/TorrentController.cs
View File

@ -17,7 +17,7 @@ namespace BittorrentDriver.Controllers
transmission = new Transmission(log);
}
[HttpPut("tracker")]
[HttpPut("starttracker")]
public string StartTracker([FromBody] int port)
{
return Try(() =>
@ -27,6 +27,28 @@ namespace BittorrentDriver.Controllers
});
}
[HttpPost("addtracker")]
public string AddTracker([FromBody] AddTrackerInput input)
{
return Try(() =>
{
Log("Adding tracker: " + input.TrackerUrl + " - " + input.LocalFile);
return transmission.AddTracker(input.TrackerUrl, input.LocalFile);
});
}
[HttpPost("postfile")]
public string PostFile([FromBody] PostFileInput input)
{
return Try(() =>
{
Log("Creating file..");
var file = transmission.PutLocalFile(input.Base64Content);
Log("File: " + file);
return file;
});
}
[HttpPut("daemon")]
public string StartDaemon([FromBody] int peerPort)
{
@ -38,7 +60,7 @@ namespace BittorrentDriver.Controllers
}
[HttpPost("create")]
public string CreateTorrent([FromBody] CreateTorrentInput input)
public CreateTorrentResult CreateTorrent([FromBody] CreateTorrentInput input)
{
return Try(() =>
{
@ -53,11 +75,11 @@ namespace BittorrentDriver.Controllers
return Try(() =>
{
Log("Downloading torrent...");
return transmission.Download(input.TorrentBase64);
return transmission.Download(input.LocalFile);
});
}
private string Try(Func<string> value)
private T Try<T>(Func<T> value)
{
try
{
@ -66,7 +88,7 @@ namespace BittorrentDriver.Controllers
catch (Exception exc)
{
log.Error(exc.ToString());
return exc.ToString();
throw;
}
}
@ -82,8 +104,19 @@ namespace BittorrentDriver.Controllers
public string TrackerUrl { get; set; } = string.Empty;
}
public class AddTrackerInput
{
public string TrackerUrl { get; set; } = string.Empty;
public string LocalFile { get; set; } = string.Empty;
}
public class DownloadTorrentInput
{
public string TorrentBase64 { get; set; } = string.Empty;
public string LocalFile { get; set; } = string.Empty;
}
public class PostFileInput
{
public string Base64Content { get; set; } = string.Empty;
}
}

20
Tools/BittorrentDriver/TorrentTracker.cs
View File

@ -12,7 +12,12 @@ namespace BittorrentDriver
var info = new ProcessStartInfo
{
FileName = "bittorrent-tracker",
Arguments = $"--port {port} &",
Arguments =
$"--port {port} " +
$"--http " +
$"--stats " +
$"--interval=3000 " + // 3 seconds
$"&",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
@ -21,6 +26,18 @@ namespace BittorrentDriver
process = Process.Start(info);
if (process == null) return "Failed to start";
process.OutputDataReceived += (sender, args) =>
{
Console.WriteLine("STDOUT: " + args.Data);
};
process.ErrorDataReceived += (sender, args) =>
{
Console.WriteLine("STDERR: " + args.Data);
};
process.BeginOutputReadLine();
process.BeginErrorReadLine();
Thread.Sleep(1000);
if (process.HasExited)
@ -31,6 +48,5 @@ namespace BittorrentDriver
}
return "OK";
}
}
}

58
Tools/BittorrentDriver/Transmission.cs
View File

@ -19,17 +19,16 @@ namespace BittorrentDriver
this.log = log;
}
public string CreateNew(int size, string trackerUrl)
public CreateTorrentResult CreateNew(int size, string trackerUrl)
{
var file = CreateFile(size);
var outFile = Path.Combine(Directory.GetCurrentDirectory(), Guid.NewGuid().ToString());
var outFile = Path.Combine(dataDir, Guid.NewGuid().ToString());
var base64 = CreateTorrentFile(file, outFile, trackerUrl);
if (File.Exists(outFile)) File.Delete(outFile);
return base64;
return new CreateTorrentResult
{
LocalFilePath = outFile,
TorrentBase64 = base64
};
}
public string StartDaemon(int peerPort)
@ -37,22 +36,48 @@ namespace BittorrentDriver
var info = new ProcessStartInfo
{
FileName = "transmission-daemon",
Arguments = $"--peerport={peerPort} --download-dir={dataDir}"
Arguments = $"--peerport={peerPort} " +
$"--download-dir={dataDir} " +
$"--watch-dir={dataDir} " +
$"--no-global-seedratio " +
$"--bind-address-ipv4=0.0.0.0 " +
$"--dht"
};
RunToComplete(info);
return "OK";
}
public string Download(string torrentBase64)
public string AddTracker(string trackerUrl, string localFile)
{
var torrentFile = Path.Combine(Directory.GetCurrentDirectory(), Guid.NewGuid().ToString() + ".torrent");
var info = new ProcessStartInfo
{
FileName = "transmission-edit",
Arguments = $"--add={trackerUrl} {localFile}"
};
RunToComplete(info);
return "OK";
}
public string PutLocalFile(string torrentBase64)
{
var torrentFile = Path.Combine(dataDir, Guid.NewGuid().ToString() + ".torrent");
File.WriteAllBytes(torrentFile, Convert.FromBase64String(torrentBase64));
return torrentFile;
}
public string Download(string localFile)
{
var peerPort = Environment.GetEnvironmentVariable("PEERPORT");
var info = new ProcessStartInfo
{
FileName = "transmission-cli",
Arguments = torrentFile
Arguments =
$"--port={peerPort} " +
$"--download-dir={dataDir} " +
$"{localFile}"
};
RunToComplete(info);
@ -86,6 +111,7 @@ namespace BittorrentDriver
private Process RunToComplete(ProcessStartInfo info)
{
log.Log($"Running: {info.FileName} ({info.Arguments})");
var process = Process.Start(info);
if (process == null) throw new Exception("Failed to start");
process.WaitForExit(TimeSpan.FromMinutes(3));
@ -96,7 +122,7 @@ namespace BittorrentDriver
{
try
{
var fileManager = new FileManager(log, dataDir);
var fileManager = new FileManager(log, dataDir, numberSubfolders: false);
var file = fileManager.GenerateFile(size.Bytes());
log.Log("Generated file: " + file.Filename);
return file;
@ -108,4 +134,10 @@ namespace BittorrentDriver
}
}
}
public class CreateTorrentResult
{
public string LocalFilePath { get; set; } = string.Empty;
public string TorrentBase64 { get; set; } = string.Empty;
}
}