diff --git a/Framework/FileUtils/FileManager.cs b/Framework/FileUtils/FileManager.cs index 0d0fb69..ff43c0c 100644 --- a/Framework/FileUtils/FileManager.cs +++ b/Framework/FileUtils/FileManager.cs @@ -31,9 +31,16 @@ namespace FileUtils public const int ChunkSize = 1024 * 1024 * 100; - public FileManager(ILog log, string rootFolder) + public FileManager(ILog log, string rootFolder, bool numberSubfolders = true) { - folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5")); + if (numberSubfolders) + { + folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5")); + } + else + { + folder = rootFolder; + } this.log = log; } diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index 529a48f..48e95d0 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -170,7 +170,7 @@ namespace KubernetesWorkflow var addresses = CreateContainerAddresses(startResult, r); log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}"); - return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses); + return new RunningContainer(log, Guid.NewGuid().ToString(), name, r, addresses); }).ToArray(); } diff --git a/Framework/KubernetesWorkflow/Types/RunningContainer.cs b/Framework/KubernetesWorkflow/Types/RunningContainer.cs index ad51265..dfbb102 100644 --- a/Framework/KubernetesWorkflow/Types/RunningContainer.cs +++ b/Framework/KubernetesWorkflow/Types/RunningContainer.cs @@ -7,8 +7,11 @@ namespace KubernetesWorkflow.Types { public class RunningContainer { - public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses) + private readonly ILog log; + + public RunningContainer(ILog log, string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses) { + this.log = log; Id = id; Name = name; Recipe = recipe; @@ -24,7 +27,7 @@ namespace KubernetesWorkflow.Types [JsonIgnore] public RunningPod RunningPod { get; internal set; } = null!; - public Address GetAddress(ILog log, string portTag) + public Address GetAddress(string portTag) { var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray(); if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag); diff --git a/ProjectPlugins/BittorrentPlugin/BittorrentContainerRecipe.cs b/ProjectPlugins/BittorrentPlugin/BittorrentContainerRecipe.cs index 8f5a61c..5e2baf6 100644 --- a/ProjectPlugins/BittorrentPlugin/BittorrentContainerRecipe.cs +++ b/ProjectPlugins/BittorrentPlugin/BittorrentContainerRecipe.cs @@ -6,7 +6,7 @@ namespace BittorrentPlugin public class BittorrentContainerRecipe : ContainerRecipeFactory { public override string AppName => "bittorrent"; - public override string Image => "thatbenbierens/bittorrentdriver:init6"; + public override string Image => "thatbenbierens/bittorrentdriver:init11"; public static string ApiPortTag = "API_PORT"; public static string TrackerPortTag = "TRACKER_PORT"; diff --git a/ProjectPlugins/BittorrentPlugin/BittorrentNode.cs b/ProjectPlugins/BittorrentPlugin/BittorrentNode.cs index e8ffeee..0bbcca5 100644 --- a/ProjectPlugins/BittorrentPlugin/BittorrentNode.cs +++ b/ProjectPlugins/BittorrentPlugin/BittorrentNode.cs @@ -1,9 +1,11 @@ using Core; using KubernetesWorkflow.Types; +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.ComponentModel; using System.Linq; +using System.Net; using System.Text; using System.Threading.Tasks; using Utils; @@ -13,34 +15,52 @@ namespace BittorrentPlugin public interface IBittorrentNode { string StartAsTracker(); - string CreateTorrent(ByteSize size, IBittorrentNode tracker); + string AddTracker(IBittorrentNode tracker, string localFile); + string PutFile(string base64); + string GetTrackerStats(); + CreateTorrentResult CreateTorrent(ByteSize size, IBittorrentNode tracker); string StartDaemon(); - string DownloadTorrent(string torrent); + string DownloadTorrent(string LocalFile); } public class BittorrentNode : IBittorrentNode { private readonly IPluginTools tools; private readonly RunningContainer container; + private readonly PodInfo podInfo; public BittorrentNode(IPluginTools tools, RunningContainer container) { this.tools = tools; this.container = container; + podInfo = tools.CreateWorkflow().GetPodInfo(container); } - public string CreateTorrent(ByteSize size, IBittorrentNode tracker) + public string StartAsTracker() { - var trackerUrl = ((BittorrentNode)tracker).TrackerAddress; + //TrackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag); var endpoint = GetEndpoint(); + return endpoint.HttpPutString("starttracker", GetTrackerAddress().Port.ToString()); + } - var torrent = endpoint.HttpPostJson("create", new CreateTorrentRequest + public string AddTracker(IBittorrentNode tracker, string localFile) + { + var endpoint = GetEndpoint(); + var trackerUrl = ((BittorrentNode)tracker).GetTrackerAddress(); + return endpoint.HttpPostJson("addtracker", new AddTrackerRequest { - Size = Convert.ToInt32(size.SizeInBytes), + LocalFile = localFile, TrackerUrl = $"{trackerUrl}/announce" }); + } - return torrent; + public string PutFile(string base64) + { + var endpoint = GetEndpoint(); + return endpoint.HttpPostJson("postfile", new PostFileRequest + { + Base64Content = base64 + }); } public string StartDaemon() @@ -50,43 +70,79 @@ namespace BittorrentPlugin return endpoint.HttpPutString("daemon", peerPortAddress.Port.ToString()); } - public string DownloadTorrent(string torrent) + public CreateTorrentResult CreateTorrent(ByteSize size, IBittorrentNode tracker) + { + var trackerUrl = ((BittorrentNode)tracker).GetTrackerAddress(); + var endpoint = GetEndpoint(); + + var json = endpoint.HttpPostJson("create", new CreateTorrentRequest + { + Size = Convert.ToInt32(size.SizeInBytes), + TrackerUrl = $"{trackerUrl}/announce" + }); + + return JsonConvert.DeserializeObject(json)!; + } + + public string DownloadTorrent(string localFile) { var endpoint = GetEndpoint(); return endpoint.HttpPostJson("download", new DownloadTorrentRequest { - TorrentBase64 = torrent + LocalFile = localFile }); } - public string StartAsTracker() + public string GetTrackerStats() { - TrackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag); - var endpoint = GetEndpoint(); - - var trackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag); - return endpoint.HttpPutString("tracker", trackerAddress.Port.ToString()); + //var http = tools.CreateHttp(TrackerAddress.ToString(), c => { }); + //var endpoint = http.CreateEndpoint(TrackerAddress, "/", container.Name); + //return endpoint.HttpGetString("stats"); + return "no"; } - public Address TrackerAddress { get; private set; } = new Address("", 0); + //public Address TrackerAddress { get; private set; } = new Address("", 0); - public class CreateTorrentRequest + public Address GetTrackerAddress() { - public int Size { get; set; } - public string TrackerUrl { get; set; } = string.Empty; - } - - public class DownloadTorrentRequest - { - public string TorrentBase64 { get; set; } = string.Empty; + var address = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag); + return new Address("http://" + podInfo.Ip, address.Port); } private IEndpoint GetEndpoint() { - var address = container.GetAddress(tools.GetLog(), BittorrentContainerRecipe.ApiPortTag); + var address = container.GetAddress(BittorrentContainerRecipe.ApiPortTag); var http = tools.CreateHttp(address.ToString(), c => { }); return http.CreateEndpoint(address, "/torrent/", container.Name); } } + + public class CreateTorrentRequest + { + public int Size { get; set; } + public string TrackerUrl { get; set; } = string.Empty; + } + + public class CreateTorrentResult + { + public string LocalFilePath { get; set; } = string.Empty; + public string TorrentBase64 { get; set; } = string.Empty; + } + + public class DownloadTorrentRequest + { + public string LocalFile { get; set; } = string.Empty; + } + + public class AddTrackerRequest + { + public string TrackerUrl { get; set; } = string.Empty; + public string LocalFile { get; set; } = string.Empty; + } + + public class PostFileRequest + { + public string Base64Content { get; set; } = string.Empty; + } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index e381414..077f386 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -19,7 +19,7 @@ namespace CodexContractsPlugin { var config = startupConfig.Get(); - var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag); + var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag); SetSchedulingAffinity(notIn: "false"); diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 704c74b..f54d885 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -189,7 +189,7 @@ namespace CodexPlugin private Address GetAddress() { - return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag); + return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag); } private string GetHttpId() diff --git a/ProjectPlugins/GethPlugin/GethNode.cs b/ProjectPlugins/GethPlugin/GethNode.cs index 08d0736..641aea4 100644 --- a/ProjectPlugins/GethPlugin/GethNode.cs +++ b/ProjectPlugins/GethPlugin/GethNode.cs @@ -57,7 +57,7 @@ namespace GethPlugin protected override NethereumInteraction StartInteraction() { - var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag); + var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag); var account = StartResult.Account; var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey); diff --git a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs index a66351c..70dd0b8 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs @@ -15,7 +15,7 @@ namespace MetricsPlugin { RunningContainer = runningContainer; log = tools.GetLog(); - var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag); + var address = RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag); endpoint = tools .CreateHttp(address.ToString()) .CreateEndpoint(address, "/api/v1/"); @@ -126,7 +126,7 @@ namespace MetricsPlugin private string GetInstanceNameForNode(IMetricsScrapeTarget target) { - return ScrapeTargetHelper.FormatTarget(log, target); + return ScrapeTargetHelper.FormatTarget(target); } private string GetInstanceStringForNode(IMetricsScrapeTarget target) diff --git a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs index 159947a..6b78c6e 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs @@ -72,15 +72,15 @@ namespace MetricsPlugin private string FormatTarget(IMetricsScrapeTarget target) { - return ScrapeTargetHelper.FormatTarget(tools.GetLog(), target); + return ScrapeTargetHelper.FormatTarget(target); } } public static class ScrapeTargetHelper { - public static string FormatTarget(ILog log, IMetricsScrapeTarget target) + public static string FormatTarget(IMetricsScrapeTarget target) { - var a = target.Container.GetAddress(log, target.MetricsPortTag); + var a = target.Container.GetAddress(target.MetricsPortTag); var host = a.Host.Replace("http://", "").Replace("https://", ""); return $"{host}:{a.Port}"; } diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index 57c383c..ca4c887 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -98,7 +98,7 @@ namespace ContinuousTests { cancelToken.ThrowIfCancellationRequested(); - var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag); + var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag); log.Log($"Checking {n.Container.Name} @ '{address}'..."); if (EnsureOnline(log, n)) diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 68ec76d..add827e 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -73,13 +73,21 @@ namespace CodexTests.BasicTests { var tracker = Ci.StartBittorrentNode(); var msg = tracker.StartAsTracker(); + msg = tracker.GetTrackerStats(); var seeder = Ci.StartBittorrentNode(); var torrent = seeder.CreateTorrent(10.MB(), tracker); + msg = seeder.AddTracker(tracker, torrent.LocalFilePath); msg = seeder.StartDaemon(); + Thread.Sleep(5000); + + msg = tracker.GetTrackerStats(); + var leecher = Ci.StartBittorrentNode(); - msg = leecher.DownloadTorrent(torrent); + var local = leecher.PutFile(torrent.TorrentBase64); + leecher.AddTracker(tracker, local); + msg = leecher.DownloadTorrent(local); var yay = 0; } diff --git a/Tools/AutoClient/Purchaser.cs b/Tools/AutoClient/Purchaser.cs index 41cc9c7..34e19e4 100644 --- a/Tools/AutoClient/Purchaser.cs +++ b/Tools/AutoClient/Purchaser.cs @@ -151,7 +151,7 @@ namespace AutoClient { try { - var sp = await GetStoragePurchase(pid)!; + var sp = (await GetStoragePurchase(pid))!; return sp.Request.Content.Cid; } catch (Exception ex) diff --git a/Tools/BittorrentDriver/Controllers/TorrentController.cs b/Tools/BittorrentDriver/Controllers/TorrentController.cs index 2753c4e..976b1f3 100644 --- a/Tools/BittorrentDriver/Controllers/TorrentController.cs +++ b/Tools/BittorrentDriver/Controllers/TorrentController.cs @@ -17,7 +17,7 @@ namespace BittorrentDriver.Controllers transmission = new Transmission(log); } - [HttpPut("tracker")] + [HttpPut("starttracker")] public string StartTracker([FromBody] int port) { return Try(() => @@ -27,6 +27,28 @@ namespace BittorrentDriver.Controllers }); } + [HttpPost("addtracker")] + public string AddTracker([FromBody] AddTrackerInput input) + { + return Try(() => + { + Log("Adding tracker: " + input.TrackerUrl + " - " + input.LocalFile); + return transmission.AddTracker(input.TrackerUrl, input.LocalFile); + }); + } + + [HttpPost("postfile")] + public string PostFile([FromBody] PostFileInput input) + { + return Try(() => + { + Log("Creating file.."); + var file = transmission.PutLocalFile(input.Base64Content); + Log("File: " + file); + return file; + }); + } + [HttpPut("daemon")] public string StartDaemon([FromBody] int peerPort) { @@ -38,7 +60,7 @@ namespace BittorrentDriver.Controllers } [HttpPost("create")] - public string CreateTorrent([FromBody] CreateTorrentInput input) + public CreateTorrentResult CreateTorrent([FromBody] CreateTorrentInput input) { return Try(() => { @@ -53,11 +75,11 @@ namespace BittorrentDriver.Controllers return Try(() => { Log("Downloading torrent..."); - return transmission.Download(input.TorrentBase64); + return transmission.Download(input.LocalFile); }); } - private string Try(Func value) + private T Try(Func value) { try { @@ -66,7 +88,7 @@ namespace BittorrentDriver.Controllers catch (Exception exc) { log.Error(exc.ToString()); - return exc.ToString(); + throw; } } @@ -82,8 +104,19 @@ namespace BittorrentDriver.Controllers public string TrackerUrl { get; set; } = string.Empty; } + public class AddTrackerInput + { + public string TrackerUrl { get; set; } = string.Empty; + public string LocalFile { get; set; } = string.Empty; + } + public class DownloadTorrentInput { - public string TorrentBase64 { get; set; } = string.Empty; + public string LocalFile { get; set; } = string.Empty; + } + + public class PostFileInput + { + public string Base64Content { get; set; } = string.Empty; } } diff --git a/Tools/BittorrentDriver/TorrentTracker.cs b/Tools/BittorrentDriver/TorrentTracker.cs index b66a1b7..c12ff01 100644 --- a/Tools/BittorrentDriver/TorrentTracker.cs +++ b/Tools/BittorrentDriver/TorrentTracker.cs @@ -12,7 +12,12 @@ namespace BittorrentDriver var info = new ProcessStartInfo { FileName = "bittorrent-tracker", - Arguments = $"--port {port} &", + Arguments = + $"--port {port} " + + $"--http " + + $"--stats " + + $"--interval=3000 " + // 3 seconds + $"&", UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, @@ -21,6 +26,18 @@ namespace BittorrentDriver process = Process.Start(info); if (process == null) return "Failed to start"; + process.OutputDataReceived += (sender, args) => + { + Console.WriteLine("STDOUT: " + args.Data); + }; + process.ErrorDataReceived += (sender, args) => + { + Console.WriteLine("STDERR: " + args.Data); + }; + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + Thread.Sleep(1000); if (process.HasExited) @@ -31,6 +48,5 @@ namespace BittorrentDriver } return "OK"; } - } } diff --git a/Tools/BittorrentDriver/Transmission.cs b/Tools/BittorrentDriver/Transmission.cs index 45fe0bb..33bcc89 100644 --- a/Tools/BittorrentDriver/Transmission.cs +++ b/Tools/BittorrentDriver/Transmission.cs @@ -19,17 +19,16 @@ namespace BittorrentDriver this.log = log; } - public string CreateNew(int size, string trackerUrl) + public CreateTorrentResult CreateNew(int size, string trackerUrl) { var file = CreateFile(size); - - var outFile = Path.Combine(Directory.GetCurrentDirectory(), Guid.NewGuid().ToString()); - + var outFile = Path.Combine(dataDir, Guid.NewGuid().ToString()); var base64 = CreateTorrentFile(file, outFile, trackerUrl); - - if (File.Exists(outFile)) File.Delete(outFile); - - return base64; + return new CreateTorrentResult + { + LocalFilePath = outFile, + TorrentBase64 = base64 + }; } public string StartDaemon(int peerPort) @@ -37,22 +36,48 @@ namespace BittorrentDriver var info = new ProcessStartInfo { FileName = "transmission-daemon", - Arguments = $"--peerport={peerPort} --download-dir={dataDir}" + Arguments = $"--peerport={peerPort} " + + $"--download-dir={dataDir} " + + $"--watch-dir={dataDir} " + + $"--no-global-seedratio " + + $"--bind-address-ipv4=0.0.0.0 " + + $"--dht" + }; + RunToComplete(info); + + return "OK"; + } + + public string AddTracker(string trackerUrl, string localFile) + { + var info = new ProcessStartInfo + { + FileName = "transmission-edit", + Arguments = $"--add={trackerUrl} {localFile}" }; RunToComplete(info); return "OK"; } - public string Download(string torrentBase64) + public string PutLocalFile(string torrentBase64) { - var torrentFile = Path.Combine(Directory.GetCurrentDirectory(), Guid.NewGuid().ToString() + ".torrent"); + var torrentFile = Path.Combine(dataDir, Guid.NewGuid().ToString() + ".torrent"); File.WriteAllBytes(torrentFile, Convert.FromBase64String(torrentBase64)); + return torrentFile; + } + + public string Download(string localFile) + { + var peerPort = Environment.GetEnvironmentVariable("PEERPORT"); var info = new ProcessStartInfo { FileName = "transmission-cli", - Arguments = torrentFile + Arguments = + $"--port={peerPort} " + + $"--download-dir={dataDir} " + + $"{localFile}" }; RunToComplete(info); @@ -86,6 +111,7 @@ namespace BittorrentDriver private Process RunToComplete(ProcessStartInfo info) { + log.Log($"Running: {info.FileName} ({info.Arguments})"); var process = Process.Start(info); if (process == null) throw new Exception("Failed to start"); process.WaitForExit(TimeSpan.FromMinutes(3)); @@ -96,7 +122,7 @@ namespace BittorrentDriver { try { - var fileManager = new FileManager(log, dataDir); + var fileManager = new FileManager(log, dataDir, numberSubfolders: false); var file = fileManager.GenerateFile(size.Bytes()); log.Log("Generated file: " + file.Filename); return file; @@ -108,4 +134,10 @@ namespace BittorrentDriver } } } + + public class CreateTorrentResult + { + public string LocalFilePath { get; set; } = string.Empty; + public string TorrentBase64 { get; set; } = string.Empty; + } }