improvements

Framework/FileUtils/FileManager.cs

@@ -31,9 +31,16 @@ namespace FileUtils
 
         public const int ChunkSize = 1024 * 1024 * 100;
 
-        public FileManager(ILog log, string rootFolder)
+        public FileManager(ILog log, string rootFolder, bool numberSubfolders = true)
         {
-            folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
+            if (numberSubfolders)
+            {
+                folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
+            }
+            else
+            {
+                folder = rootFolder;
+            }
 
             this.log = log;
         }

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -170,7 +170,7 @@ namespace KubernetesWorkflow
                 var addresses = CreateContainerAddresses(startResult, r);
                 log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
 
-                return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
+                return new RunningContainer(log, Guid.NewGuid().ToString(), name, r, addresses);
 
             }).ToArray();
         }

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -7,8 +7,11 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningContainer
     {
-        public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        private readonly ILog log;
+
+        public RunningContainer(ILog log, string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
         {
+            this.log = log;
             Id = id;
             Name = name;
             Recipe = recipe;
@@ -24,7 +27,7 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
 
-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
             if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);

ProjectPlugins/BittorrentPlugin/BittorrentContainerRecipe.cs

@@ -6,7 +6,7 @@ namespace BittorrentPlugin
     public class BittorrentContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "bittorrent";
-        public override string Image => "thatbenbierens/bittorrentdriver:init6";
+        public override string Image => "thatbenbierens/bittorrentdriver:init11";
 
         public static string ApiPortTag = "API_PORT";
         public static string TrackerPortTag = "TRACKER_PORT";

ProjectPlugins/BittorrentPlugin/BittorrentNode.cs

@@ -1,9 +1,11 @@
 using Core;
 using KubernetesWorkflow.Types;
+using Newtonsoft.Json;
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Linq;
+using System.Net;
 using System.Text;
 using System.Threading.Tasks;
 using Utils;
@@ -13,34 +15,52 @@ namespace BittorrentPlugin
     public interface IBittorrentNode
     {
         string StartAsTracker();
-        string CreateTorrent(ByteSize size, IBittorrentNode tracker);
+        string AddTracker(IBittorrentNode tracker, string localFile);
+        string PutFile(string base64);
+        string GetTrackerStats();
+        CreateTorrentResult CreateTorrent(ByteSize size, IBittorrentNode tracker);
         string StartDaemon();
-        string DownloadTorrent(string torrent);
+        string DownloadTorrent(string LocalFile);
     }
 
     public class BittorrentNode : IBittorrentNode
     {
         private readonly IPluginTools tools;
         private readonly RunningContainer container;
+        private readonly PodInfo podInfo;
 
         public BittorrentNode(IPluginTools tools, RunningContainer container)
         {
             this.tools = tools;
             this.container = container;
+            podInfo = tools.CreateWorkflow().GetPodInfo(container);
         }
 
-        public string CreateTorrent(ByteSize size, IBittorrentNode tracker)
+        public string StartAsTracker()
         {
-            var trackerUrl = ((BittorrentNode)tracker).TrackerAddress;
+            //TrackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
             var endpoint = GetEndpoint();
+            return endpoint.HttpPutString("starttracker", GetTrackerAddress().Port.ToString());
+        }
 
-            var torrent = endpoint.HttpPostJson("create", new CreateTorrentRequest
+        public string AddTracker(IBittorrentNode tracker, string localFile)
+        {
+            var endpoint = GetEndpoint();
+            var trackerUrl = ((BittorrentNode)tracker).GetTrackerAddress();
+            return endpoint.HttpPostJson("addtracker", new AddTrackerRequest
             {
-                Size = Convert.ToInt32(size.SizeInBytes),
+                LocalFile = localFile,
                 TrackerUrl = $"{trackerUrl}/announce"
             });
+        }
 
-            return torrent;
+        public string PutFile(string base64)
+        {
+            var endpoint = GetEndpoint();
+            return endpoint.HttpPostJson("postfile", new PostFileRequest
+            {
+                Base64Content = base64
+            });
         }
 
         public string StartDaemon()
@@ -50,43 +70,79 @@ namespace BittorrentPlugin
             return endpoint.HttpPutString("daemon", peerPortAddress.Port.ToString());
         }
 
-        public string DownloadTorrent(string torrent)
+        public CreateTorrentResult CreateTorrent(ByteSize size, IBittorrentNode tracker)
+        {
+            var trackerUrl = ((BittorrentNode)tracker).GetTrackerAddress();
+            var endpoint = GetEndpoint();
+
+            var json = endpoint.HttpPostJson("create", new CreateTorrentRequest
+            {
+                Size = Convert.ToInt32(size.SizeInBytes),
+                TrackerUrl = $"{trackerUrl}/announce"
+            });
+
+            return JsonConvert.DeserializeObject<CreateTorrentResult>(json)!;
+        }
+
+        public string DownloadTorrent(string localFile)
         {
             var endpoint = GetEndpoint();
 
             return endpoint.HttpPostJson("download", new DownloadTorrentRequest
             {
-                TorrentBase64 = torrent
+                LocalFile = localFile
             });
         }
 
-        public string StartAsTracker()
+        public string GetTrackerStats()
         {
-            TrackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
+            //var http = tools.CreateHttp(TrackerAddress.ToString(), c => { });
-            var endpoint = GetEndpoint();
+            //var endpoint = http.CreateEndpoint(TrackerAddress, "/", container.Name);
-
+            //return endpoint.HttpGetString("stats");
-            var trackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
+            return "no";
-            return endpoint.HttpPutString("tracker", trackerAddress.Port.ToString());
         }
 
-        public Address TrackerAddress { get; private set; } = new Address("", 0);
+        //public Address TrackerAddress { get; private set; } = new Address("", 0);
 
-        public class CreateTorrentRequest
+        public Address GetTrackerAddress()
         {
-            public int Size { get; set; }
+            var address = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
-            public string TrackerUrl { get; set; } = string.Empty;
+            return new Address("http://" + podInfo.Ip, address.Port);
-        }
-
-        public class DownloadTorrentRequest
-        {
-            public string TorrentBase64 { get; set; } = string.Empty;
         }
 
         private IEndpoint GetEndpoint()
         {
-            var address = container.GetAddress(tools.GetLog(), BittorrentContainerRecipe.ApiPortTag);
+            var address = container.GetAddress(BittorrentContainerRecipe.ApiPortTag);
             var http = tools.CreateHttp(address.ToString(), c => { });
             return http.CreateEndpoint(address, "/torrent/", container.Name);
         }
     }
+
+    public class CreateTorrentRequest
+    {
+        public int Size { get; set; }
+        public string TrackerUrl { get; set; } = string.Empty;
+    }
+
+    public class CreateTorrentResult
+    {
+        public string LocalFilePath { get; set; } = string.Empty;
+        public string TorrentBase64 { get; set; } = string.Empty;
+    }
+
+    public class DownloadTorrentRequest
+    {
+        public string LocalFile { get; set; } = string.Empty;
+    }
+
+    public class AddTrackerRequest
+    {
+        public string TrackerUrl { get; set; } = string.Empty;
+        public string LocalFile { get; set; } = string.Empty;
+    }
+
+    public class PostFileRequest
+    {
+        public string Base64Content { get; set; } = string.Empty;
+    }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -19,7 +19,7 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+            var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
 
             SetSchedulingAffinity(notIn: "false");
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -189,7 +189,7 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private string GetHttpId()

ProjectPlugins/GethPlugin/GethNode.cs

@@ -57,7 +57,7 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = StartResult.Account;
 
             var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -15,7 +15,7 @@ namespace MetricsPlugin
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
-            var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag);
+            var address = RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag);
             endpoint = tools
                 .CreateHttp(address.ToString())
                 .CreateEndpoint(address, "/api/v1/");
@@ -126,7 +126,7 @@ namespace MetricsPlugin
 
         private string GetInstanceNameForNode(IMetricsScrapeTarget target)
         {
-            return ScrapeTargetHelper.FormatTarget(log, target);
+            return ScrapeTargetHelper.FormatTarget(target);
         }
 
         private string GetInstanceStringForNode(IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -72,15 +72,15 @@ namespace MetricsPlugin
 
         private string FormatTarget(IMetricsScrapeTarget target)
         {
-            return ScrapeTargetHelper.FormatTarget(tools.GetLog(), target);
+            return ScrapeTargetHelper.FormatTarget(target);
         }
     }
 
     public static class ScrapeTargetHelper
     {
-        public static string FormatTarget(ILog log, IMetricsScrapeTarget target)
+        public static string FormatTarget(IMetricsScrapeTarget target)
         {
-            var a = target.Container.GetAddress(log, target.MetricsPortTag);
+            var a = target.Container.GetAddress(target.MetricsPortTag);
             var host = a.Host.Replace("http://", "").Replace("https://", "");
             return $"{host}:{a.Port}";
         }

Tests/CodexContinuousTests/StartupChecker.cs

@@ -98,7 +98,7 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag);
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
                 log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -73,13 +73,21 @@ namespace CodexTests.BasicTests
         {
             var tracker = Ci.StartBittorrentNode();
             var msg = tracker.StartAsTracker();
+            msg = tracker.GetTrackerStats();
 
             var seeder = Ci.StartBittorrentNode();
             var torrent = seeder.CreateTorrent(10.MB(), tracker);
+            msg = seeder.AddTracker(tracker, torrent.LocalFilePath);
             msg = seeder.StartDaemon();
 
+            Thread.Sleep(5000);
+
+            msg = tracker.GetTrackerStats();
+
             var leecher = Ci.StartBittorrentNode();
-            msg = leecher.DownloadTorrent(torrent);
+            var local = leecher.PutFile(torrent.TorrentBase64);
+            leecher.AddTracker(tracker, local);
+            msg = leecher.DownloadTorrent(local);
 
             var yay = 0;
         }

Tools/AutoClient/Purchaser.cs

@@ -151,7 +151,7 @@ namespace AutoClient
         {
             try
             {
-                var sp = await GetStoragePurchase(pid)!;
+                var sp = (await GetStoragePurchase(pid))!;
                 return sp.Request.Content.Cid;
             }
             catch (Exception ex)

Tools/BittorrentDriver/Controllers/TorrentController.cs

@@ -17,7 +17,7 @@ namespace BittorrentDriver.Controllers
             transmission = new Transmission(log);
         }
 
-        [HttpPut("tracker")]
+        [HttpPut("starttracker")]
         public string StartTracker([FromBody] int port)
         {
             return Try(() =>
@@ -27,6 +27,28 @@ namespace BittorrentDriver.Controllers
             });
         }
 
+        [HttpPost("addtracker")]
+        public string AddTracker([FromBody] AddTrackerInput input)
+        {
+            return Try(() =>
+            {
+                Log("Adding tracker: " + input.TrackerUrl + " - " + input.LocalFile);
+                return transmission.AddTracker(input.TrackerUrl, input.LocalFile);
+            });
+        }
+
+        [HttpPost("postfile")]
+        public string PostFile([FromBody] PostFileInput input)
+        {
+            return Try(() =>
+            {
+                Log("Creating file..");
+                var file = transmission.PutLocalFile(input.Base64Content);
+                Log("File: " + file);
+                return file;
+            });
+        }
+
         [HttpPut("daemon")]
         public string StartDaemon([FromBody] int peerPort)
         {
@@ -38,7 +60,7 @@ namespace BittorrentDriver.Controllers
         }
 
         [HttpPost("create")]
-        public string CreateTorrent([FromBody] CreateTorrentInput input)
+        public CreateTorrentResult CreateTorrent([FromBody] CreateTorrentInput input)
         {
             return Try(() =>
             {
@@ -53,11 +75,11 @@ namespace BittorrentDriver.Controllers
             return Try(() =>
             {
                 Log("Downloading torrent...");
-                return transmission.Download(input.TorrentBase64);
+                return transmission.Download(input.LocalFile);
             });
         }
 
-        private string Try(Func<string> value)
+        private T Try<T>(Func<T> value)
         {
             try
             {
@@ -66,7 +88,7 @@ namespace BittorrentDriver.Controllers
             catch (Exception exc)
             {
                 log.Error(exc.ToString());
-                return exc.ToString();
+                throw;
             }
         }
 
@@ -82,8 +104,19 @@ namespace BittorrentDriver.Controllers
         public string TrackerUrl { get; set; } = string.Empty;
     }
 
+    public class AddTrackerInput
+    {
+        public string TrackerUrl { get; set; } = string.Empty;
+        public string LocalFile { get; set; } = string.Empty;
+    }
+
     public class DownloadTorrentInput
     {
-        public string TorrentBase64 { get; set; } = string.Empty;
+        public string LocalFile { get; set; } = string.Empty;
+    }
+
+    public class PostFileInput
+    {
+        public string Base64Content { get; set; } = string.Empty;
     }
 }

Tools/BittorrentDriver/TorrentTracker.cs

@@ -12,7 +12,12 @@ namespace BittorrentDriver
             var info = new ProcessStartInfo
             {
                 FileName = "bittorrent-tracker",
-                Arguments = $"--port {port} &",
+                Arguments = 
+                    $"--port {port} " +
+                    $"--http " +
+                    $"--stats " +
+                    $"--interval=3000 " + // 3 seconds
+                    $"&",
                 UseShellExecute = false,
                 RedirectStandardOutput = true,
                 RedirectStandardError = true,
@@ -21,6 +26,18 @@ namespace BittorrentDriver
             process = Process.Start(info);
             if (process == null) return "Failed to start";
 
+            process.OutputDataReceived += (sender, args) =>
+            {
+                Console.WriteLine("STDOUT: " + args.Data);
+            };
+            process.ErrorDataReceived += (sender, args) =>
+            {
+                Console.WriteLine("STDERR: " + args.Data);
+            };
+
+            process.BeginOutputReadLine();
+            process.BeginErrorReadLine();
+
             Thread.Sleep(1000);
 
             if (process.HasExited)
@@ -31,6 +48,5 @@ namespace BittorrentDriver
             }
             return "OK";
         }
-
     }
 }

Tools/BittorrentDriver/Transmission.cs

@@ -19,17 +19,16 @@ namespace BittorrentDriver
             this.log = log;
         }
 
-        public string CreateNew(int size, string trackerUrl)
+        public CreateTorrentResult CreateNew(int size, string trackerUrl)
         {
             var file = CreateFile(size);
-
+            var outFile = Path.Combine(dataDir, Guid.NewGuid().ToString());
-            var outFile = Path.Combine(Directory.GetCurrentDirectory(), Guid.NewGuid().ToString());
-
             var base64 = CreateTorrentFile(file, outFile, trackerUrl);
-
+            return new CreateTorrentResult
-            if (File.Exists(outFile)) File.Delete(outFile);
+            {
-
+                LocalFilePath = outFile,
-            return base64;
+                TorrentBase64 = base64
+            };
         }
 
         public string StartDaemon(int peerPort)
@@ -37,22 +36,48 @@ namespace BittorrentDriver
             var info = new ProcessStartInfo
             {
                 FileName = "transmission-daemon",
-                Arguments = $"--peerport={peerPort} --download-dir={dataDir}"
+                Arguments = $"--peerport={peerPort} " +
+                $"--download-dir={dataDir} " +
+                $"--watch-dir={dataDir} " +
+                $"--no-global-seedratio " +
+                $"--bind-address-ipv4=0.0.0.0 " +
+                $"--dht"
             };
             RunToComplete(info);
 
             return "OK";
         }
         
-        public string Download(string torrentBase64)
+        public string AddTracker(string trackerUrl, string localFile)
         {
-            var torrentFile = Path.Combine(Directory.GetCurrentDirectory(), Guid.NewGuid().ToString() + ".torrent");
+            var info = new ProcessStartInfo
+            {
+                FileName = "transmission-edit",
+                Arguments = $"--add={trackerUrl} {localFile}"
+            };
+            RunToComplete(info);
+
+            return "OK";
+        }
+
+        public string PutLocalFile(string torrentBase64)
+        {
+            var torrentFile = Path.Combine(dataDir, Guid.NewGuid().ToString() + ".torrent");
             File.WriteAllBytes(torrentFile, Convert.FromBase64String(torrentBase64));
+            return torrentFile;
+        }
+
+        public string Download(string localFile)
+        {
+            var peerPort = Environment.GetEnvironmentVariable("PEERPORT");
 
             var info = new ProcessStartInfo
             {
                 FileName = "transmission-cli",
-                Arguments = torrentFile
+                Arguments = 
+                    $"--port={peerPort} " +
+                    $"--download-dir={dataDir} " +
+                    $"{localFile}"
             };
             RunToComplete(info);
 
@@ -86,6 +111,7 @@ namespace BittorrentDriver
 
         private Process RunToComplete(ProcessStartInfo info)
         {
+            log.Log($"Running: {info.FileName} ({info.Arguments})");
             var process = Process.Start(info);
             if (process == null) throw new Exception("Failed to start");
             process.WaitForExit(TimeSpan.FromMinutes(3));
@@ -96,7 +122,7 @@ namespace BittorrentDriver
         {
             try
             {
-                var fileManager = new FileManager(log, dataDir);
+                var fileManager = new FileManager(log, dataDir, numberSubfolders: false);
                 var file = fileManager.GenerateFile(size.Bytes());
                 log.Log("Generated file: " + file.Filename);
                 return file;
@@ -108,4 +134,10 @@ namespace BittorrentDriver
             }
         }
     }
+
+    public class CreateTorrentResult
+    {
+        public string LocalFilePath { get; set; } = string.Empty;
+        public string TorrentBase64 { get; set; } = string.Empty;
+    } 
 }