trying to get back tracker stats

Framework/Core/Endpoint.cs

@@ -16,6 +16,7 @@ namespace Core
         TResponse HttpPostString<TResponse>(string route, string body);
         string HttpPostStream(string route, Stream stream);
         Stream HttpGetStream(string route);
+        string HttpPutString(string route, string body);
         T Deserialize<T>(string json);
     }
 
@@ -114,6 +115,16 @@ namespace Core
             }, $"HTTP-GET-STREAM: {route}");
         }
 
+        public string HttpPutString(string route, string body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = Time.Wait(client.PutAsync(GetUrl() + route,
+                    new StringContent(body, MediaTypeHeaderValue.Parse("application/json"))));
+                return Time.Wait(response.Content.ReadAsStringAsync());
+            }, $"HTTP-PUT-STR: {route}");
+        }
+
         public T Deserialize<T>(string json)
         {
             var errors = new List<string>();

Framework/FileUtils/FileManager.cs

@@ -31,9 +31,16 @@ namespace FileUtils
 
         public const int ChunkSize = 1024 * 1024 * 100;
 
-        public FileManager(ILog log, string rootFolder)
+        public FileManager(ILog log, string rootFolder, bool numberSubfolders = true)
         {
-            folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
+            if (numberSubfolders)
+            {
+                folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
+            }
+            else
+            {
+                folder = rootFolder;
+            }
 
             this.log = log;
         }

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -170,7 +170,7 @@ namespace KubernetesWorkflow
                 var addresses = CreateContainerAddresses(startResult, r);
                 log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
 
-                return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
+                return new RunningContainer(log, Guid.NewGuid().ToString(), name, r, addresses);
 
             }).ToArray();
         }

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -7,8 +7,11 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningContainer
     {
-        public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        private readonly ILog log;
+
+        public RunningContainer(ILog log, string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
         {
+            this.log = log;
             Id = id;
             Name = name;
             Recipe = recipe;
@@ -24,7 +27,7 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
 
-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
             if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);

ProjectPlugins/BittorrentPlugin/BittorrentContainerRecipe.cs

@@ -0,0 +1,22 @@
+using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+
+namespace BittorrentPlugin
+{
+    public class BittorrentContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "bittorrent";
+        public override string Image => "thatbenbierens/bittorrentdriver:init12";
+
+        public static string ApiPortTag = "API_PORT";
+        public static string TrackerPortTag = "TRACKER_PORT";
+        public static string PeerPortTag = "PEER_PORT";
+
+        protected override void Initialize(StartupConfig config)
+        {
+            AddInternalPortAndVar("TRACKERPORT", TrackerPortTag);
+            AddInternalPortAndVar("PEERPORT", PeerPortTag);
+            AddExposedPortAndVar("APIPORT", ApiPortTag);
+        }
+    }
+}

ProjectPlugins/BittorrentPlugin/BittorrentNode.cs

@@ -0,0 +1,146 @@
+using Core;
+using KubernetesWorkflow.Types;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace BittorrentPlugin
+{
+    public interface IBittorrentNode
+    {
+        string StartAsTracker();
+        string AddTracker(IBittorrentNode tracker, string localFile);
+        string PutFile(string base64);
+        string GetTrackerStats();
+        CreateTorrentResult CreateTorrent(ByteSize size, IBittorrentNode tracker);
+        string StartDaemon();
+        string DownloadTorrent(string LocalFile);
+    }
+
+    public class BittorrentNode : IBittorrentNode
+    {
+        private readonly IPluginTools tools;
+        private readonly RunningContainer container;
+        private readonly PodInfo podInfo;
+
+        public BittorrentNode(IPluginTools tools, RunningContainer container)
+        {
+            this.tools = tools;
+            this.container = container;
+            podInfo = tools.CreateWorkflow().GetPodInfo(container);
+        }
+
+        public string StartAsTracker()
+        {
+            //TrackerAddress = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
+            var endpoint = GetEndpoint();
+            return endpoint.HttpPutString("starttracker", GetTrackerAddress().Port.ToString());
+        }
+
+        public string AddTracker(IBittorrentNode tracker, string localFile)
+        {
+            var endpoint = GetEndpoint();
+            var trackerUrl = ((BittorrentNode)tracker).GetTrackerAddress();
+            return endpoint.HttpPostJson("addtracker", new AddTrackerRequest
+            {
+                LocalFile = localFile,
+                TrackerUrl = $"{trackerUrl}/announce"
+            });
+        }
+
+        public string PutFile(string base64)
+        {
+            var endpoint = GetEndpoint();
+            return endpoint.HttpPostJson("postfile", new PostFileRequest
+            {
+                Base64Content = base64
+            });
+        }
+
+        public string StartDaemon()
+        {
+            var endpoint = GetEndpoint();
+            var peerPortAddress = container.GetInternalAddress(BittorrentContainerRecipe.PeerPortTag);
+            return endpoint.HttpPutString("daemon", peerPortAddress.Port.ToString());
+        }
+
+        public CreateTorrentResult CreateTorrent(ByteSize size, IBittorrentNode tracker)
+        {
+            var trackerUrl = ((BittorrentNode)tracker).GetTrackerAddress();
+            var endpoint = GetEndpoint();
+
+            var json = endpoint.HttpPostJson("create", new CreateTorrentRequest
+            {
+                Size = Convert.ToInt32(size.SizeInBytes),
+                TrackerUrl = $"{trackerUrl}/announce"
+            });
+
+            return JsonConvert.DeserializeObject<CreateTorrentResult>(json)!;
+        }
+
+        public string DownloadTorrent(string localFile)
+        {
+            var endpoint = GetEndpoint();
+
+            return endpoint.HttpPostJson("download", new DownloadTorrentRequest
+            {
+                LocalFile = localFile
+            });
+        }
+
+        public string GetTrackerStats()
+        {
+            var endpoint = GetEndpoint();
+            return endpoint.HttpGetString("stats");
+        }
+
+        //public Address TrackerAddress { get; private set; } = new Address("", 0);
+
+        public Address GetTrackerAddress()
+        {
+            var address = container.GetInternalAddress(BittorrentContainerRecipe.TrackerPortTag);
+            return new Address("http://" + podInfo.Ip, address.Port);
+        }
+
+        private IEndpoint GetEndpoint()
+        {
+            var address = container.GetAddress(BittorrentContainerRecipe.ApiPortTag);
+            var http = tools.CreateHttp(address.ToString(), c => { });
+            return http.CreateEndpoint(address, "/torrent/", container.Name);
+        }
+    }
+
+    public class CreateTorrentRequest
+    {
+        public int Size { get; set; }
+        public string TrackerUrl { get; set; } = string.Empty;
+    }
+
+    public class CreateTorrentResult
+    {
+        public string LocalFilePath { get; set; } = string.Empty;
+        public string TorrentBase64 { get; set; } = string.Empty;
+    }
+
+    public class DownloadTorrentRequest
+    {
+        public string LocalFile { get; set; } = string.Empty;
+    }
+
+    public class AddTrackerRequest
+    {
+        public string TrackerUrl { get; set; } = string.Empty;
+        public string LocalFile { get; set; } = string.Empty;
+    }
+
+    public class PostFileRequest
+    {
+        public string Base64Content { get; set; } = string.Empty;
+    }
+}

ProjectPlugins/BittorrentPlugin/BittorrentPlugin.cs

@@ -0,0 +1,34 @@
+using Core;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+
+namespace BittorrentPlugin
+{
+    public class BittorrentPlugin : IProjectPlugin
+    {
+        private readonly IPluginTools tools;
+
+        public BittorrentPlugin(IPluginTools tools)
+        {
+            this.tools = tools;
+        }
+
+        public void Announce()
+        {
+            tools.GetLog().Log("Loaded Bittorrent plugin");
+        }
+
+        public void Decommission()
+        {
+        }
+
+        public IBittorrentNode StartNode()
+        {
+            var flow = tools.CreateWorkflow();
+            var pod = flow.Start(1, new BittorrentContainerRecipe(), new StartupConfig()).WaitForOnline();
+            var container = pod.Containers.Single();
+
+            return new BittorrentNode(tools, container);
+        }
+    }
+}

ProjectPlugins/BittorrentPlugin/BittorrentPlugin.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+	<TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/BittorrentPlugin/CoreInterfaceExtensions.cs

@@ -0,0 +1,17 @@
+using Core;
+
+namespace BittorrentPlugin
+{
+    public static class CoreInterfaceExtensions
+    {
+        public static IBittorrentNode StartBittorrentNode(this CoreInterface ci)
+        {
+            return Plugin(ci).StartNode();
+        }
+
+        private static BittorrentPlugin Plugin(CoreInterface ci)
+        {
+            return ci.GetPlugin<BittorrentPlugin>();
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -19,7 +19,7 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+            var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
 
             SetSchedulingAffinity(notIn: "false");
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -189,7 +189,7 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private string GetHttpId()

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-64b82de-dist-tests";
-
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-656ce37-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";

ProjectPlugins/GethPlugin/GethNode.cs

@@ -57,7 +57,7 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = StartResult.Account;
 
             var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -15,7 +15,7 @@ namespace MetricsPlugin
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
-            var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag);
+            var address = RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag);
             endpoint = tools
                 .CreateHttp(address.ToString())
                 .CreateEndpoint(address, "/api/v1/");
@@ -126,7 +126,7 @@ namespace MetricsPlugin
 
         private string GetInstanceNameForNode(IMetricsScrapeTarget target)
         {
-            return ScrapeTargetHelper.FormatTarget(log, target);
+            return ScrapeTargetHelper.FormatTarget(target);
         }
 
         private string GetInstanceStringForNode(IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -72,15 +72,15 @@ namespace MetricsPlugin
 
         private string FormatTarget(IMetricsScrapeTarget target)
         {
-            return ScrapeTargetHelper.FormatTarget(tools.GetLog(), target);
+            return ScrapeTargetHelper.FormatTarget(target);
         }
     }
 
     public static class ScrapeTargetHelper
     {
-        public static string FormatTarget(ILog log, IMetricsScrapeTarget target)
+        public static string FormatTarget(IMetricsScrapeTarget target)
         {
-            var a = target.Container.GetAddress(log, target.MetricsPortTag);
+            var a = target.Container.GetAddress(target.MetricsPortTag);
             var host = a.Host.Replace("http://", "").Replace("https://", "");
             return $"{host}:{a.Port}";
         }

Tests/CodexContinuousTests/StartupChecker.cs

@@ -98,7 +98,7 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag);
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
                 log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -2,6 +2,7 @@
 using DistTestCore;
 using GethPlugin;
 using MetricsPlugin;
+using BittorrentPlugin;
 using NUnit.Framework;
 using Utils;
 
@@ -66,5 +67,29 @@ namespace CodexTests.BasicTests
             Assert.That(bootN, Is.EqualTo(followN));
             Assert.That(discN, Is.LessThan(bootN));
         }
+
+        [Test]
+        public void BittorrentPluginTest()
+        {
+            var tracker = Ci.StartBittorrentNode();
+            var msg = tracker.StartAsTracker();
+            msg = tracker.GetTrackerStats();
+
+            var seeder = Ci.StartBittorrentNode();
+            var torrent = seeder.CreateTorrent(10.MB(), tracker);
+            msg = seeder.AddTracker(tracker, torrent.LocalFilePath);
+            msg = seeder.StartDaemon();
+
+            Thread.Sleep(5000);
+
+            msg = tracker.GetTrackerStats();
+
+            var leecher = Ci.StartBittorrentNode();
+            var local = leecher.PutFile(torrent.TorrentBase64);
+            leecher.AddTracker(tracker, local);
+            msg = leecher.DownloadTorrent(local);
+
+            var yay = 0;
+        }
     }
 }

Tests/CodexTests/CodexTests.csproj

@@ -14,6 +14,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\BittorrentPlugin\BittorrentPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexDiscordBotPlugin\CodexDiscordBotPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />

Tools/AutoClient/App.cs

@@ -0,0 +1,40 @@
+using Logging;
+
+namespace AutoClient
+{
+    public class App
+    {
+        public App(Configuration config)
+        {
+            Config = config;
+
+            Log = new LogSplitter(
+                new FileLog(Path.Combine(config.LogPath, "autoclient")),
+                new ConsoleLog()
+            );
+
+            Generator = CreateGenerator();
+            CidRepo = new CidRepo(config);
+            Performance = new Performance(new LogSplitter(
+                new FileLog(Path.Combine(config.LogPath, "performance")),
+                new ConsoleLog()
+            ));
+        }
+
+        public Configuration Config { get; }
+        public ILog Log { get; }
+        public IFileGenerator Generator { get; }
+        public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
+        public CidRepo CidRepo { get; }
+        public Performance Performance { get; }
+
+        private IFileGenerator CreateGenerator()
+        {
+            if (Config.FileSizeMb > 0)
+            {
+                return new RandomFileGenerator(Config, Log);
+            }
+            return new ImageGenerator(Log);
+        }
+    }
+}

Tools/AutoClient/CidRepo.cs

@@ -0,0 +1,85 @@
+namespace AutoClient
+{
+    public class CidRepo
+    {
+        private readonly Random random = new Random();
+        private readonly object _lock = new object();
+        private readonly List<CidEntry> entries = new List<CidEntry>();
+        private readonly Configuration config;
+
+        public CidRepo(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public void Add(string nodeId, string cid, long knownSize)
+        {
+            lock (_lock)
+            {
+                entries.Add(new CidEntry(nodeId, cid, knownSize));
+            }
+        }
+
+        public void AddEncoded(string originalCid, string encodedCid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == originalCid);
+                if (entry == null) return;
+
+                entry.Encoded = encodedCid;
+            }
+        }
+
+        public string? GetForeignCid(string myNodeId)
+        {
+            lock (_lock)
+            {
+                while (true)
+                {
+                    if (!entries.Any()) return null;
+                    var available = entries.Where(e => e.NodeId != myNodeId).ToArray();
+                    if (!available.Any()) return null;
+
+                    var i = random.Next(0, available.Length);
+                    var entry = available[i];
+
+                    if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromMinutes(config.ContractDurationMinutes)))
+                    {
+                        entries.Remove(entry);
+                    }
+                    else
+                    {
+                        return entry.Cid;
+                    }
+                }
+            }
+        }
+
+        public long? GetSizeForCid(string cid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == cid);
+                if (entry == null) return null;
+                return entry.KnownSize;
+            }
+        }
+    }
+
+    public class CidEntry
+    {
+        public CidEntry(string nodeId, string cid, long knownSize)
+        {
+            NodeId = nodeId;
+            Cid = cid;
+            KnownSize = knownSize;
+        }
+
+        public string NodeId { get; }
+        public string Cid { get; }
+        public string Encoded { get; set; } = string.Empty;
+        public long KnownSize { get; }
+        public DateTime CreatedUtc { get; } = DateTime.UtcNow;
+    }
+}

Tools/AutoClient/CodexUser.cs

@@ -0,0 +1,57 @@
+using CodexOpenApi;
+using Logging;
+using Utils;
+
+namespace AutoClient
+{
+    public class CodexUser
+    {
+        private readonly App app;
+        private readonly CodexApi codex;
+        private readonly HttpClient client;
+        private readonly Address address;
+        private readonly List<Purchaser> purchasers = new List<Purchaser>();
+        private Task starterTask = Task.CompletedTask;
+        private readonly string nodeId = Guid.NewGuid().ToString();
+
+        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
+        {
+            this.app = app;
+            this.codex = codex;
+            this.client = client;
+            this.address = address;
+        }
+
+        public void Start(int index)
+        {
+            for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
+            {
+                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
+            }
+
+            var delayPerPurchaser =
+                TimeSpan.FromSeconds(10 * index) +
+                TimeSpan.FromMinutes(app.Config.ContractDurationMinutes) / app.Config.NumConcurrentPurchases;
+
+            starterTask = Task.Run(() => StartPurchasers(delayPerPurchaser));
+        }
+
+        private async Task StartPurchasers(TimeSpan delayPerPurchaser)
+        {
+            foreach (var purchaser in purchasers)
+            {
+                purchaser.Start();
+                await Task.Delay(delayPerPurchaser);
+            }
+        }
+
+        public void Stop()
+        {
+            starterTask.Wait();
+            foreach (var purchaser in purchasers)
+            {
+                purchaser.Stop();
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -4,11 +4,8 @@ namespace AutoClient
 {
     public class Configuration
     {
-        [Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")]
-        public string CodexHost { get; set; } = "http://localhost";
-
-        [Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")]
-        public int CodexPort { get; set; } = 8080;
+        [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
+        public string CodexEndpoints { get; set; } = "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -22,11 +19,11 @@ namespace AutoClient
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
         public int ContractExpiryMinutes { get; set; } = 15;
 
-        [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")]
-        public int NumHosts { get; set; } = 5;
+        [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
+        public int NumHosts { get; set; } = 10;
 
-        [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 2)")]
-        public int HostTolerance { get; set; } = 2;
+        [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
+        public int HostTolerance { get; set; } = 5;
 
         [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
         public int Price { get; set; } = 10;

Tools/AutoClient/ImageGenerator.cs

@@ -11,16 +11,16 @@ namespace AutoClient
 
     public class ImageGenerator : IFileGenerator
     {
-        private LogSplitter log;
+        private readonly ILog log;
 
-        public ImageGenerator(LogSplitter log)
+        public ImageGenerator(ILog log)
         {
             this.log = log;
         }
 
         public async Task<string> Generate()
         {
-            log.Log("Fetching random image from picsum.photos...");
+            log.Debug("Fetching random image from picsum.photos...");
             var httpClient = new HttpClient();
             var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");
 

Tools/AutoClient/Performance.cs

@@ -0,0 +1,65 @@
+using Logging;
+
+namespace AutoClient
+{
+    public class Performance
+    {
+        private readonly ILog log;
+
+        public Performance(ILog log)
+        {
+            this.log = log;
+        }
+
+        public void DownloadFailed(Exception ex)
+        {
+            Log($"Download failed: {ex}");
+        }
+
+        public void DownloadSuccessful(long size, TimeSpan time)
+        {
+            long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
+            if (milliseconds < 1) milliseconds = 1;
+            long bytesPerSecond = 1000 * (size / milliseconds);
+            Log($"Download successful: {bytesPerSecond} bytes per second");
+        }
+
+        public void StorageContractCancelled()
+        {
+            Log("Contract cancelled");
+        }
+
+        public void StorageContractErrored(string error)
+        {
+            Log($"Contract errored: {error}");
+        }
+
+        public void StorageContractFinished()
+        {
+            Log("Contract finished");
+        }
+
+        public void StorageContractStarted()
+        {
+            Log("Contract started");
+        }
+
+        public void UploadFailed(Exception ex)
+        {
+            Log($"Upload failed: {ex}");
+        }
+
+        public void UploadSuccessful(long size, TimeSpan time)
+        {
+            long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
+            if (milliseconds < 1) milliseconds = 1;
+            long bytesPerSecond = 1000 * (size / milliseconds);
+            Log($"Upload successful: {bytesPerSecond} bytes per second");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

Tools/AutoClient/Program.cs

@@ -1,16 +1,20 @@
 using ArgsUniform;
 using AutoClient;
 using CodexOpenApi;
-using Core;
-using Logging;
 using Utils;
 
-public static class Program
+public class Program
 {
+    private readonly App app;
+
+    public Program(Configuration config)
+    {
+        app = new App(config);
+    }
+
     public static async Task Main(string[] args)
     {
         var cts = new CancellationTokenSource();
-        var cancellationToken = cts.Token;
         Console.CancelKeyPress += (sender, args) => cts.Cancel();
 
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
@@ -21,58 +25,70 @@ public static class Program
             throw new Exception("Number of concurrent purchases must be > 0");
         }
 
-        var log = new LogSplitter(
-            new FileLog(Path.Combine(config.LogPath, "autoclient")),
-            new ConsoleLog()
-        );
+        var p = new Program(config);
+        await p.Run();
+    }
+
+    public async Task Run()
+    {
+        var codexUsers = await CreateUsers();
+
+        var i = 0;
+        foreach (var user in codexUsers)
+        {
+            user.Start(i);
+            i++;
+        }
+
+        app.Cts.Token.WaitHandle.WaitOne();
+
+        foreach (var user in codexUsers) user.Stop();
+
+        app.Log.Log("Done");
+    }
+
+    private async Task<CodexUser[]> CreateUsers()
+    {
+        var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
+        var result = new List<CodexUser>();
+
+        foreach (var e in endpointStrs)
+        {
+            result.Add(await CreateUser(e));
+        }
+
+        return result.ToArray();
+    }
+
+    private async Task<CodexUser> CreateUser(string endpoint)
+    {
+        var splitIndex = endpoint.LastIndexOf(':');
+        var host = endpoint.Substring(0, splitIndex);
+        var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
 
         var address = new Address(
-            host: config.CodexHost,
-            port: config.CodexPort
+            host: host,
+            port: port
         );
 
-        log.Log($"Start. Address: {address}");
-
-        var generator = CreateGenerator(config, log);
-
         var client = new HttpClient();
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
-        await CheckCodex(codex, log);
+        app.Log.Log($"Checking Codex at {address}...");
+        await CheckCodex(codex);
+        app.Log.Log("OK");
 
-        var purchasers = new List<Purchaser>();
-        for (var i = 0; i < config.NumConcurrentPurchases; i++)
-        {
-            purchasers.Add(
-                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken)
-            );
-        }
-
-        var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases;
-        foreach (var purchaser in purchasers) 
-        {
-            purchaser.Start();
-            await Task.Delay(delayPerPurchaser);
-        }
-
-        cancellationToken.WaitHandle.WaitOne();
-
-        log.Log("Done.");
+        return new CodexUser(
+            app,
+            codex,
+            client,
+            address
+        );
     }
 
-    private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log)
+    private async Task CheckCodex(CodexApi codex)
     {
-        if (config.FileSizeMb > 0)
-        {
-            return new RandomFileGenerator(config, log);
-        }
-        return new ImageGenerator(log);
-    }
-
-    private static async Task CheckCodex(CodexApi codex, ILog log)
-    {
-        log.Log("Checking Codex...");
         try
         {
             var info = await codex.GetDebugInfoAsync();
@@ -80,7 +96,7 @@ public static class Program
         }
         catch (Exception ex)
         {
-            log.Log($"Codex not OK: {ex}");
+            app.Log.Error($"Codex not OK: {ex}");
             throw;
         }
     }

Tools/AutoClient/Purchaser.cs

@@ -8,36 +8,76 @@ namespace AutoClient
 {
     public class Purchaser
     {
+        private readonly App app;
+        private readonly string nodeId;
         private readonly ILog log;
         private readonly HttpClient client;
         private readonly Address address;
         private readonly CodexApi codex;
-        private readonly Configuration config;
-        private readonly IFileGenerator generator;
-        private readonly CancellationToken ct;
+        private Task workerTask = Task.CompletedTask;
 
-        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct)
+        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
         {
+            this.app = app;
+            this.nodeId = nodeId;
             this.log = log;
             this.client = client;
             this.address = address;
             this.codex = codex;
-            this.config = config;
-            this.generator = generator;
-            this.ct = ct;
         }
 
         public void Start()
         {
-            Task.Run(Worker);
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
         }
 
         private async Task Worker()
         {
-            while (!ct.IsCancellationRequested)
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
             {
-                var pid = await StartNewPurchase();
-                await WaitTillFinished(pid);
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(nodeId);
+            if (cid == null) return;
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (size == null) return;
+
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+                {
+                    using var fileStream = File.OpenWrite(filename);
+                    var fileResponse = await codex.DownloadNetworkAsync(cid);
+                    fileResponse.Stream.CopyTo(fileStream);
+                }
+                var time = sw.Elapsed;
+                File.Delete(filename);
+                app.Performance.DownloadSuccessful(size.Value, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
             }
         }
 
@@ -50,72 +90,96 @@ namespace AutoClient
 
         private async Task<string> CreateFile()
         {
-            return await generator.Generate();
+            return await app.Generator.Generate();
         }
 
         private async Task<ContentId> UploadFile(string filename)
         {
-            // Copied from CodexNode :/
             using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(nodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
 
-            log.Log($"Uploading file {filename}...");
-            var response = await codex.UploadAsync(fileStream, ct);
+        private async Task<ContentId> UploadStream(FileStream fileStream)
+        {
+            log.Debug($"Uploading file...");
+            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
 
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
 
-            log.Log($"Uploaded file. Received contentId: '{response}'.");
+            log.Debug($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
 
         private async Task<string> RequestStorage(ContentId cid)
         {
-            log.Log("Requesting storage for " + cid.Id);
+            log.Debug("Requesting storage for " + cid.Id);
             var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
             {
-                Collateral = config.RequiredCollateral.ToString(),
-                Duration = (config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = config.NumHosts,
-                Reward = config.Price.ToString(),
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
                 ProofProbability = "15",
-                Tolerance = config.HostTolerance
-            }, ct);
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
 
-            log.Log("Purchase ID: " + result);
+            log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
 
             return result;
         }
 
-        private async Task<string?> GetPurchaseState(string pid)
+        private async Task<string> GetEncodedCid(string pid)
         {
             try
             {
-                // openapi still don't match code.
-                var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
-                if (string.IsNullOrEmpty(str)) return null;
-                var sp = JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-                log.Log($"Purchase {pid} is {sp.State}");
-                if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}");
-                return sp.State;
+                var sp = (await GetStoragePurchase(pid))!;
+                return sp.Request.Content.Cid;
             }
-            catch
+            catch (Exception ex)
             {
-                return null;
+                log.Error(ex.ToString());
+                throw;
             }
         }
 
+        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
         private async Task WaitTillFinished(string pid)
         {
-            log.Log("Waiting...");
             try
             {
                 var emptyResponseTolerance = 10;
-                while (true)
+                while (!app.Cts.Token.IsCancellationRequested)
                 {
-                    var status = (await GetPurchaseState(pid))?.ToLowerInvariant();
-                    if (string.IsNullOrEmpty(status))
+                    var purchase = await GetStoragePurchase(pid);
+                    if (purchase == null)
                     {
+                        await FixedShortDelay();
                         emptyResponseTolerance--;
                         if (emptyResponseTolerance == 0)
                         {
@@ -123,19 +187,28 @@ namespace AutoClient
                             await ExpiryTimeDelay();
                             return;
                         }
+                        continue;
                     }
-                    else
+                    var status = purchase.State.ToLowerInvariant();
+                    if (status.Contains("cancel"))
                     {
-                        if (status.Contains("cancel") ||
-                            status.Contains("error") ||
-                            status.Contains("finished"))
-                        {
-                            return;
-                        }
-                        if (status.Contains("started"))
-                        {
-                            await FixedDurationDelay();
-                        }
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (status.Contains("error"))
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (status.Contains("finished"))
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (status.Contains("started"))
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
                     }
 
                     await FixedShortDelay();
@@ -150,17 +223,17 @@ namespace AutoClient
 
         private async Task FixedDurationDelay()
         {
-            await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct);
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
         }
 
         private async Task ExpiryTimeDelay()
         {
-            await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct);
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
         }
 
         private async Task FixedShortDelay()
         {
-            await Task.Delay(15 * 1000, ct);
+            await Task.Delay(15 * 1000, app.Cts.Token);
         }
     }
 }

Tools/BittorrentDriver/BittorrentDriver.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <UserSecretsId>4d58719c-20df-4407-bfb4-0f65a324a118</UserSecretsId>
+    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.20.1" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\FileUtils\FileUtils.csproj" />
+  </ItemGroup>
+
+</Project>

Tools/BittorrentDriver/BittorrentDriver.csproj.user

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <ActiveDebugProfile>Container (Dockerfile)</ActiveDebugProfile>
+  </PropertyGroup>
+</Project>

Tools/BittorrentDriver/BittorrentDriver.http

@@ -0,0 +1,6 @@
+@BittorrentDriver_HostAddress = http://localhost:5160
+
+GET {{BittorrentDriver_HostAddress}}/weatherforecast/
+Accept: application/json
+
+###

Tools/BittorrentDriver/Controllers/TorrentController.cs

@@ -0,0 +1,131 @@
+using Logging;
+using Microsoft.AspNetCore.Mvc;
+using System.Diagnostics;
+
+namespace BittorrentDriver.Controllers
+{
+    [ApiController]
+    [Route("[controller]")]
+    public class TorrentController : ControllerBase
+    {
+        private readonly ILog log = new ConsoleLog();
+        private readonly TorrentTracker tracker = new TorrentTracker();
+        private readonly Transmission transmission;
+
+        public TorrentController()
+        {
+            transmission = new Transmission(log);
+        }
+
+        [HttpPut("starttracker")]
+        public string StartTracker([FromBody] int port)
+        {
+            return Try(() =>
+            {
+                Log("Starting tracker...");
+                return tracker.Start(port);
+            });
+        }
+
+        [HttpPost("addtracker")]
+        public string AddTracker([FromBody] AddTrackerInput input)
+        {
+            return Try(() =>
+            {
+                Log("Adding tracker: " + input.TrackerUrl + " - " + input.LocalFile);
+                return transmission.AddTracker(input.TrackerUrl, input.LocalFile);
+            });
+        }
+
+        [HttpGet("stats")]
+        public string GetTrackerStats()
+        {
+            return Try(() =>
+            {
+                return tracker.GetStats();
+            });
+        }
+
+        [HttpPost("postfile")]
+        public string PostFile([FromBody] PostFileInput input)
+        {
+            return Try(() =>
+            {
+                Log("Creating file..");
+                var file = transmission.PutLocalFile(input.Base64Content);
+                Log("File: " + file);
+                return file;
+            });
+        }
+
+        [HttpPut("daemon")]
+        public string StartDaemon([FromBody] int peerPort)
+        {
+            return Try(() =>
+            {
+                Log("Starting daemon...");
+                return transmission.StartDaemon(peerPort);
+            });
+        }
+
+        [HttpPost("create")]
+        public CreateTorrentResult CreateTorrent([FromBody] CreateTorrentInput input)
+        {
+            return Try(() =>
+            {
+                Log("Creating torrent file...");
+                return transmission.CreateNew(input.Size, input.TrackerUrl);
+            });
+        }
+
+        [HttpPost("download")]
+        public string DownloadTorrent([FromBody] DownloadTorrentInput input)
+        {
+            return Try(() =>
+            {
+                Log("Downloading torrent...");
+                return transmission.Download(input.LocalFile);
+            });
+        }
+
+        private T Try<T>(Func<T> value)
+        {
+            try
+            {
+                return value();
+            }
+            catch (Exception exc)
+            {
+                log.Error(exc.ToString());
+                throw;
+            }
+        }
+
+        private void Log(string v)
+        {
+            log.Log(v);
+        }
+    }
+
+    public class CreateTorrentInput
+    {
+        public int Size { get; set; }
+        public string TrackerUrl { get; set; } = string.Empty;
+    }
+
+    public class AddTrackerInput
+    {
+        public string TrackerUrl { get; set; } = string.Empty;
+        public string LocalFile { get; set; } = string.Empty;
+    }
+
+    public class DownloadTorrentInput
+    {
+        public string LocalFile { get; set; } = string.Empty;
+    }
+
+    public class PostFileInput
+    {
+        public string Base64Content { get; set; } = string.Empty;
+    }
+}

Tools/BittorrentDriver/Dockerfile

@@ -0,0 +1,33 @@
+# Variables
+ARG IMAGE=ubuntu:24.10
+ARG APP_HOME=/app
+
+
+# Build
+FROM ${IMAGE} AS builder
+ARG APP_HOME
+RUN apt-get update
+RUN apt-get install dotnet-sdk-8.0 -y
+
+WORKDIR ${APP_HOME}
+COPY ./Tools/BittorrentDriver ./Tools/BittorrentDriver
+COPY ./Framework ./Framework
+RUN dotnet restore Tools/BittorrentDriver
+RUN dotnet publish Tools/BittorrentDriver -c Release -o out
+
+# Create
+FROM ${IMAGE}
+ARG DEBIAN_FRONTEND=noninteractive
+ARG APP_HOME
+ENV APP_HOME=${APP_HOME}
+
+# Set up npm and bittorrent-tracker
+RUN apt-get update
+RUN apt-get install npm aspnetcore-runtime-8.0 -y
+RUN npm install -g bittorrent-tracker
+# Set up transmission
+RUN apt-get install transmission-cli transmission-common transmission-daemon -y
+
+WORKDIR ${APP_HOME}
+COPY --from=builder ${APP_HOME}/out .
+CMD dotnet ${APP_HOME}/BittorrentDriver.dll

Tools/BittorrentDriver/Program.cs

@@ -0,0 +1,41 @@
+
+namespace BittorrentDriver
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            var builder = WebApplication.CreateBuilder(args);
+
+            var listenPort = Environment.GetEnvironmentVariable("APIPORT");
+            if (string.IsNullOrEmpty(listenPort)) listenPort = "31100";
+
+            builder.WebHost.ConfigureKestrel((context, options) =>
+            {
+                options.ListenAnyIP(Convert.ToInt32(listenPort));
+            });
+
+            builder.Services.AddControllers();
+            builder.Services.AddEndpointsApiExplorer();
+            builder.Services.AddSwaggerGen();
+
+            var app = builder.Build();
+
+            if (app.Environment.IsDevelopment())
+            {
+                app.UseSwagger();
+                app.UseSwaggerUI();
+            }
+
+            app.UseHttpsRedirection();
+
+            app.UseAuthorization();
+
+            app.MapControllers();
+
+            Console.WriteLine("TorrentController BittorrentDriver listening on port " + listenPort);
+
+            app.Run();
+        }
+    }
+}

Tools/BittorrentDriver/Properties/launchSettings.json

@@ -0,0 +1,52 @@
+{
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "http://localhost:5160"
+    },
+    "https": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "dotnetRunMessages": true,
+      "applicationUrl": "https://localhost:7134;http://localhost:5160"
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "Container (Dockerfile)": {
+      "commandName": "Docker",
+      "launchBrowser": true,
+      "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/swagger",
+      "environmentVariables": {
+        "ASPNETCORE_HTTPS_PORTS": "8081",
+        "ASPNETCORE_HTTP_PORTS": "8080"
+      },
+      "publishAllPorts": true,
+      "useSSL": true
+    }
+  },
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:32045",
+      "sslPort": 44353
+    }
+  }
+}

Tools/BittorrentDriver/README.md

@@ -0,0 +1,4 @@
+# Run from repo root folder:
+
+docker build -t thatbenbierens/bittorrentdriver:init -f .\Tools\BittorrentDriver\Dockerfile .
+docker push thatbenbierens/bittorrentdriver:init

Tools/BittorrentDriver/TorrentTracker.cs

@@ -0,0 +1,67 @@
+using System.Diagnostics;
+
+namespace BittorrentDriver
+{
+    public class TorrentTracker
+    {
+        private Process? process;
+        private int? trackerPort;
+
+        public string Start(int port)
+        {
+            if (process != null) throw new Exception("Already started");
+            trackerPort = port;
+
+            var info = new ProcessStartInfo
+            {
+                FileName = "bittorrent-tracker",
+                Arguments = 
+                    $"--port {port} " +
+                    $"--http " +
+                    $"--stats " +
+                    $"--interval=3000 " + // 3 seconds
+                    $"&",
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+            };
+
+            process = Process.Start(info);
+            if (process == null) return "Failed to start";
+
+            process.OutputDataReceived += (sender, args) =>
+            {
+                Console.WriteLine("STDOUT: " + args.Data);
+            };
+            process.ErrorDataReceived += (sender, args) =>
+            {
+                Console.WriteLine("STDERR: " + args.Data);
+            };
+
+            process.BeginOutputReadLine();
+            process.BeginErrorReadLine();
+
+            Thread.Sleep(1000);
+
+            if (process.HasExited)
+            {
+                return
+                    $"STDOUT: {process.StandardOutput.ReadToEnd()} " +
+                    $"STDERR: {process.StandardError.ReadToEnd()}";
+            }
+            return "OK";
+        }
+
+        public string GetStats()
+        {
+            if (!trackerPort.HasValue) throw new Exception("Port value not set");
+
+            using var client = new HttpClient();
+            var task = client.GetAsync($"http://localhost:{trackerPort.Value}/stats");
+            task.Wait();
+            var strTask = task.Result.Content.ReadAsStringAsync();
+            strTask.Wait();
+            return strTask.Result;
+        }
+    }
+}

Tools/BittorrentDriver/Transmission.cs

@@ -0,0 +1,143 @@
+using FileUtils;
+using Logging;
+using System.Buffers.Text;
+using System.Diagnostics;
+using System.Text;
+using Utils;
+
+namespace BittorrentDriver
+{
+    public class Transmission
+    {
+        private readonly string dataDir;
+        private readonly ILog log;
+
+        public Transmission(ILog log)
+        {
+            dataDir = Path.Combine(Directory.GetCurrentDirectory(), "files");
+            Directory.CreateDirectory(dataDir);
+            this.log = log;
+        }
+
+        public CreateTorrentResult CreateNew(int size, string trackerUrl)
+        {
+            var file = CreateFile(size);
+            var outFile = Path.Combine(dataDir, Guid.NewGuid().ToString());
+            var base64 = CreateTorrentFile(file, outFile, trackerUrl);
+            return new CreateTorrentResult
+            {
+                LocalFilePath = outFile,
+                TorrentBase64 = base64
+            };
+        }
+
+        public string StartDaemon(int peerPort)
+        {
+            var info = new ProcessStartInfo
+            {
+                FileName = "transmission-daemon",
+                Arguments = $"--peerport={peerPort} " +
+                $"--download-dir={dataDir} " +
+                $"--watch-dir={dataDir} " +
+                $"--no-global-seedratio " +
+                $"--bind-address-ipv4=0.0.0.0 " +
+                $"--dht"
+            };
+            RunToComplete(info);
+
+            return "OK";
+        }
+        
+        public string AddTracker(string trackerUrl, string localFile)
+        {
+            var info = new ProcessStartInfo
+            {
+                FileName = "transmission-edit",
+                Arguments = $"--add={trackerUrl} {localFile}"
+            };
+            RunToComplete(info);
+
+            return "OK";
+        }
+
+        public string PutLocalFile(string torrentBase64)
+        {
+            var torrentFile = Path.Combine(dataDir, Guid.NewGuid().ToString() + ".torrent");
+            File.WriteAllBytes(torrentFile, Convert.FromBase64String(torrentBase64));
+            return torrentFile;
+        }
+
+        public string Download(string localFile)
+        {
+            var peerPort = Environment.GetEnvironmentVariable("PEERPORT");
+
+            var info = new ProcessStartInfo
+            {
+                FileName = "transmission-cli",
+                Arguments = 
+                    $"--port={peerPort} " +
+                    $"--download-dir={dataDir} " +
+                    $"{localFile}"
+            };
+            RunToComplete(info);
+
+            return "OK";
+        }
+
+        private string CreateTorrentFile(TrackedFile file, string outFile, string trackerUrl)
+        {
+            try
+            {
+                var info = new ProcessStartInfo
+                {
+                    FileName = "transmission-create",
+                    Arguments = $"-o {outFile} -t {trackerUrl} {file.Filename}",
+                };
+
+                var process = RunToComplete(info);
+
+                log.Log(nameof(CreateTorrentFile) + " exited with: " + process.ExitCode);
+
+                if (!File.Exists(outFile)) throw new Exception("Outfile not created.");
+
+                return Convert.ToBase64String(File.ReadAllBytes(outFile));
+            }
+            catch (Exception ex)
+            {
+                log.Error("Failed to create torrent file: " + ex);
+                throw;
+            }
+        }
+
+        private Process RunToComplete(ProcessStartInfo info)
+        {
+            log.Log($"Running: {info.FileName} ({info.Arguments})");
+            var process = Process.Start(info);
+            if (process == null) throw new Exception("Failed to start");
+            process.WaitForExit(TimeSpan.FromMinutes(3));
+            return process;
+        }
+
+        private TrackedFile CreateFile(int size)
+        {
+            try
+            {
+                var fileManager = new FileManager(log, dataDir, numberSubfolders: false);
+                var file = fileManager.GenerateFile(size.Bytes());
+                log.Log("Generated file: " + file.Filename);
+                return file;
+            }
+            catch (Exception ex)
+            {
+                log.Error("Failed to create file: " + ex);
+                throw;
+            }
+        }
+    }
+
+    public class CreateTorrentResult
+    {
+        public string LocalFilePath { get; set; } = string.Empty;
+        public string TorrentBase64 { get; set; } = string.Empty;
+    } 
+}

Tools/BittorrentDriver/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

Tools/BittorrentDriver/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

Tools/MarketInsights/Dockerfile

@@ -3,8 +3,6 @@
 FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
 USER app
 WORKDIR /app
-EXPOSE 8080
-EXPOSE 8081
 
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG BUILD_CONFIGURATION=Release

Tools/MarketInsights/Program.cs

@@ -1,6 +1,4 @@
 using ArgsUniform;
-using Microsoft.Extensions.Options;
-using Nethereum.Model;
 using System.Reflection;
 
 namespace MarketInsights
@@ -28,6 +26,14 @@ namespace MarketInsights
 
             var builder = WebApplication.CreateBuilder(args);
 
+            var listenPort = Environment.GetEnvironmentVariable("APIPORT");
+            if (string.IsNullOrEmpty(listenPort)) listenPort = "31090";
+
+            builder.WebHost.ConfigureKestrel((context, options) =>
+            {
+                options.ListenAnyIP(Convert.ToInt32(listenPort));
+            });
+
             builder.Services.AddSingleton(appState);
 
             builder.Services.AddControllers();
@@ -54,6 +60,8 @@ namespace MarketInsights
 
             app.MapControllers();
 
+            Console.WriteLine("MarketInsights listening on port " + listenPort);
+
             updater.Run();
             app.Run();
         }

cs-codex-dist-testing.sln

@@ -74,7 +74,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OverwatchTranscript", "Fram
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools\TranscriptAnalysis\TranscriptAnalysis.csproj", "{C0EEBD32-23CB-45EC-A863-79FB948508C8}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BittorrentPlugin", "ProjectPlugins\BittorrentPlugin\BittorrentPlugin.csproj", "{79866016-8CB3-4A30-9E5E-54070F27BE1E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BittorrentDriver", "Tools\BittorrentDriver\BittorrentDriver.csproj", "{AB9F7F80-9071-49A8-8B9D-D5B5D3E56560}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -202,6 +206,14 @@ Global
 		{004614DF-1C65-45E3-882D-59AE44282573}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.Build.0 = Release|Any CPU
+		{79866016-8CB3-4A30-9E5E-54070F27BE1E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{79866016-8CB3-4A30-9E5E-54070F27BE1E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{79866016-8CB3-4A30-9E5E-54070F27BE1E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{79866016-8CB3-4A30-9E5E-54070F27BE1E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AB9F7F80-9071-49A8-8B9D-D5B5D3E56560}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AB9F7F80-9071-49A8-8B9D-D5B5D3E56560}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AB9F7F80-9071-49A8-8B9D-D5B5D3E56560}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AB9F7F80-9071-49A8-8B9D-D5B5D3E56560}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -237,6 +249,8 @@ Global
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
 		{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
+		{79866016-8CB3-4A30-9E5E-54070F27BE1E} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124}
+		{AB9F7F80-9071-49A8-8B9D-D5B5D3E56560} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}