Merge branch 'feature/location'

CodexDistTestCore/CodexNodeGroup.cs

@@ -1,4 +1,5 @@
-using k8s.Models;
+using CodexDistTestCore.Config;
+using k8s.Models;
 using System.Collections;
 
 namespace CodexDistTestCore
@@ -11,10 +12,12 @@ namespace CodexDistTestCore
 
     public class CodexNodeGroup : ICodexNodeGroup
     {
+        private readonly TestLog log;
         private readonly IK8sManager k8SManager;
 
-        public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
+        public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
         {
+            this.log = log;
             OrderNumber = orderNumber;
             Origin = origin;
             this.k8SManager = k8SManager;
@@ -63,7 +66,7 @@ namespace CodexDistTestCore
             return new V1ObjectMeta
             {
                 Name = "codex-test-entrypoint-" + OrderNumber,
-                NamespaceProperty = K8sOperations.K8sNamespace
+                NamespaceProperty = K8sCluster.K8sNamespace
             };
         }
 
@@ -72,10 +75,17 @@ namespace CodexDistTestCore
             return new V1ObjectMeta
             {
                 Name = "codex-test-node-" + OrderNumber,
-                NamespaceProperty = K8sOperations.K8sNamespace
+                NamespaceProperty = K8sCluster.K8sNamespace
             };
         }
 
+        public CodexNodeLog DownloadLog(IOnlineCodexNode node)
+        {
+            var logDownloader = new PodLogDownloader(log, k8SManager);
+            var n = (OnlineCodexNode)node;
+            return logDownloader.DownloadLog(n);
+        }
+
         public Dictionary<string, string> GetSelector()
         {
             return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };

CodexDistTestCore/CodexNodeLog.cs

@@ -0,0 +1,29 @@
+using NUnit.Framework;
+
+namespace CodexDistTestCore
+{
+    public class CodexNodeLog
+    {
+        private readonly LogFile logFile;
+
+        public CodexNodeLog(LogFile logFile)
+        {
+            this.logFile = logFile;
+        }
+
+        public void AssertLogContains(string expectedString)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (line.Contains(expectedString)) return;
+                line = streamReader.ReadLine();
+            }
+
+            Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}");
+        }
+    }
+}

CodexDistTestCore/Config/CodexDockerImage.cs

@@ -1,6 +1,6 @@
 using k8s.Models;
 
-namespace CodexDistTestCore
+namespace CodexDistTestCore.Config
 {
     public class CodexDockerImage
     {

CodexDistTestCore/Config/FileManagerConfig.cs

@@ -0,0 +1,7 @@
+namespace CodexDistTestCore.Config
+{
+    public class FileManagerConfig
+    {
+        public const string Folder = "TestDataFiles";
+    }
+}

CodexDistTestCore/Config/K8sCluster.cs

@@ -0,0 +1,39 @@
+using k8s;
+
+namespace CodexDistTestCore.Config
+{
+    public class K8sCluster
+    {
+        public const string K8sNamespace = "codex-test-namespace";
+        private const string KubeConfigFile = "C:\\kube\\config";
+        private readonly Dictionary<Location, string> K8sNodeLocationMap = new Dictionary<Location, string>
+        {
+            { Location.BensLaptop, "worker01" },
+            { Location.BensOldGamingMachine, "worker02" },
+        };
+
+        private KubernetesClientConfiguration? config;
+
+        public KubernetesClientConfiguration GetK8sClientConfig()
+        {
+            if (config != null) return config;
+            config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
+            return config;
+        }
+
+        public string GetIp()
+        {
+            var c = GetK8sClientConfig();
+
+            var host = c.Host.Replace("https://", "");
+
+            return host.Substring(0, host.IndexOf(':'));
+        }
+
+        public string GetNodeLabelForLocation(Location location)
+        {
+            if (location == Location.Unspecified) return string.Empty;
+            return K8sNodeLocationMap[location];
+        }
+    }
+}

CodexDistTestCore/Config/LogConfig.cs

@@ -0,0 +1,7 @@
+namespace CodexDistTestCore.Config
+{
+    public class LogConfig
+    {
+        public const string LogRoot = "D:/CodexTestLogs";
+    }
+}

CodexDistTestCore/DistTest.cs

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using CodexDistTestCore.Config;
+using NUnit.Framework;
 
 namespace CodexDistTestCore
 {
@@ -41,7 +42,10 @@ namespace CodexDistTestCore
             }
             else
             {
+                var dockerImage = new CodexDockerImage();
                 log = new TestLog();
+                log.Log($"Using docker image '{dockerImage.GetImageTag()}'");
+
                 fileManager = new FileManager(log);
                 k8sManager = new K8sManager(log, fileManager);
             }
@@ -52,7 +56,8 @@ namespace CodexDistTestCore
         {
             try
             {
-                log.EndTest(k8sManager);
+                log.EndTest();
+                IncludeLogsOnTestFailure();
                 k8sManager.DeleteAllResources();
                 fileManager.DeleteAllTestFiles();
             }
@@ -72,6 +77,39 @@ namespace CodexDistTestCore
         {
             return new OfflineCodexNodes(k8sManager, numberOfNodes);
         }
+
+        private void IncludeLogsOnTestFailure()
+        {
+            var result = TestContext.CurrentContext.Result;
+            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
+            {
+                if (IsDownloadingLogsEnabled())
+                {
+                    log.Log("Downloading all CodexNode logs because of test failure...");
+                    k8sManager.ForEachOnlineGroup(DownloadLogs);
+                }
+                else
+                {
+                    log.Log("Skipping download of all CodexNode logs due to [DontDownloadLogsOnFailure] attribute.");
+                }
+            }
+        }
+
+        private void DownloadLogs(CodexNodeGroup group)
+        {
+            foreach (var node in group)
+            {
+                var downloader = new PodLogDownloader(log, k8sManager);
+                var n = (OnlineCodexNode)node;
+                downloader.DownloadLog(n);
+            }
+        }
+
+        private bool IsDownloadingLogsEnabled()
+        {
+            var testProperties = TestContext.CurrentContext.Test.Properties;
+            return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);
+        }
     }
 
     public static class GlobalTestFailure

CodexDistTestCore/FileManager.cs

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using CodexDistTestCore.Config;
+using NUnit.Framework;
 
 namespace CodexDistTestCore
 {
@@ -12,20 +13,19 @@ namespace CodexDistTestCore
     public class FileManager : IFileManager
     {
         public const int ChunkSize = 1024 * 1024;
-        private const string Folder = "TestDataFiles";
         private readonly Random random = new Random();
         private readonly List<TestFile> activeFiles = new List<TestFile>();
         private readonly TestLog log;
 
         public FileManager(TestLog log)
         {
-            if (!Directory.Exists(Folder)) Directory.CreateDirectory(Folder);
+            if (!Directory.Exists(FileManagerConfig.Folder)) Directory.CreateDirectory(FileManagerConfig.Folder);
             this.log = log;
         }
 
         public TestFile CreateEmptyTestFile()
         {
-            var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin"));
+            var result = new TestFile(Path.Combine(FileManagerConfig.Folder, Guid.NewGuid().ToString() + "_test.bin"));
             File.Create(result.Filename).Close();
             activeFiles.Add(result);
             return result;

CodexDistTestCore/K8sManager.cs

@@ -4,12 +4,13 @@
     {
         ICodexNodeGroup BringOnline(OfflineCodexNodes node);
         IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
+        void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler);
     }
 
     public class K8sManager : IK8sManager
     {
         private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource();
-        private readonly List<CodexNodeGroup> onlineCodexNodes = new List<CodexNodeGroup>();
+        private readonly List<CodexNodeGroup> onlineCodexNodeGroups = new List<CodexNodeGroup>();
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly TestLog log;
         private readonly IFileManager fileManager;
@@ -47,17 +48,22 @@
             K8s(k => k.DeleteAllResources());
         }
 
-        public void FetchAllPodsLogs(IPodLogsHandler logHandler)
+        public void ForEachOnlineGroup(Action<CodexNodeGroup> action)
         {
-            K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler));
+            foreach (var group in onlineCodexNodeGroups) action(group);
+        }
+
+        public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
+        {
+            K8s(k => k.FetchPodLog(node, logHandler));
         }
 
         private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
         {
             var containers = CreateContainers(offline.NumberOfNodes);
             var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
-            var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
+            var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
-            onlineCodexNodes.Add(result);
+            onlineCodexNodeGroups.Add(result);
             return result;
         }
 
@@ -72,7 +78,7 @@
         private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
         {
             var n = (CodexNodeGroup)node;
-            onlineCodexNodes.Remove(n);
+            onlineCodexNodeGroups.Remove(n);
             return n;
         }
 

CodexDistTestCore/K8sOperations.cs

@@ -1,4 +1,5 @@
-using k8s;
+using CodexDistTestCore.Config;
+using k8s;
 using k8s.Models;
 using NUnit.Framework;
 
@@ -6,9 +7,8 @@ namespace CodexDistTestCore
 {
     public class K8sOperations
     {
-        public const string K8sNamespace = "codex-test-namespace";
-
         private readonly CodexDockerImage dockerImage = new CodexDockerImage();
+        private readonly K8sCluster k8sCluster = new K8sCluster();
         private readonly Kubernetes client;
         private readonly KnownK8sPods knownPods;
 
@@ -16,9 +16,7 @@ namespace CodexDistTestCore
         {
             this.knownPods = knownPods;
 
-            // todo: If the default KubeConfig file does not suffice, change it here:
+            client = new Kubernetes(k8sCluster.GetK8sClientConfig());
-            var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
-            client = new Kubernetes(config);
         }
 
         public void Close()
@@ -53,25 +51,10 @@ namespace CodexDistTestCore
             WaitUntilNamespaceDeleted();
         }
 
-        public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler)
+        public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
         {
-            var logNumberSource = new NumberSource(0);
+            var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name);
-            foreach (var online in onlines)
+            logHandler.Log(stream);
-            {
-                foreach (var node in online)
-                {
-                    WritePodLogs(online, node, logHandler, logNumberSource);
-                }
-            }
-        }
-
-        private void WritePodLogs(CodexNodeGroup online, IOnlineCodexNode node, IPodLogsHandler logHandler, NumberSource logNumberSource)
-        {
-            var n = (OnlineCodexNode)node;
-            var nodeDescription = $"{online.Describe()} contains {n.GetName()}";
-
-            var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name);
-            logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream);
         }
 
         private void FetchPodInfo(CodexNodeGroup online)
@@ -208,6 +191,7 @@ namespace CodexDistTestCore
                         },
                         Spec = new V1PodSpec
                         {
+                            NodeSelector = CreateNodeSelector(offline),
                             Containers = CreateDeploymentContainers(online, offline)
                         }
                     }
@@ -217,6 +201,16 @@ namespace CodexDistTestCore
             online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
         }
 
+        private IDictionary<string, string> CreateNodeSelector(OfflineCodexNodes offline)
+        {
+            if (offline.Location == Location.Unspecified) return new Dictionary<string, string>();
+
+            return new Dictionary<string, string> 
+            { 
+                { "codex-test-location", k8sCluster.GetNodeLabelForLocation(offline.Location) }
+            };
+        }
+
         private List<V1Container> CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline)
         {
             var result = new List<V1Container>();
@@ -276,6 +270,11 @@ namespace CodexDistTestCore
             }
         }
 
+        private string K8sNamespace
+        {
+            get { return K8sCluster.K8sNamespace; }
+        }
+
         #endregion
 
         private bool IsTestNamespaceOnline()

CodexDistTestCore/OfflineCodexNodes.cs

@@ -2,6 +2,7 @@
 {
     public interface IOfflineCodexNodes
     {
+        IOfflineCodexNodes At(Location location);
         IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
         IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
         IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
@@ -17,11 +18,19 @@
         Error
     }
 
+    public enum Location
+    {
+        Unspecified,
+        BensLaptop,
+        BensOldGamingMachine,
+    }
+
     public class OfflineCodexNodes : IOfflineCodexNodes
     {
         private readonly IK8sManager k8SManager;
         
         public int NumberOfNodes { get; }
+        public Location Location { get; private set; }
         public CodexLogLevel? LogLevel { get; private set; }
         public IOnlineCodexNode? BootstrapNode { get; private set; }
         public ByteSize? StorageQuota { get; private set; }
@@ -30,6 +39,7 @@
         {
             this.k8SManager = k8SManager;
             NumberOfNodes = numberOfNodes;
+            Location = Location.Unspecified;
         }
 
         public ICodexNodeGroup BringOnline()
@@ -37,6 +47,12 @@
             return k8SManager.BringOnline(this);
         }
 
+        public IOfflineCodexNodes At(Location location)
+        {
+            Location = location;
+            return this;
+        }
+
         public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node)
         {
             BootstrapNode = node;

CodexDistTestCore/OnlineCodexNode.cs

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using CodexDistTestCore.Config;
+using NUnit.Framework;
 
 namespace CodexDistTestCore
 {
@@ -8,6 +9,7 @@ namespace CodexDistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId);
         void ConnectToPeer(IOnlineCodexNode node);
+        CodexNodeLog DownloadLog();
     }
 
     public class OnlineCodexNode : IOnlineCodexNode
@@ -15,6 +17,7 @@ namespace CodexDistTestCore
         private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
         private const string UploadFailedMessage = "Unable to store block";
 
+        private readonly K8sCluster k8sCluster = new K8sCluster();
         private readonly TestLog log;
         private readonly IFileManager fileManager;
 
@@ -77,6 +80,16 @@ namespace CodexDistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
+        public CodexNodeLog DownloadLog()
+        {
+            return Group.DownloadLog(this);
+        }
+
+        public string Describe()
+        {
+            return $"{Group.Describe()} contains {GetName()}";
+        }
+
         private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
         {
             var multiAddress = peerInfo.addrs.First();
@@ -101,7 +114,7 @@ namespace CodexDistTestCore
 
         private Http Http()
         {
-            return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1");
+            return new Http(ip: k8sCluster.GetIp(), port: Container.ServicePort, baseUrl: "/api/codex/v1");
         }
 
         private void Log(string msg)

CodexDistTestCore/PodLogDownloader.cs

@@ -0,0 +1,73 @@
+using NUnit.Framework;
+
+namespace CodexDistTestCore
+{
+    public interface IPodLogHandler
+    {
+        void Log(Stream log);
+    }
+
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
+    {
+        public DontDownloadLogsOnFailureAttribute()
+            : base(Timing.UseLongTimeoutsKey)
+        {
+        }
+    }
+
+    public class PodLogDownloader
+    {
+        public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure";
+
+        private readonly TestLog log;
+        private readonly IK8sManager k8SManager;
+
+        public PodLogDownloader(TestLog log, IK8sManager k8sManager)
+        {
+            this.log = log;
+            k8SManager = k8sManager;
+        }
+
+        public CodexNodeLog DownloadLog(OnlineCodexNode node)
+        {
+            var description = node.Describe();
+            var subFile = log.CreateSubfile();
+
+            log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}");
+            var handler = new PodLogDownloadHandler(description, subFile);
+            k8SManager.FetchPodLog(node, handler);
+            return handler.CreateCodexNodeLog();
+        }
+    }
+
+    public class PodLogDownloadHandler : IPodLogHandler
+    {
+        private readonly string description;
+        private readonly LogFile log;
+
+        public PodLogDownloadHandler(string description, LogFile log)
+        {
+            this.description = description;
+            this.log = log;
+        }
+
+        public CodexNodeLog CreateCodexNodeLog()
+        {
+            return new CodexNodeLog(log);
+        }
+
+        public void Log(Stream stream)
+        {
+            log.Write($"{description} -->> {log.FilenameWithoutPath}");
+            log.WriteRaw(description);
+            var reader = new StreamReader(stream);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                log.WriteRaw(line);
+                line = reader.ReadLine();
+            }
+        }
+    }
+}

CodexDistTestCore/PodLogsHandler.cs

@@ -1,7 +0,0 @@
-namespace CodexDistTestCore
-{
-    public interface IPodLogsHandler
-    {
-        void Log(int id, string podDescription, Stream log);
-    }
-}

CodexDistTestCore/TestLog.cs

@@ -1,16 +1,20 @@
-using NUnit.Framework;
+using CodexDistTestCore.Config;
+using NUnit.Framework;
 
 namespace CodexDistTestCore
 {
     public class TestLog
     {
-        public const string LogRoot = "D:/CodexTestLogs";
+        private readonly NumberSource subfileNumberSource = new NumberSource(0);
         private readonly LogFile file;
+        private readonly DateTime now;
 
         public TestLog()
         {
+            now = DateTime.UtcNow;
+
             var name = GetTestName();
-            file = new LogFile(name);
+            file = new LogFile(now, name);
 
             Log($"Begin: {name}");
         }
@@ -25,26 +29,23 @@ namespace CodexDistTestCore
             Log($"[ERROR] {message}");
         }
 
-        public void EndTest(K8sManager k8sManager)
+        public void EndTest()
         {
             var result = TestContext.CurrentContext.Result;
 
             Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
-            
             if (!string.IsNullOrEmpty(result.Message))
             {
                 Log(result.Message);
-            }
-
-            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
-            {
                 Log($"{result.StackTrace}");
-
-                var logWriter = new PodLogWriter(file);
-                logWriter.IncludeFullPodLogging(k8sManager);
             }
         }
 
+        public LogFile CreateSubfile()
+        {
+            return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}");
+        }
+
         private static string GetTestName()
         {
             var test = TestContext.CurrentContext.Test;
@@ -60,70 +61,36 @@ namespace CodexDistTestCore
         }
     }
 
-    public class PodLogWriter : IPodLogsHandler
-    {
-        private readonly LogFile file;
-
-        public PodLogWriter(LogFile file)
-        {
-            this.file = file;
-        }
-
-        public void IncludeFullPodLogging(K8sManager k8sManager)
-        {
-            file.Write("Full pod logging:");
-            k8sManager.FetchAllPodsLogs(this);
-        }
-
-        public void Log(int id, string podDescription, Stream log)
-        {
-            var logFile = id.ToString().PadLeft(6, '0');
-            file.Write($"{podDescription} -->> {logFile}");
-            LogRaw(podDescription, logFile);
-            var reader = new StreamReader(log);
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                LogRaw(line, logFile);
-                line = reader.ReadLine();
-            }
-        }
-
-        private void LogRaw(string message, string filename)
-        {
-            file!.WriteRaw(message, filename);
-        }
-    }
-
     public class LogFile
     {
         private readonly string filepath;
-        private readonly string filename;
 
-        public LogFile(string name)
+        public LogFile(DateTime now, string name)
         {
-            var now = DateTime.UtcNow;
-
             filepath = Path.Join(
-                TestLog.LogRoot,
+                LogConfig.LogRoot,
                 $"{now.Year}-{Pad(now.Month)}",
                 Pad(now.Day));
 
             Directory.CreateDirectory(filepath);
 
-            filename = Path.Combine(filepath, $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}");
+            FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log";
+            FullFilename = Path.Combine(filepath, FilenameWithoutPath);
         }
 
+        public string FullFilename { get; }
+        public string FilenameWithoutPath { get; }
+
         public void Write(string message)
         {
             WriteRaw($"{GetTimestamp()} {message}");
         }
 
-        public void WriteRaw(string message, string subfile = "")
+        public void WriteRaw(string message)
         {
             try
             {
-                File.AppendAllLines(filename + subfile + ".log", new[] { message });
+                File.AppendAllLines(FullFilename, new[] { message });
             }
             catch (Exception ex)
             {

Tests/BasicTests/SimpleTests.cs

@@ -1,4 +1,5 @@
 using CodexDistTestCore;
+using CodexDistTestCore.Config;
 using NUnit.Framework;
 
 namespace Tests.BasicTests
@@ -19,13 +20,20 @@ namespace Tests.BasicTests
             Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision()));
         }
 
+        [Test, DontDownloadLogsOnFailure]
+        public void CanAccessLogs()
+        {
+            var node = SetupCodexNodes(1).BringOnline()[0];
+
+            var log = node.DownloadLog();
+
+            log.AssertLogContains("Started codex node");
+        }
+
         [Test]
         public void OneClientTest()
         {
-            var primary = SetupCodexNodes(1)
+            var primary = SetupCodexNodes(1).BringOnline()[0];
-                            .WithLogLevel(CodexLogLevel.Trace)
-                            .WithStorageQuota(2.MB())
-                            .BringOnline()[0];
 
             var testFile = GenerateTestFile(1.MB());
 
@@ -37,12 +45,9 @@ namespace Tests.BasicTests
         }
 
         [Test]
-        public void TwoClientOnePodTest()
+        public void TwoClientsOnePodTest()
         {
-            var group = SetupCodexNodes(2)
+            var group = SetupCodexNodes(2).BringOnline();
-                        .WithLogLevel(CodexLogLevel.Trace)
-                        .WithStorageQuota(2.MB())
-                        .BringOnline();
 
             var primary = group[0];
             var secondary = group[1];
@@ -51,14 +56,24 @@ namespace Tests.BasicTests
         }
 
         [Test]
-        public void TwoClientTwoPodTest()
+        public void TwoClientsTwoPodsTest()
+        {
+            var primary = SetupCodexNodes(1).BringOnline()[0];
+
+            var secondary = SetupCodexNodes(1).BringOnline()[0];
+
+            PerformTwoClientTest(primary, secondary);
+        }
+
+        [Test]
+        public void TwoClientsTwoLocationsTest()
         {
             var primary = SetupCodexNodes(1)
-                            .WithStorageQuota(2.MB())
+                            .At(Location.BensLaptop)
                             .BringOnline()[0];
 
             var secondary = SetupCodexNodes(1)
-                            .WithStorageQuota(2.MB())
+                            .At(Location.BensOldGamingMachine)
                             .BringOnline()[0];
 
             PerformTwoClientTest(primary, secondary);