Fixes re-purchase issue

Tools/AutoClient/Modes/FolderStore/FileStatus.cs

@@ -0,0 +1,45 @@
+using static AutoClient.Modes.FolderStore.FileWorker;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileStatus : JsonBacked<WorkerStatus>
+    {
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo)
+            : base(app, folder, filePath + ".json")
+        {
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public bool IsCurrentlyRunning()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyFailed()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            var mostRecent = GetMostRecent();
+            if (mostRecent == null) return false;
+
+            return mostRecent.Expiry.HasValue;
+        }
+
+        protected WorkerPurchase? GetMostRecent()
+        {
+            if (!State.Purchases.Any()) return null;
+            var maxCreated = State.Purchases.Max(p => p.Created);
+            return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FileWorker.cs

@@ -1,30 +1,39 @@
-using static AutoClient.Modes.FolderStore.FileWorker;
+using Logging;
+using Nethereum.Contracts;
 
 namespace AutoClient.Modes.FolderStore
 {
-    public class FileWorker : JsonBacked<WorkerStatus>
+    public class FileWorker : FileStatus
     {
         private readonly App app;
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
         private readonly PurchaseInfo purchaseInfo;
         private readonly string sourceFilename;
+        private readonly Action onNewPurchase;
+        private readonly CodexNode codex;
 
-        public FileWorker(App app, PurchaseInfo purchaseInfo, string folder, string filename)
+        public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, string filename, Action onNewPurchase)
-            : base(app, folder, filename + ".json")
+            : base(app, folder, filename + ".json", purchaseInfo)
         {
             this.app = app;
+            log = new LogPrefixer(app.Log, GetFileTag(filename));
+            this.instance = instance;
             this.purchaseInfo = purchaseInfo;
             sourceFilename = filename;
+            this.onNewPurchase = onNewPurchase;
+            codex = new CodexNode(app, instance);
         }
 
         public int FailureCounter => State.FailureCounter;
 
-        public async Task Update(ICodexInstance instance, Action shouldRevisitSoon)
+        public async Task Update()
         {
             try
             {
-                var codex = new CodexNode(app, instance);
+                Log($"Updating for '{sourceFilename}'...");
-                await EnsureCid(instance, codex);
+                await EnsureCid();
-                await EnsureRecentPurchase(instance, codex, shouldRevisitSoon);
+                await EnsureRecentPurchase();
                 SaveState();
                 app.Log.Log("");
             }
@@ -35,64 +44,95 @@ namespace AutoClient.Modes.FolderStore
             }
         }
 
-        private async Task EnsureRecentPurchase(ICodexInstance instance, CodexNode codex, Action shouldRevisitSoon)
+        private async Task EnsureCid()
         {
-            app.Log.Log($"Ensuring recent purchase for '{sourceFilename}'...");
+            Log($"Ensuring CID...");
+            if (!string.IsNullOrEmpty(State.Cid))
+            {
+                var found = true;
+                try
+                {
+                    var manifest = await instance.Codex.DownloadNetworkManifestAsync(State.Cid);
+                    if (manifest == null) found = false;
+                }
+                catch
+                {
+                    found = false;
+                }
+
+                if (!found)
+                {
+                    Log($"Existing CID '{State.Cid}' could not be found in the network.");
+                    State.Cid = "";
+                }
+                else
+                {
+                    Log($"Existing CID '{State.Cid}' was successfully found in the network.");
+                }
+            }
+
+            if (string.IsNullOrEmpty(State.Cid))
+            {
+                Log($"Uploading...");
+                var cid = await codex.UploadFile(sourceFilename);
+                Log("Got CID: " + cid);
+                State.Cid = cid.Id;
+                Thread.Sleep(1000);
+            }
+        }
+
+        private async Task EnsureRecentPurchase()
+        {
+            Log($"Ensuring recent purchase...");
             var recent = GetMostRecent();
             if (recent == null)
             {
-                app.Log.Log($"No recent purchase for '{sourceFilename}'.");
+                Log($"No recent purchase.");
-                await MakeNewPurchase(instance, codex);
+                await MakeNewPurchase();
-                shouldRevisitSoon();
                 return;
             }
 
-            await UpdatePurchase(recent, instance, codex);
+            await UpdatePurchase(recent);
 
             if (recent.Expiry.HasValue)
             {
-                app.Log.Log($"Purchase for '{sourceFilename}' has failed or expired.");
+                Log($"Purchase has failed or expired.");
-                await MakeNewPurchase(instance, codex);
+                await MakeNewPurchase();
-                shouldRevisitSoon();
                 State.FailureCounter++;
                 return;
             }
 
             if (recent.Finish.HasValue)
             {
-                app.Log.Log($"Purchase for '{sourceFilename}' has finished.");
+                Log($"Purchase has finished.");
-                await MakeNewPurchase(instance, codex);
+                await MakeNewPurchase();
-                shouldRevisitSoon();
                 return;
             }
 
-            if (recent.Started.HasValue &&
+            var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe;
-                recent.Created + purchaseInfo.PurchaseDurationSafe > DateTime.UtcNow)
+            if (recent.Started.HasValue && DateTime.UtcNow > safeEnd)
             {
-                app.Log.Log($"Purchase for '{sourceFilename}' is going to expire soon.");
+                Log($"Purchase is going to expire soon.");
-                await MakeNewPurchase(instance, codex);
+                await MakeNewPurchase();
-                shouldRevisitSoon();
                 return;
             }
 
             if (!recent.Submitted.HasValue)
             {
-                app.Log.Log($"Purchase for '{sourceFilename}' is waiting to be submitted.");
+                Log($"Purchase is waiting to be submitted.");
-                shouldRevisitSoon();
                 return;
             }
 
             if (recent.Submitted.HasValue && !recent.Started.HasValue)
             {
-                app.Log.Log($"Purchase for '{sourceFilename}' is submitted and waiting to start.");
+                Log($"Purchase is submitted and waiting to start.");
-                shouldRevisitSoon();
                 return;
             }
 
-            app.Log.Log($"Purchase for '{sourceFilename}' is running.");
+            Log($"Purchase is running.");
         }
 
-        private async Task UpdatePurchase(WorkerPurchase recent, ICodexInstance instance, CodexNode codex)
+        private async Task UpdatePurchase(WorkerPurchase recent)
         {
             if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!");
             var now = DateTime.UtcNow;
@@ -100,7 +140,7 @@ namespace AutoClient.Modes.FolderStore
             var purchase = await codex.GetStoragePurchase(recent.Pid);
             if (purchase == null)
             {
-                app.Log.Log($"No purchase information found for PID '{recent.Pid}' for file '{sourceFilename}'. Consider this one expired.");
+                Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired.");
                 recent.Expiry = now;
                 return;
             }
@@ -131,17 +171,14 @@ namespace AutoClient.Modes.FolderStore
                 if (!recent.Finish.HasValue) recent.Finish = now;
             }
 
-            app.Log.Log($"Updated purchase information for PID '{recent.Pid}' for file '{sourceFilename}': " +
+            Log($"Updated purchase information for PID '{recent.Pid}'.");
-                $"Submitted: {recent.Submitted.HasValue} " +
-                $"Started: {recent.Started.HasValue} " +
-                $"Expiry: {recent.Expiry.HasValue} " +
-                $"Finish: {recent.Finish.HasValue}");
         }
 
-        private async Task MakeNewPurchase(ICodexInstance instance, CodexNode codex)
+        private async Task MakeNewPurchase()
         {
             if (string.IsNullOrEmpty(State.Cid)) throw new Exception("No cid!");
 
+            Log($"Creating new purchase...");
             var response = await codex.RequestStorage(new CodexPlugin.ContentId(State.Cid));
             if (string.IsNullOrEmpty(response) ||
                 response == "Unable to encode manifest" ||
@@ -153,83 +190,39 @@ namespace AutoClient.Modes.FolderStore
                 throw new InvalidOperationException(response);
             }
 
-            State.Purchases = State.Purchases.Concat([
+            var newPurchase = new WorkerPurchase
-                new WorkerPurchase
             {
                 Created = DateTime.UtcNow,
                 Pid = response
-                }
+            };
-            ]).ToArray();
+            State.Purchases = State.Purchases.Concat([newPurchase]).ToArray();
 
-            app.Log.Log($"New purchase created for '{sourceFilename}'. PID: '{response}'");
+            Log($"New purchase created. PID: '{response}'. Waiting for submit...");
             Thread.Sleep(500);
+            onNewPurchase();
+
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            while (DateTime.UtcNow < timeout)
+            {
+                await UpdatePurchase(newPurchase);
+                if (newPurchase.Submitted.HasValue)
+                {
+                    Log("New purchase successfully submitted.");
+                    return;
+                }
+            }
+            Log("New purchase was not submitted within 5-minute timeout. Will check again later...");
         }
 
-        private async Task EnsureCid(ICodexInstance instance, CodexNode codex)
+        private void Log(string msg)
         {
-            app.Log.Log($"Ensuring CID for '{sourceFilename}'...");
+            log.Log(msg);
-            if (!string.IsNullOrEmpty(State.Cid))
-            {
-                var found = true;
-                try
-                {
-                    var manifest = await instance.Codex.DownloadNetworkManifestAsync(State.Cid);
-                    if (manifest == null) found = false;
-                }
-                catch
-                {
-                    found = false;
         }
 
-                if (!found)
+        private string GetFileTag(string filename)
         {
-                    app.Log.Log($"Existing CID '{State.Cid}' for '{sourceFilename}' could not be found in the network.");
+            var i = Math.Abs(filename.GetHashCode() % 9999);
-                    State.Cid = "";
+            return $"({i.ToString("0000")}) ";
-                }
-                else
-                {
-                    app.Log.Log($"Existing CID '{State.Cid}' for '{sourceFilename}' was successfully found in the network.");
-                }
-            }
-
-            if (string.IsNullOrEmpty(State.Cid))
-            {
-                app.Log.Log($"Uploading '{sourceFilename}'...");
-                var cid = await codex.UploadFile(sourceFilename);
-                app.Log.Log("Got CID: " + cid);
-                State.Cid = cid.Id;
-                Thread.Sleep(1000);
-            }
-        }
-
-        private WorkerPurchase? GetMostRecent()
-        {
-            if (!State.Purchases.Any()) return null;
-            var maxCreated = State.Purchases.Max(p => p.Created);
-            return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
-        }
-
-        public bool IsCurrentlyRunning()
-        {
-            if (!State.Purchases.Any()) return false;
-
-            return State.Purchases.Any(p =>
-                p.Submitted.HasValue &&
-                p.Started.HasValue &&
-                !p.Expiry.HasValue &&
-                !p.Finish.HasValue &&
-                p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
-            );
-        }
-
-        public bool IsCurrentlyFailed()
-        {
-            if (!State.Purchases.Any()) return false;
-
-            var mostRecent = GetMostRecent();
-            if (mostRecent == null) return false;
-
-            return mostRecent.Expiry.HasValue;
         }
 
         [Serializable]

Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs

@@ -2,13 +2,13 @@
 {
     public class FolderWorkDispatcher
     {
-        private readonly List<string> files = new List<string>();
+        private readonly string[] files = Array.Empty<string>();
-        private readonly List<string> revisitSoon = new List<string>();
+        private int index = 0;
-        public bool Revisiting { get; private set; } = false;
 
         public FolderWorkDispatcher(string folder)
         {
             var fs = Directory.GetFiles(folder);
+            var result = new List<string>();
             foreach (var f in fs)
             {
                 if (!f.ToLowerInvariant().Contains(".json"))
@@ -16,40 +16,23 @@
                     var info = new FileInfo(f);
                     if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB
                     {
-                        files.Add(f);
+                        result.Add(f);
                     }
                 }
             }
+            files = result.ToArray();
         }
 
         public string GetFileToCheck()
         {
-            if (Revisiting)
+            var file = files[index];
-            {
+            index = (index + 1) % files.Length;
-                if (!revisitSoon.Any())
-                {
-                    Revisiting = false;
-                    return GetFileToCheck();
-                }
-
-                var file = revisitSoon.First();
-                revisitSoon.RemoveAt(0);
             return file;
         }
-            else
-            {
-                var file = files.First();
-                files.RemoveAt(0);
-                files.Add(file);
 
-                if (revisitSoon.Count > 3) Revisiting = true;
+        public void ResetIndex()
-                return file;
-            }
-        }
-
-        public void RevisitSoon(string file)
         {
-            revisitSoon.Add(file);
+            index = 0;
         }
     }
 }

Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs

@@ -26,7 +26,7 @@ namespace AutoClient.Modes.FolderStore
             {
                 try
                 {
-                    var worker = new FileWorker(app, purchaseInfo, Folder, file.Substring(0, file.Length - 5));
+                    var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo);
                     total++;
                     if (worker.IsCurrentlyRunning()) successful++;
                     if (worker.IsCurrentlyFailed()) failed++;

Tools/AutoClient/Modes/FolderStoreMode.cs

@@ -38,8 +38,7 @@ namespace AutoClient.Modes
             var i = 0;
             while (!cts.IsCancellationRequested)
             {
-                if (app.FolderWorkDispatcher.Revisiting) Thread.Sleep(2000);
+                Thread.Sleep(2000);
-                else Thread.Sleep(5000);
 
                 var worker = await ProcessWorkItem(instance);
                 if (worker.FailureCounter > 5)
@@ -60,15 +59,16 @@ namespace AutoClient.Modes
         private async Task<FileWorker> ProcessWorkItem(ICodexInstance instance)
         {
             var file = app.FolderWorkDispatcher.GetFileToCheck();
-            var worker = new FileWorker(app, purchaseInfo, folder, file);
+            var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnNewPurchase);
-            await worker.Update(instance, () =>
+            await worker.Update();
-            {
-                app.FolderWorkDispatcher.RevisitSoon(file);
-            });
-
             return worker;
         }
 
+        private void OnNewPurchase()
+        {
+            app.FolderWorkDispatcher.ResetIndex();
+        }
+
         public void Stop()
         {
             cts.Cancel();

Tools/AutoClient/Program.cs

@@ -1,6 +1,7 @@
 using ArgsUniform;
 using AutoClient;
 using AutoClient.Modes;
+using AutoClient.Modes.FolderStore;
 using CodexOpenApi;
 using Utils;