diff --git a/ProjectPlugins/CodexClient/CodexNode.cs b/ProjectPlugins/CodexClient/CodexNode.cs index 4a444117..7bf5f4d2 100644 --- a/ProjectPlugins/CodexClient/CodexNode.cs +++ b/ProjectPlugins/CodexClient/CodexNode.cs @@ -299,7 +299,7 @@ namespace CodexClient private void InitializePeerNodeId() { - var debugInfo = Time.Retry(codexAccess.GetDebugInfo, "ensure online"); + var debugInfo = codexAccess.GetDebugInfo(); if (!debugInfo.Version.IsValid()) { throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}"); diff --git a/Tools/BiblioTech/AdminChecker.cs b/Tools/BiblioTech/AdminChecker.cs index 96f89691..5c7ff103 100644 --- a/Tools/BiblioTech/AdminChecker.cs +++ b/Tools/BiblioTech/AdminChecker.cs @@ -35,6 +35,7 @@ namespace BiblioTech public async Task SendInAdminChannel(string[] lines) { + if (adminChannel == null) return; var chunker = new LineChunker(lines); var chunks = chunker.GetChunks(); if (!chunks.Any()) return; diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs index 47508b4b..6fb2f28e 100644 --- a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -15,6 +15,7 @@ namespace BiblioTech.CodexChecking Task CouldNotDownloadCid(); Task GiveCidToUser(string cid); Task GiveDataFileToUser(string fileContent); + Task CodexUnavailable(); Task ToAdminChannel(string msg); } @@ -43,7 +44,13 @@ namespace BiblioTech.CodexChecking repo.SaveChanges(); } - var cid = await UploadData(check.UniqueData); + var cid = UploadData(check.UniqueData); + if (cid == null) + { + await handler.CodexUnavailable(); + return; + } + await handler.GiveCidToUser(cid); } @@ -87,7 +94,7 @@ namespace BiblioTech.CodexChecking return; } - var manifest = await GetManifest(receivedCid); + var manifest = GetManifest(receivedCid); if (manifest == null) { await handler.CouldNotDownloadCid(); @@ -96,7 +103,12 @@ namespace BiblioTech.CodexChecking if (IsManifestLengthCompatible(handler, check, manifest)) { - if (await IsContentCorrect(handler, check, receivedCid)) + var correct = IsContentCorrect(handler, check, receivedCid); + if (!correct.HasValue) { + await handler.CodexUnavailable(); + return; + } + if (correct.Value) { await CheckNowCompleted(handler, check, userId, "UploadCheck"); return; @@ -120,7 +132,7 @@ namespace BiblioTech.CodexChecking check.CompletedUtc < expiry; } - private async Task UploadData(string uniqueData) + private string? UploadData(string uniqueData) { var filePath = Path.Combine(config.ChecksDataPath, Guid.NewGuid().ToString()); @@ -129,7 +141,7 @@ namespace BiblioTech.CodexChecking File.WriteAllText(filePath, uniqueData); var file = new TrackedFile(log, filePath, "checkData"); - return await codexWrapper.OnCodex(node => + return codexWrapper.OnCodex(node => { return node.UploadFile(file).Id; }); @@ -145,11 +157,11 @@ namespace BiblioTech.CodexChecking } } - private async Task GetManifest(string receivedCid) + private Manifest? GetManifest(string receivedCid) { try { - return await codexWrapper.OnCodex(node => + return codexWrapper.OnCodex(node => { return node.DownloadManifestOnly(new ContentId(receivedCid)).Manifest; }); @@ -172,11 +184,11 @@ namespace BiblioTech.CodexChecking manifestLength < (dataLength + 1); } - private async Task IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid) + private bool? IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid) { try { - var content = await codexWrapper.OnCodex(node => + var content = codexWrapper.OnCodex(node => { var file = node.DownloadContent(new ContentId(receivedCid)); if (file == null) return string.Empty; @@ -190,6 +202,8 @@ namespace BiblioTech.CodexChecking } }); + if (content == null) return null; + Log($"Checking content: content={content},check={check.UniqueData}"); return content == check.UniqueData; } diff --git a/Tools/BiblioTech/CodexChecking/CodexWrapper.cs b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs index f7f871fa..f1f35822 100644 --- a/Tools/BiblioTech/CodexChecking/CodexWrapper.cs +++ b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs @@ -21,38 +21,72 @@ namespace BiblioTech.CodexChecking var httpFactory = CreateHttpFactory(); factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath); + + Task.Run(CheckCodexNode); } - public async Task OnCodex(Action action) + public T? OnCodex(Func func) where T : class { - await Task.Run(() => + lock (codexLock) { - lock (codexLock) - { - action(Get()); - } - }); - } - - public async Task OnCodex(Func func) - { - return await Task.Run(() => - { - lock (codexLock) - { - return func(Get()); - } - }); - } - - private ICodexNode Get() - { - if (currentCodexNode == null) - { - currentCodexNode = CreateCodex(); + if (currentCodexNode == null) return null; + return func(currentCodexNode); } + } - return currentCodexNode; + private void CheckCodexNode() + { + Thread.Sleep(TimeSpan.FromSeconds(10.0)); + + while (true) + { + lock (codexLock) + { + var newNode = GetNewCodexNode(); + if (newNode != null && currentCodexNode == null) ShowConnectionRestored(); + if (newNode == null && currentCodexNode != null) ShowConnectionLost(); + currentCodexNode = newNode; + } + + Thread.Sleep(TimeSpan.FromMinutes(15.0)); + } + } + + private ICodexNode? GetNewCodexNode() + { + try + { + if (currentCodexNode != null) + { + try + { + // Current instance is responsive? Keep it. + var info = currentCodexNode.GetDebugInfo(); + if (info != null && info.Version != null && + !string.IsNullOrEmpty(info.Version.Revision)) return currentCodexNode; + } + catch + { + } + } + + return CreateCodex(); + } + catch (Exception ex) + { + log.Error("Exception when trying to check codex node: " + ex.Message); + return null; + } + } + + private void ShowConnectionLost() + { + Program.AdminChecker.SendInAdminChannel("Codex node connection lost."); + } + + private void ShowConnectionRestored() + { + Program.AdminChecker.SendInAdminChannel("Codex node connection restored."); } private ICodexNode CreateCodex() @@ -76,7 +110,7 @@ namespace BiblioTech.CodexChecking { if (string.IsNullOrEmpty(config.CodexEndpointAuth) || !config.CodexEndpointAuth.Contains(":")) { - return new HttpFactory(log); + return new HttpFactory(log, new SnappyTimeSet()); } var tokens = config.CodexEndpointAuth.Split(':'); diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs index f7b1c4f0..c10ef23e 100644 --- a/Tools/BiblioTech/Commands/CheckResponseHandler.cs +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -26,6 +26,11 @@ namespace BiblioTech.Commands await context.Followup("Could not download the CID."); } + public async Task CodexUnavailable() + { + await context.Followup("Couldn't perform check: Our Codex node appears unavailable. Try again later?"); + } + public async Task GiveCidToUser(string cid) { await context.Followup(