moves to tools folder

Use ETH_PRIVATE_KEY variable instead of deprecated PRIV_KEY

Framework/Utils/BlockInterval.cs

@@ -15,12 +15,13 @@
                 To = from;
             }
             TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
+        public ulong NumberOfBlocks { get; }
 
         public override string ToString()
         {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -75,11 +75,12 @@ namespace CodexContractsPlugin.ChainMonitor
                 throw new Exception(msg);
             }
 
-            log.Log($"ChainState updating: {events.BlockInterval}");
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "39-0C-32-A3-EA-90-4F-29-1C-67-12-F1-D5-BE-31-67-8D-90-43-1E-F2-02-63-5B-0C-49-F7-1E-E5-EC-F7-00";
+        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -63,10 +63,10 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(fileStream),
+                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
@@ -82,7 +82,7 @@ namespace CodexPlugin
 
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync("", "")));
+            return mapper.Map(OnCodex(api => api.ListDataAsync()));
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
@@ -261,4 +261,18 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -109,7 +109,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -18,6 +18,7 @@ namespace CodexPlugin
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDatasetList LocalFiles();
@@ -138,6 +139,11 @@ namespace CodexPlugin
         }
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            return UploadFile(file, "application/x-binary", $"attachment; filename=\"{file.Filename}\"", onFailure);
+        }
+
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -145,10 +151,11 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(input, onFailure);
             });
 
             var response = measurement.Value;

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -182,8 +182,8 @@ namespace CodexPlugin
             return new Manifest
             {
                 BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
-                RootHash = manifest.RootHash,
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
                 Protected = manifest.Protected
             };
         }

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -344,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        rootHash:
+        treeCid:
           $ref: "#/components/schemas/Cid"
-          description: "Root hash of the content"
-        originalBytes:
+          description: "Unique data identifier"
+        datasetSize:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -359,15 +359,18 @@ components:
           description: "Indicates if content is protected by erasure-coding"
         filename:
           type: string
+          nullable: true
           description: "The original name of the uploaded content (optional)"
           example: codex.png
         mimetype:
           type: string
+          nullable: true
           description: "The original mimetype of the uploaded content (optional)"
           example: image/png
         uploadedAt:
           type: integer
           format: int64
+          nullable: true
           description: "The UTC upload timestamp in seconds"
           example: 1729244192
 
@@ -443,21 +446,6 @@ paths:
       summary: "Lists manifest CIDs stored locally in node."
       tags: [ Data ]
       operationId: listData
-      parameters:
-        - name: content-type
-          in: header
-          required: false
-          description: The content type of the file. Must be valid.
-          schema:
-            type: string
-            example: "image/png"
-        - name: content-disposition
-          in: header
-          required: false
-          description: The content disposition used to send the filename.
-          schema:
-            type: string
-            example: "attachment; filename=\"codex.png\""
       responses:
         "200":
           description: Retrieved list of content CIDs
@@ -478,6 +466,21 @@ paths:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:

Tools/AutoClient/Purchaser.cs

@@ -119,7 +119,7 @@ namespace AutoClient
             {
                 var info = new FileInfo(filename);
                 var sw = System.Diagnostics.Stopwatch.StartNew();
-                var cid = await UploadStream(fileStream);
+                var cid = await UploadStream(fileStream, filename);
                 var time = sw.Elapsed;
                 app.Performance.UploadSuccessful(info.Length, time);
                 app.CidRepo.Add(nodeId, cid.Id, info.Length);
@@ -132,10 +132,13 @@ namespace AutoClient
             }
         }
 
-        private async Task<ContentId> UploadStream(FileStream fileStream)
+        private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
         {
             log.Debug($"Uploading file...");
-            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
+            var response = await codex.UploadAsync(
+                content_type: "application/x-binary",
+                content_disposition: $"attachment; filename=\"{filename}\"",
+                fileStream, app.Cts.Token);
 
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");

Tools/BiblioTech/CodexCidChecker.cs

@@ -1,5 +1,6 @@
 using CodexOpenApi;
 using IdentityModel.Client;
+using Logging;
 using Utils;
 
 namespace BiblioTech
@@ -8,11 +9,13 @@ namespace BiblioTech
     {
         private static readonly string nl = Environment.NewLine;
         private readonly Configuration config;
+        private readonly ILog log;
         private CodexApi? currentCodexNode;
 
-        public CodexCidChecker(Configuration config)
+        public CodexCidChecker(Configuration config, ILog log)
         {
             this.config = config;
+            this.log = log;
         }
 
         public async Task<CheckResponse> PerformCheck(string cid)
@@ -63,7 +66,7 @@ namespace BiblioTech
                 success: true,
                 title: $"Success: '{content.Cid}'",
                 error: "",
-                $"size: {content.Manifest.OriginalBytes} bytes",
+                $"size: {content.Manifest.DatasetSize} bytes",
                 $"blockSize: {content.Manifest.BlockSize} bytes",
                 $"protected: {content.Manifest.Protected}"
             );
@@ -150,6 +153,7 @@ namespace BiblioTech
             }
             catch (Exception e)
             {
+                log.Error(e.ToString());
                 return false;
             }
         }

Tools/BiblioTech/Program.cs

@@ -80,7 +80,7 @@ namespace BiblioTech
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
-            var checker = new CodexCidChecker(Config);
+            var checker = new CodexCidChecker(Config, Log);
             var notifyCommand = new NotifyCommand();
             var associateCommand = new UserAssociateCommand(notifyCommand);
             var sprCommand = new SprCommand();

Tools/CodexUnitTestCrusher/CodexUnitTestCrusher.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

Tools/CodexUnitTestCrusher/Program.cs

@@ -0,0 +1,215 @@
+namespace CodexUnitTestCrusher
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            var p = new Program();
+            p.Run();
+        }
+
+        private const bool AddRandomSleeps = true;
+        private const bool GenerateLoopTestRunners = true;
+
+        private readonly string Root = "C:\\Projects\\nim-codex";
+        private readonly string[] Exclude =
+        [
+            "vendor"
+        ];
+
+        private readonly List<string> scriptPaths = new List<string>();
+        private readonly string Include = "import std/random";
+        private readonly string SleepLine = "await sleepAsync(rand(10))";
+        private readonly int NumCompiles = 10;
+        private readonly int NumRuns = 100;
+        private readonly string[] TestRunner =
+        [
+            "set -e",
+            "for i in {0..<NUMCOMPILES>}",
+            "do",
+            "    echo \"#1\" >> \"<TESTFILE>\"",
+                        "    for j in {0..<NUMRUNS>}",
+                        "    do",
+            "        nim c -r \"<TESTFILE>\"",
+            "    done",
+            "done",
+            "rm <SCRIPTFILE>",
+        ];
+
+        public void Run()
+        {
+            TraverseFolder(Root);
+            CreateRunScripts();
+        }
+
+        private void CreateRunScripts()
+        {
+            var lineLines = new List<List<string>>();
+            lineLines.Add(new List<string>());
+            lineLines.Add(new List<string>());
+            lineLines.Add(new List<string>());
+
+            var i = 0;
+            foreach (var script in scriptPaths)
+            {
+                lineLines[i].Add($"sh \"{script}\"");
+                i = (i + 1) % lineLines.Count;
+            }
+            File.WriteAllLines(@"C:\Projects\nim-codex\runall1.sh", lineLines[0]);
+            File.WriteAllLines(@"C:\Projects\nim-codex\runall2.sh", lineLines[1]);
+            File.WriteAllLines(@"C:\Projects\nim-codex\runall3.sh", lineLines[2]);
+        }
+
+        private void TraverseFolder(string root)
+        {
+            if (Exclude.Any(x => root.Contains(x))) return;
+
+            var folder = Directory.GetDirectories(root);
+            foreach (var dir in folder) TraverseFolder(dir);
+
+            var files = Directory.GetFiles(root);
+            foreach (var file in files) ProcessFile(file);
+        }
+
+        private void ProcessFile(string file)
+        {
+            if (!file.EndsWith(".nim")) return;
+
+            if (AddRandomSleeps) AddRandomSleepsToNimFile(file);
+            if (GenerateLoopTestRunners) GenerateTestRunner(file);
+        }
+
+        private void GenerateTestRunner(string file)
+        {
+            var filename = Path.GetFileName(file);
+            if (!filename.StartsWith("test")) return;
+            var path = Path.GetDirectoryName(file);
+
+            var testFile = file;
+            var scriptFile = filename.Replace(".nim", ".sh");
+            WriteScriptFile(path!, testFile, scriptFile);
+        }
+
+        private void WriteScriptFile(string path, string testFile, string scriptFile)
+        {
+            var lines = TestRunner.Select(l =>
+            {
+                return l
+                    .Replace("<NUMCOMPILES>", NumCompiles.ToString())
+                    .Replace("<NUMRUNS>", NumRuns.ToString())
+                    .Replace("<TESTFILE>", testFile.ToString())
+                    .Replace("<SCRIPTFILE>", scriptFile.ToString())
+                ;
+
+            }).ToArray();
+
+            var fullPath = Path.Combine(path, scriptFile);
+            File.WriteAllLines(fullPath, lines);
+            scriptPaths.Add(fullPath);
+        }
+
+        private void AddRandomSleepsToNimFile(string file)
+        {
+            Console.WriteLine("Processing file: " + file);
+
+            var lines = File.ReadAllLines(file).ToList();
+            if (!lines.Any(l => l == Include))
+            {
+                AddInclude(lines);
+            }
+
+            var modified = false;
+            for (int i = 0; i < lines.Count; i++)
+            {
+                if (ProcessLine(i, lines[i], lines))
+                {
+                    i++;
+                    modified = true;
+                }
+            }
+
+            if (modified) File.WriteAllLines(file, lines);
+        }
+
+        private bool ProcessLine(int i, string line, List<string> lines)
+        {
+            if (IsComment(line)) return false;
+
+            if (line.Contains("await "))
+            {
+                if (!line.Contains("sleep"))
+                {
+                    var previous = GetPreviousLine(i, lines);
+                    if (previous != null)
+                    {
+                        var trim = previous.Trim();
+                        // previous line was "let" ???
+                        if (trim == "let")
+                        {
+                            // insert before let.
+                            InsertSleepLine(i - 1, lines);
+                            return true;
+                        }
+                        // previous line was "without =?" ??
+                        if (trim.StartsWith("without") && trim.EndsWith("=?"))
+                        {
+                            // insert before without.
+                            InsertSleepLine(i - 1, lines);
+                            return true;
+                        }
+                        // previous line was "const" ??
+                        if (trim == "const") return false;
+                    }
+
+                    var indent = GetIndent(line);
+                    if (indent.Length < 3) InsertSleepLine(i, lines);
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        private void InsertSleepLine(int i, List<string> lines)
+        {
+            lines.Insert(i, GetIndent(lines[i]) + SleepLine);
+        }
+
+        private string? GetPreviousLine(int i, List<string> lines)
+        {
+            var idx = i - 1;
+            if (idx < 0) return null;
+            return lines[idx];
+        }
+
+        private bool IsComment(string line)
+        {
+            return line.Trim().StartsWith("#");
+        }
+
+        private string GetIndent(string line)
+        {
+            var result = "";
+
+            while (line.StartsWith(" "))
+            {
+                result += " ";
+                line = line.Substring(1);
+            }
+
+            return result;
+        }
+
+        private void AddInclude(List<string> lines)
+        {
+            for (var i = 0; i < lines.Count; i++)
+            {
+                var line = lines[i];
+                if (line.StartsWith("import "))
+                {
+                    lines.Insert(i, Include);
+                    return;
+                }
+            }
+        }
+    }
+}

cs-codex-dist-testing.sln

@@ -78,6 +78,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\Mar
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexUnitTestCrusher", "Tools\CodexUnitTestCrusher\CodexUnitTestCrusher.csproj", "{E20302D1-DF71-4E02-B362-3E46675E6E8F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -208,6 +210,10 @@ Global
 		{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E20302D1-DF71-4E02-B362-3E46675E6E8F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E20302D1-DF71-4E02-B362-3E46675E6E8F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E20302D1-DF71-4E02-B362-3E46675E6E8F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E20302D1-DF71-4E02-B362-3E46675E6E8F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -244,6 +250,7 @@ Global
 		{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
+		{E20302D1-DF71-4E02-B362-3E46675E6E8F} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}