Fix Windows MAX_PATH constraint issue in CI. (#4576)

* Fix MAX_PATH limitation in tests.

* Fix posix issues.

* Fix compilation issue.

research/simutils.nim

@@ -6,9 +6,8 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  stew/io2,
+  stats, strformat, times,
-  stats, os, strformat, times,
+  ../tests/testblockutil, ../tests/consensus_spec/os_ops,
-  ../tests/testblockutil,
   ../beacon_chain/beacon_chain_db,
   ../beacon_chain/spec/datatypes/[phase0, altair],
   ../beacon_chain/spec/[beaconstate, deposit_snapshots, forks, helpers],

tests/consensus_spec/altair/test_fixture_operations.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  std/[os, sequtils, sets],
+  std/[sequtils, sets],
   # Utilities
   chronicles,
   unittest2,
@@ -19,7 +19,7 @@ import
   ../../../beacon_chain/spec/datatypes/altair,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ../../helpers/debug_state
 
 const

tests/consensus_spec/altair/test_fixture_rewards.nim

@@ -8,14 +8,12 @@
 {.used.}
 
 import
-  # Standard library
-  std/os,
   # Beacon chain internals
   ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
   ../../beacon_chain/spec/datatypes/altair,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils
+  ../fixtures_utils, ../os_ops
 
 const
   RewardsDirBase = SszTestsDir/const_preset/"altair"/"rewards"

tests/consensus_spec/altair/test_fixture_ssz_consensus_objects.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, strutils, streams, strformat,
+  strutils, streams, strformat,
   macros,
   # Third-party
   yaml,
@@ -18,7 +18,7 @@ import
   # Status libraries
   snappy,
   # Test utilities
-  ../../testutil, ../fixtures_utils
+  ../../testutil, ../fixtures_utils, ../os_ops
 
 # SSZ tests of consensus objects (minimal/mainnet preset specific)
 

tests/consensus_spec/altair/test_fixture_state_transition_epoch.nim

@@ -14,12 +14,10 @@ import
   ../../../beacon_chain/spec/datatypes/altair,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ./test_fixture_rewards,
   ../../helpers/debug_state
 
-from std/os import
-  DirSep, dirExists, fileExists, pcDir, walkDir, walkDirRec, `/`
 from std/sequtils import mapIt, toSeq
 from std/strutils import rsplit
 

tests/consensus_spec/bellatrix/test_fixture_operations.nim

@@ -17,10 +17,9 @@ import
   ../../../beacon_chain/spec/datatypes/bellatrix,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ../../helpers/debug_state
 
-from std/os import fileExists, walkDir, `/`
 from std/sequtils import mapIt, toSeq
 from std/strutils import contains
 
@@ -132,7 +131,7 @@ suite baseDescription & "Execution Payload " & preset():
         executionPayload: bellatrix.ExecutionPayload):
         Result[void, cstring] =
       let payloadValid =
-        readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
+        os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
           contains("execution_valid: true")
       func executePayload(_: bellatrix.ExecutionPayload): bool = payloadValid
       process_execution_payload(

tests/consensus_spec/bellatrix/test_fixture_rewards.nim

@@ -8,14 +8,12 @@
 {.used.}
 
 import
-  # Standard library
-  std/os,
   # Beacon chain internals
   ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
   ../../beacon_chain/spec/datatypes/[altair, bellatrix],
   # Test utilities
   ../../testutil,
-  ../fixtures_utils
+  ../fixtures_utils, ../os_ops
 
 const
   RewardsDirBase = SszTestsDir/const_preset/"bellatrix"/"rewards"

tests/consensus_spec/bellatrix/test_fixture_ssz_consensus_objects.nim

@@ -15,9 +15,8 @@ import
   # Status libraries
   snappy,
   # Test utilities
-  ../../testutil, ../fixtures_utils
+  ../../testutil, ../fixtures_utils, ../os_ops
 
-from std/os import dirExists, pcDir, walkDir, `/`
 from std/streams import close, openFileStream
 from std/strformat import `&`
 from std/strutils import toLowerAscii

tests/consensus_spec/bellatrix/test_fixture_state_transition_epoch.nim

@@ -13,12 +13,10 @@ import
   ../../../beacon_chain/spec/datatypes/[altair, bellatrix],
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ./test_fixture_rewards,
   ../../helpers/debug_state
 
-from std/os import
-  DirSep, dirExists, fileExists, pcDir, walkDir, walkDirRec, `/`
 from std/strutils import rsplit
 from std/sequtils import mapIt, toSeq
 

tests/consensus_spec/capella/test_fixture_operations.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  std/[os, sequtils, sets, strutils],
+  std/[sequtils, sets, strutils],
   # Utilities
   chronicles,
   unittest2,
@@ -19,7 +19,7 @@ import
   ../../../beacon_chain/spec/datatypes/capella,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ../../helpers/debug_state
 
 const
@@ -147,7 +147,7 @@ suite baseDescription & "Execution Payload " & preset():
         executionPayload: capella.ExecutionPayload):
         Result[void, cstring] =
       let payloadValid =
-        readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
+        os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
           contains("execution_valid: true")
       func executePayload(_: capella.ExecutionPayload): bool = payloadValid
       process_execution_payload(

tests/consensus_spec/capella/test_fixture_rewards.nim

@@ -8,14 +8,12 @@
 {.used.}
 
 import
-  # Standard library
-  std/os,
   # Beacon chain internals
   ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
   ../../beacon_chain/spec/datatypes/[altair, capella],
   # Test utilities
   ../../testutil,
-  ../fixtures_utils
+  ../fixtures_utils, ../os_ops
 
 const
   RewardsDirBase = SszTestsDir/const_preset/"capella"/"rewards"

tests/consensus_spec/capella/test_fixture_ssz_consensus_objects.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, strutils, streams, strformat,
+  strutils, streams, strformat,
   macros, sets,
   # Third-party
   yaml,
@@ -18,7 +18,7 @@ import
   # Status libraries
   snappy,
   # Test utilities
-  ../../testutil, ../fixtures_utils
+  ../../testutil, ../fixtures_utils, ../os_ops
 
 from ../../beacon_chain/spec/datatypes/bellatrix import PowBlock
 

tests/consensus_spec/capella/test_fixture_state_transition_epoch.nim

@@ -15,12 +15,10 @@ import
   ../../../beacon_chain/spec/datatypes/[altair, capella],
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ./test_fixture_rewards,
   ../../helpers/debug_state
 
-from std/os import
-  DirSep, dirExists, fileExists, pcDir, walkDir, walkDirRec, `/`
 from std/sequtils import mapIt, toSeq
 from std/strutils import rsplit
 

tests/consensus_spec/eip4844/test_fixture_operations.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  std/[os, sequtils, sets, strutils],
+  std/[sequtils, sets, strutils],
   # Utilities
   chronicles,
   unittest2,
@@ -19,7 +19,7 @@ import
   ../../../beacon_chain/spec/datatypes/eip4844,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ../../helpers/debug_state
 
 const
@@ -150,7 +150,7 @@ suite baseDescription & "Execution Payload " & preset():
         executionPayload: eip4844.ExecutionPayload):
         Result[void, cstring] =
       let payloadValid =
-        readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
+        os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
           contains("execution_valid: true")
       func executePayload(_: eip4844.ExecutionPayload): bool = payloadValid
       process_execution_payload(

tests/consensus_spec/eip4844/test_fixture_rewards.nim

@@ -8,14 +8,12 @@
 {.used.}
 
 import
-  # Standard library
-  std/os,
   # Beacon chain internals
   ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
   ../../beacon_chain/spec/datatypes/[altair, eip4844],
   # Test utilities
   ../../testutil,
-  ../fixtures_utils
+  ../fixtures_utils, ../os_ops
 
 const
   RewardsDirBase = SszTestsDir/const_preset/"eip4844"/"rewards"

tests/consensus_spec/eip4844/test_fixture_ssz_consensus_objects.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, strutils, streams, strformat,
+  strutils, streams, strformat,
   macros, sets,
   # Third-party
   yaml,
@@ -18,7 +18,7 @@ import
   # Status libraries
   snappy,
   # Test utilities
-  ../../testutil, ../fixtures_utils
+  ../../testutil, ../fixtures_utils, ../os_ops
 
 from ../../beacon_chain/spec/datatypes/bellatrix import PowBlock
 from ../../beacon_chain/spec/datatypes/capella import

tests/consensus_spec/eip4844/test_fixture_state_transition_epoch.nim

@@ -15,12 +15,10 @@ import
   ../../../beacon_chain/spec/datatypes/[altair, eip4844],
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ./test_fixture_rewards,
   ../../helpers/debug_state
 
-from std/os import
-  DirSep, dirExists, fileExists, pcDir, walkDir, walkDirRec, `/`
 from std/sequtils import mapIt, toSeq
 from std/strutils import rsplit
 

tests/consensus_spec/fixtures_utils.nim

@@ -7,8 +7,9 @@
 
 import
   # Standard library
-  std/[os, strutils, typetraits],
+  std/[strutils, typetraits],
   # Internals
+  ./os_ops,
   ../../beacon_chain/spec/datatypes/[phase0, altair, bellatrix],
   ../../beacon_chain/spec/[
     eth2_merkleization, eth2_ssz_serialization, forks],
@@ -90,16 +91,13 @@ const
 proc parseTest*(path: string, Format: typedesc[Json], T: typedesc): T =
   try:
     # debugEcho "          [Debug] Loading file: \"", path, '\"'
-    result = Format.loadFile(path, T)
+    result = Format.decode(readFileBytes(path), T)
   except SerializationError as err:
     writeStackTrace()
     stderr.write $Format & " load issue for file \"", path, "\"\n"
     stderr.write err.formatMsg(path), "\n"
     quit 1
 
-template readFileBytes*(path: string): seq[byte] =
-  cast[seq[byte]](readFile(path))
-
 proc sszDecodeEntireInput*(input: openArray[byte], Decoded: type): Decoded =
   let stream = unsafeMemoryInput(input)
   var reader = init(SszReader, stream)

tests/consensus_spec/os_ops.nim

@@ -0,0 +1,31 @@
+import std/os
+import stew/io2
+
+export walkDir, PathComponent, walkDirRec, walkPattern, `/`, relativePath,
+       os.DirSep, os.splitPath
+export io2.readAllBytes
+
+proc fileExists*(path: string): bool = io2.isFile(path)
+
+proc dirExists*(path: string): bool = io2.isDir(path)
+
+proc readFile*(filename: string): string =
+  let res = io2.readAllChars(filename)
+  if res.isErr():
+    writeStackTrace()
+    stderr.write "Could not load data from file \"", filename, "\"\n"
+    stderr.write "(" & $int(res.error()) & ") " & ioErrorMsg(res.error()), "\n"
+    quit 1
+  res.get()
+
+proc readFileChars*(path: string): string =
+  readFile(path)
+
+proc readFileBytes*(path: string): seq[byte] =
+  let res = io2.readAllBytes(path)
+  if res.isErr():
+    writeStackTrace()
+    stderr.write "Could not load data from file \"", path, "\"\n"
+    stderr.write "(" & $int(res.error()) & ") " & ioErrorMsg(res.error()), "\n"
+    quit 1
+  res.get()

tests/consensus_spec/phase0/test_fixture_operations.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  std/[os, sequtils, sets],
+  std/[sequtils, sets],
   # Utilities
   chronicles,
   unittest2,
@@ -19,7 +19,7 @@ import
   ../../../beacon_chain/spec/datatypes/phase0,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ../../helpers/debug_state
 
 const

tests/consensus_spec/phase0/test_fixture_rewards.nim

@@ -9,13 +9,12 @@
 
 import
   # Standard library
-  std/os,
   # Beacon chain internals
   ../../beacon_chain/spec/[validator, helpers, state_transition_epoch],
   ../../beacon_chain/spec/datatypes/phase0,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils
+  ../fixtures_utils, ../os_ops
 
 const
   RewardsDirBase = SszTestsDir/const_preset/"phase0"/"rewards"

tests/consensus_spec/phase0/test_fixture_ssz_consensus_objects.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, strutils, streams, strformat,
+  strutils, streams, strformat,
   macros, sets,
   # Third-party
   yaml,
@@ -18,7 +18,7 @@ import
   # Status libraries
   snappy,
   # Test utilities
-  ../../testutil, ../fixtures_utils
+  ../../testutil, ../fixtures_utils, ../os_ops
 
 # SSZ tests of consensus objects (minimal/mainnet preset specific)
 

tests/consensus_spec/phase0/test_fixture_state_transition_epoch.nim

@@ -14,11 +14,10 @@ import
   ../../../beacon_chain/spec/datatypes/phase0,
   # Test utilities
   ../../testutil,
-  ../fixtures_utils,
+  ../fixtures_utils, ../os_ops,
   ./test_fixture_rewards,
   ../../helpers/debug_state
 
-from std/os import DirSep, fileExists, pcDir, walkDir, walkDirRec, `/`
 from std/sequtils import mapIt, toSeq
 from std/strutils import rsplit
 

tests/consensus_spec/test_fixture_fork.nim

@@ -13,11 +13,9 @@ import
   ../../beacon_chain/spec/datatypes/phase0,
   # Test utilities
   ../testutil,
-  ./fixtures_utils,
+  ./fixtures_utils, ./os_ops,
   ../helpers/debug_state
 
-from std/os import walkDir, `/`
-
 proc runTest(
     BeaconStateAnte, BeaconStatePost: type, forkNameName, forkDir: static[string],
     upgrade_func: auto, unitTestName: string) =

tests/consensus_spec/test_fixture_fork_choice.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  std/[json, os, sequtils, strutils, tables],
+  std/[json, sequtils, strutils, tables],
   # Status libraries
   stew/results, chronicles,
   eth/keys, taskpools,
@@ -26,7 +26,7 @@ import
   yaml,
   # Test
   ../testutil, ../testdbutil,
-  ./fixtures_utils
+  ./fixtures_utils, ./os_ops
 
 # Test format described at https://github.com/ethereum/consensus-specs/tree/v1.2.0-rc.1/tests/formats/fork_choice
 # Note that our implementation has been optimized with "ProtoArray"
@@ -127,7 +127,7 @@ proc initialLoad(
   (dag, fkChoice)
 
 proc loadOps(path: string, fork: ConsensusFork): seq[Operation] =
-  let stepsYAML = readFile(path/"steps.yaml")
+  let stepsYAML = os_ops.readFile(path/"steps.yaml")
   let steps = yaml.loadToJson(stepsYAML)
 
   result = @[]
@@ -411,7 +411,7 @@ proc runTest(testType: static[string], path: string, fork: ConsensusFork) =
       # Some test files have very long paths
       skip()
     else:
-      if os.splitPath(path).tail in SKIP:
+      if os_ops.splitPath(path).tail in SKIP:
         skip()
       else:
         doRunTest(path, fork)
@@ -421,7 +421,7 @@ template fcSuite(suiteName: static[string], testPathElem: static[string]) =
     const presetPath = SszTestsDir/const_preset
     for kind, path in walkDir(presetPath, relative = true, checkDir = true):
       let testsPath = presetPath/path/testPathElem
-      if kind != pcDir or not dirExists(testsPath):
+      if kind != pcDir or not os_ops.dirExists(testsPath):
         continue
       let fork = forkForPathComponent(path).valueOr:
         raiseAssert "Unknown test fork: " & testsPath

tests/consensus_spec/test_fixture_light_client_single_merkle_proof.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  std/[os, sequtils, streams],
+  std/[sequtils, streams],
   # Status libraries
   stew/bitops2,
   # Third-party
@@ -18,7 +18,7 @@ import
   ../../../beacon_chain/spec/helpers,
   # Test utilities
   ../testutil,
-  ./fixtures_utils
+  ./fixtures_utils, ./os_ops
 
 proc runTest[T](path: string, objType: typedesc[T]) =
   test "Light client - Single merkle proof - " & path.relativePath(SszTestsDir):

tests/consensus_spec/test_fixture_light_client_sync.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  std/[json, os, streams],
+  std/[json, streams],
   # Status libraries
   stew/byteutils,
   # Third-party
@@ -18,7 +18,7 @@ import
   ../../../beacon_chain/spec/[forks, light_client_sync],
   # Test utilities
   ../testutil,
-  ./fixtures_utils
+  ./fixtures_utils, ./os_ops
 
 type
   TestMeta = object
@@ -53,7 +53,7 @@ type
     checks: TestChecks
 
 proc loadSteps(path: string, fork_digests: ForkDigests): seq[TestStep] =
-  let stepsYAML = readFile(path/"steps.yaml")
+  let stepsYAML = os_ops.readFile(path/"steps.yaml")
   let steps = yaml.loadToJson(stepsYAML)
 
   result = @[]

tests/consensus_spec/test_fixture_light_client_update_ranking.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  std/[algorithm, os, streams],
+  std/[algorithm, streams],
   # Status libraries
   stew/base10,
   # Third-party
@@ -18,7 +18,7 @@ import
   ../../../beacon_chain/spec/helpers,
   # Test utilities
   ../testutil,
-  ./fixtures_utils
+  ./fixtures_utils, ./os_ops
 
 type
   TestMeta = object

tests/consensus_spec/test_fixture_sanity_blocks.nim

@@ -11,9 +11,9 @@ import
   chronicles,
   ../../beacon_chain/spec/datatypes/phase0,
   ../../beacon_chain/spec/state_transition,
+  ./os_ops,
   ../testutil
 
-from std/os import fileExists, walkDir, walkPattern, `/`
 from std/sequtils import toSeq
 from ../../../beacon_chain/spec/forks import
   ForkedEpochInfo, ForkedHashedBeaconState, fromSszBytes, getStateRoot, new

tests/consensus_spec/test_fixture_sanity_slots.nim

@@ -8,7 +8,7 @@
 {.used.}
 
 import ../../beacon_chain/spec/forks
-from std/os import walkDir, `/`
+import os_ops
 from std/strutils import parseInt
 from ./fixtures_utils import SszTestsDir, parseTest
 from ../testutil import check, preset, suite, test

tests/consensus_spec/test_fixture_ssz_generic_types.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, strutils, streams, strformat, strscans,
+  strutils, streams, strformat, strscans,
   macros, typetraits,
   # Status libraries
   faststreams, snappy, stint, ../testutil,
@@ -19,7 +19,7 @@ import
   ../../beacon_chain/spec/digest,
   ../../beacon_chain/spec/datatypes/base,
   # Test utilities
-  ./fixtures_utils
+  ./fixtures_utils, ./os_ops
 
 # Parsing definitions
 # ------------------------------------------------------------------------

tests/consensus_spec/test_fixture_transition.nim

@@ -9,9 +9,9 @@
 
 import
   yaml,
-  ../../beacon_chain/spec/[state_transition, forks]
+  ../../beacon_chain/spec/[state_transition, forks],
+  ./os_ops
 
-from std/os import walkDir, walkPattern, `/`
 from std/sequtils import toSeq
 from streams import close, openFileStream
 from ../testutil import preset, suite, test

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 447b23d3bf6eb7be6531385e4db9124772c98068
+Subproject commit 406a5c986e32fbc28e230fcfee74b095ce1e4533