Fix Windows MAX_PATH constraint issue in CI. (#4576)

* Fix MAX_PATH limitation in tests.

* Fix posix issues.

* Fix compilation issue.
This commit is contained in:
Eugene Kabanov 2023-02-10 22:59:38 +02:00 committed by GitHub
parent 09dd64df32
commit e51095e2f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 87 additions and 81 deletions

View File

@ -6,9 +6,8 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
stew/io2,
stats, os, strformat, times,
../tests/testblockutil,
stats, strformat, times,
../tests/testblockutil, ../tests/consensus_spec/os_ops,
../beacon_chain/beacon_chain_db,
../beacon_chain/spec/datatypes/[phase0, altair],
../beacon_chain/spec/[beaconstate, deposit_snapshots, forks, helpers],

View File

@ -9,7 +9,7 @@
import
# Standard library
std/[os, sequtils, sets],
std/[sequtils, sets],
# Utilities
chronicles,
unittest2,
@ -19,7 +19,7 @@ import
../../../beacon_chain/spec/datatypes/altair,
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
../../helpers/debug_state
const

View File

@ -8,14 +8,12 @@
{.used.}
import
# Standard library
std/os,
# Beacon chain internals
../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
../../beacon_chain/spec/datatypes/altair,
# Test utilities
../../testutil,
../fixtures_utils
../fixtures_utils, ../os_ops
const
RewardsDirBase = SszTestsDir/const_preset/"altair"/"rewards"

View File

@ -9,7 +9,7 @@
import
# Standard library
os, strutils, streams, strformat,
strutils, streams, strformat,
macros,
# Third-party
yaml,
@ -18,7 +18,7 @@ import
# Status libraries
snappy,
# Test utilities
../../testutil, ../fixtures_utils
../../testutil, ../fixtures_utils, ../os_ops
# SSZ tests of consensus objects (minimal/mainnet preset specific)

View File

@ -14,12 +14,10 @@ import
../../../beacon_chain/spec/datatypes/altair,
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
./test_fixture_rewards,
../../helpers/debug_state
from std/os import
DirSep, dirExists, fileExists, pcDir, walkDir, walkDirRec, `/`
from std/sequtils import mapIt, toSeq
from std/strutils import rsplit

View File

@ -17,10 +17,9 @@ import
../../../beacon_chain/spec/datatypes/bellatrix,
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
../../helpers/debug_state
from std/os import fileExists, walkDir, `/`
from std/sequtils import mapIt, toSeq
from std/strutils import contains
@ -132,7 +131,7 @@ suite baseDescription & "Execution Payload " & preset():
executionPayload: bellatrix.ExecutionPayload):
Result[void, cstring] =
let payloadValid =
readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
contains("execution_valid: true")
func executePayload(_: bellatrix.ExecutionPayload): bool = payloadValid
process_execution_payload(

View File

@ -8,14 +8,12 @@
{.used.}
import
# Standard library
std/os,
# Beacon chain internals
../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
../../beacon_chain/spec/datatypes/[altair, bellatrix],
# Test utilities
../../testutil,
../fixtures_utils
../fixtures_utils, ../os_ops
const
RewardsDirBase = SszTestsDir/const_preset/"bellatrix"/"rewards"

View File

@ -15,9 +15,8 @@ import
# Status libraries
snappy,
# Test utilities
../../testutil, ../fixtures_utils
../../testutil, ../fixtures_utils, ../os_ops
from std/os import dirExists, pcDir, walkDir, `/`
from std/streams import close, openFileStream
from std/strformat import `&`
from std/strutils import toLowerAscii

View File

@ -13,12 +13,10 @@ import
../../../beacon_chain/spec/datatypes/[altair, bellatrix],
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
./test_fixture_rewards,
../../helpers/debug_state
from std/os import
DirSep, dirExists, fileExists, pcDir, walkDir, walkDirRec, `/`
from std/strutils import rsplit
from std/sequtils import mapIt, toSeq

View File

@ -9,7 +9,7 @@
import
# Standard library
std/[os, sequtils, sets, strutils],
std/[sequtils, sets, strutils],
# Utilities
chronicles,
unittest2,
@ -19,7 +19,7 @@ import
../../../beacon_chain/spec/datatypes/capella,
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
../../helpers/debug_state
const
@ -147,7 +147,7 @@ suite baseDescription & "Execution Payload " & preset():
executionPayload: capella.ExecutionPayload):
Result[void, cstring] =
let payloadValid =
readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
contains("execution_valid: true")
func executePayload(_: capella.ExecutionPayload): bool = payloadValid
process_execution_payload(

View File

@ -8,14 +8,12 @@
{.used.}
import
# Standard library
std/os,
# Beacon chain internals
../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
../../beacon_chain/spec/datatypes/[altair, capella],
# Test utilities
../../testutil,
../fixtures_utils
../fixtures_utils, ../os_ops
const
RewardsDirBase = SszTestsDir/const_preset/"capella"/"rewards"

View File

@ -9,7 +9,7 @@
import
# Standard library
os, strutils, streams, strformat,
strutils, streams, strformat,
macros, sets,
# Third-party
yaml,
@ -18,7 +18,7 @@ import
# Status libraries
snappy,
# Test utilities
../../testutil, ../fixtures_utils
../../testutil, ../fixtures_utils, ../os_ops
from ../../beacon_chain/spec/datatypes/bellatrix import PowBlock

View File

@ -15,12 +15,10 @@ import
../../../beacon_chain/spec/datatypes/[altair, capella],
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
./test_fixture_rewards,
../../helpers/debug_state
from std/os import
DirSep, dirExists, fileExists, pcDir, walkDir, walkDirRec, `/`
from std/sequtils import mapIt, toSeq
from std/strutils import rsplit

View File

@ -9,7 +9,7 @@
import
# Standard library
std/[os, sequtils, sets, strutils],
std/[sequtils, sets, strutils],
# Utilities
chronicles,
unittest2,
@ -19,7 +19,7 @@ import
../../../beacon_chain/spec/datatypes/eip4844,
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
../../helpers/debug_state
const
@ -150,7 +150,7 @@ suite baseDescription & "Execution Payload " & preset():
executionPayload: eip4844.ExecutionPayload):
Result[void, cstring] =
let payloadValid =
readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
contains("execution_valid: true")
func executePayload(_: eip4844.ExecutionPayload): bool = payloadValid
process_execution_payload(

View File

@ -8,14 +8,12 @@
{.used.}
import
# Standard library
std/os,
# Beacon chain internals
../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
../../beacon_chain/spec/datatypes/[altair, eip4844],
# Test utilities
../../testutil,
../fixtures_utils
../fixtures_utils, ../os_ops
const
RewardsDirBase = SszTestsDir/const_preset/"eip4844"/"rewards"

View File

@ -9,7 +9,7 @@
import
# Standard library
os, strutils, streams, strformat,
strutils, streams, strformat,
macros, sets,
# Third-party
yaml,
@ -18,7 +18,7 @@ import
# Status libraries
snappy,
# Test utilities
../../testutil, ../fixtures_utils
../../testutil, ../fixtures_utils, ../os_ops
from ../../beacon_chain/spec/datatypes/bellatrix import PowBlock
from ../../beacon_chain/spec/datatypes/capella import

View File

@ -15,12 +15,10 @@ import
../../../beacon_chain/spec/datatypes/[altair, eip4844],
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
./test_fixture_rewards,
../../helpers/debug_state
from std/os import
DirSep, dirExists, fileExists, pcDir, walkDir, walkDirRec, `/`
from std/sequtils import mapIt, toSeq
from std/strutils import rsplit

View File

@ -7,8 +7,9 @@
import
# Standard library
std/[os, strutils, typetraits],
std/[strutils, typetraits],
# Internals
./os_ops,
../../beacon_chain/spec/datatypes/[phase0, altair, bellatrix],
../../beacon_chain/spec/[
eth2_merkleization, eth2_ssz_serialization, forks],
@ -90,16 +91,13 @@ const
proc parseTest*(path: string, Format: typedesc[Json], T: typedesc): T =
try:
# debugEcho " [Debug] Loading file: \"", path, '\"'
result = Format.loadFile(path, T)
result = Format.decode(readFileBytes(path), T)
except SerializationError as err:
writeStackTrace()
stderr.write $Format & " load issue for file \"", path, "\"\n"
stderr.write err.formatMsg(path), "\n"
quit 1
template readFileBytes*(path: string): seq[byte] =
cast[seq[byte]](readFile(path))
proc sszDecodeEntireInput*(input: openArray[byte], Decoded: type): Decoded =
let stream = unsafeMemoryInput(input)
var reader = init(SszReader, stream)

View File

@ -0,0 +1,31 @@
import std/os
import stew/io2
export walkDir, PathComponent, walkDirRec, walkPattern, `/`, relativePath,
os.DirSep, os.splitPath
export io2.readAllBytes
proc fileExists*(path: string): bool = io2.isFile(path)
proc dirExists*(path: string): bool = io2.isDir(path)
proc readFile*(filename: string): string =
let res = io2.readAllChars(filename)
if res.isErr():
writeStackTrace()
stderr.write "Could not load data from file \"", filename, "\"\n"
stderr.write "(" & $int(res.error()) & ") " & ioErrorMsg(res.error()), "\n"
quit 1
res.get()
proc readFileChars*(path: string): string =
readFile(path)
proc readFileBytes*(path: string): seq[byte] =
let res = io2.readAllBytes(path)
if res.isErr():
writeStackTrace()
stderr.write "Could not load data from file \"", path, "\"\n"
stderr.write "(" & $int(res.error()) & ") " & ioErrorMsg(res.error()), "\n"
quit 1
res.get()

View File

@ -9,7 +9,7 @@
import
# Standard library
std/[os, sequtils, sets],
std/[sequtils, sets],
# Utilities
chronicles,
unittest2,
@ -19,7 +19,7 @@ import
../../../beacon_chain/spec/datatypes/phase0,
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
../../helpers/debug_state
const

View File

@ -9,13 +9,12 @@
import
# Standard library
std/os,
# Beacon chain internals
../../beacon_chain/spec/[validator, helpers, state_transition_epoch],
../../beacon_chain/spec/datatypes/phase0,
# Test utilities
../../testutil,
../fixtures_utils
../fixtures_utils, ../os_ops
const
RewardsDirBase = SszTestsDir/const_preset/"phase0"/"rewards"

View File

@ -9,7 +9,7 @@
import
# Standard library
os, strutils, streams, strformat,
strutils, streams, strformat,
macros, sets,
# Third-party
yaml,
@ -18,7 +18,7 @@ import
# Status libraries
snappy,
# Test utilities
../../testutil, ../fixtures_utils
../../testutil, ../fixtures_utils, ../os_ops
# SSZ tests of consensus objects (minimal/mainnet preset specific)

View File

@ -14,11 +14,10 @@ import
../../../beacon_chain/spec/datatypes/phase0,
# Test utilities
../../testutil,
../fixtures_utils,
../fixtures_utils, ../os_ops,
./test_fixture_rewards,
../../helpers/debug_state
from std/os import DirSep, fileExists, pcDir, walkDir, walkDirRec, `/`
from std/sequtils import mapIt, toSeq
from std/strutils import rsplit

View File

@ -13,11 +13,9 @@ import
../../beacon_chain/spec/datatypes/phase0,
# Test utilities
../testutil,
./fixtures_utils,
./fixtures_utils, ./os_ops,
../helpers/debug_state
from std/os import walkDir, `/`
proc runTest(
BeaconStateAnte, BeaconStatePost: type, forkNameName, forkDir: static[string],
upgrade_func: auto, unitTestName: string) =

View File

@ -9,7 +9,7 @@
import
# Standard library
std/[json, os, sequtils, strutils, tables],
std/[json, sequtils, strutils, tables],
# Status libraries
stew/results, chronicles,
eth/keys, taskpools,
@ -26,7 +26,7 @@ import
yaml,
# Test
../testutil, ../testdbutil,
./fixtures_utils
./fixtures_utils, ./os_ops
# Test format described at https://github.com/ethereum/consensus-specs/tree/v1.2.0-rc.1/tests/formats/fork_choice
# Note that our implementation has been optimized with "ProtoArray"
@ -127,7 +127,7 @@ proc initialLoad(
(dag, fkChoice)
proc loadOps(path: string, fork: ConsensusFork): seq[Operation] =
let stepsYAML = readFile(path/"steps.yaml")
let stepsYAML = os_ops.readFile(path/"steps.yaml")
let steps = yaml.loadToJson(stepsYAML)
result = @[]
@ -411,7 +411,7 @@ proc runTest(testType: static[string], path: string, fork: ConsensusFork) =
# Some test files have very long paths
skip()
else:
if os.splitPath(path).tail in SKIP:
if os_ops.splitPath(path).tail in SKIP:
skip()
else:
doRunTest(path, fork)
@ -421,7 +421,7 @@ template fcSuite(suiteName: static[string], testPathElem: static[string]) =
const presetPath = SszTestsDir/const_preset
for kind, path in walkDir(presetPath, relative = true, checkDir = true):
let testsPath = presetPath/path/testPathElem
if kind != pcDir or not dirExists(testsPath):
if kind != pcDir or not os_ops.dirExists(testsPath):
continue
let fork = forkForPathComponent(path).valueOr:
raiseAssert "Unknown test fork: " & testsPath

View File

@ -9,7 +9,7 @@
import
# Standard library
std/[os, sequtils, streams],
std/[sequtils, streams],
# Status libraries
stew/bitops2,
# Third-party
@ -18,7 +18,7 @@ import
../../../beacon_chain/spec/helpers,
# Test utilities
../testutil,
./fixtures_utils
./fixtures_utils, ./os_ops
proc runTest[T](path: string, objType: typedesc[T]) =
test "Light client - Single merkle proof - " & path.relativePath(SszTestsDir):

View File

@ -9,7 +9,7 @@
import
# Standard library
std/[json, os, streams],
std/[json, streams],
# Status libraries
stew/byteutils,
# Third-party
@ -18,7 +18,7 @@ import
../../../beacon_chain/spec/[forks, light_client_sync],
# Test utilities
../testutil,
./fixtures_utils
./fixtures_utils, ./os_ops
type
TestMeta = object
@ -53,7 +53,7 @@ type
checks: TestChecks
proc loadSteps(path: string, fork_digests: ForkDigests): seq[TestStep] =
let stepsYAML = readFile(path/"steps.yaml")
let stepsYAML = os_ops.readFile(path/"steps.yaml")
let steps = yaml.loadToJson(stepsYAML)
result = @[]

View File

@ -9,7 +9,7 @@
import
# Standard library
std/[algorithm, os, streams],
std/[algorithm, streams],
# Status libraries
stew/base10,
# Third-party
@ -18,7 +18,7 @@ import
../../../beacon_chain/spec/helpers,
# Test utilities
../testutil,
./fixtures_utils
./fixtures_utils, ./os_ops
type
TestMeta = object

View File

@ -11,9 +11,9 @@ import
chronicles,
../../beacon_chain/spec/datatypes/phase0,
../../beacon_chain/spec/state_transition,
./os_ops,
../testutil
from std/os import fileExists, walkDir, walkPattern, `/`
from std/sequtils import toSeq
from ../../../beacon_chain/spec/forks import
ForkedEpochInfo, ForkedHashedBeaconState, fromSszBytes, getStateRoot, new

View File

@ -8,7 +8,7 @@
{.used.}
import ../../beacon_chain/spec/forks
from std/os import walkDir, `/`
import os_ops
from std/strutils import parseInt
from ./fixtures_utils import SszTestsDir, parseTest
from ../testutil import check, preset, suite, test

View File

@ -9,7 +9,7 @@
import
# Standard library
os, strutils, streams, strformat, strscans,
strutils, streams, strformat, strscans,
macros, typetraits,
# Status libraries
faststreams, snappy, stint, ../testutil,
@ -19,7 +19,7 @@ import
../../beacon_chain/spec/digest,
../../beacon_chain/spec/datatypes/base,
# Test utilities
./fixtures_utils
./fixtures_utils, ./os_ops
# Parsing definitions
# ------------------------------------------------------------------------

View File

@ -9,9 +9,9 @@
import
yaml,
../../beacon_chain/spec/[state_transition, forks]
../../beacon_chain/spec/[state_transition, forks],
./os_ops
from std/os import walkDir, walkPattern, `/`
from std/sequtils import toSeq
from streams import close, openFileStream
from ../testutil import preset, suite, test

2
vendor/nim-stew vendored

@ -1 +1 @@
Subproject commit 447b23d3bf6eb7be6531385e4db9124772c98068
Subproject commit 406a5c986e32fbc28e230fcfee74b095ce1e4533