diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce060c2..71e8cfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,14 +1,16 @@ -name: nim-faststreams CI -on: [push, pull_request] +name: CI +on: + push: + branches: + - master + pull_request: + workflow_dispatch: jobs: build: strategy: fail-fast: false - max-parallel: 20 matrix: - branch: [master] - test_lang: [c, cpp] target: - os: linux cpu: amd64 @@ -18,27 +20,33 @@ jobs: cpu: amd64 - os: windows cpu: amd64 - - os: windows - cpu: i386 + #- os: windows + #cpu: i386 + branch: [version-1-2, version-1-4, version-1-6, devel] include: - target: os: linux builder: ubuntu-18.04 + shell: bash - target: os: macos builder: macos-10.15 + shell: bash - target: os: windows builder: windows-2019 + shell: msys2 {0} - name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.test_lang }} (${{ matrix.branch }})' + defaults: + run: + shell: ${{ matrix.shell }} + + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' runs-on: ${{ matrix.builder }} + continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }} steps: - - name: Checkout nim-faststreams + - name: Checkout uses: actions/checkout@v2 - with: - path: nim-faststreams - submodules: false - name: Install build dependencies (Linux i386) if: runner.os == 'Linux' && matrix.target.cpu == 'i386' @@ -60,98 +68,93 @@ jobs: chmod 755 external/bin/gcc external/bin/g++ echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - name: Restore MinGW-W64 (Windows) from cache - if: runner.os == 'Windows' - id: windows-mingw-cache - uses: actions/cache@v2 + - name: MSYS2 (Windows i386) + if: runner.os == 'Windows' && matrix.target.cpu == 'i386' + uses: msys2/setup-msys2@v2 with: - path: external/mingw-${{ matrix.target.cpu }} - key: 'mingw-${{ matrix.target.cpu }}' + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' id: windows-dlls-cache uses: actions/cache@v2 with: - path: external/dlls-${{ matrix.target.cpu }} - key: 'dlls-${{ matrix.target.cpu }}' + path: external/dlls + key: 'dlls' - - name: Install MinGW64 dependency (Windows) - if: > - steps.windows-mingw-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' - shell: bash - run: | - mkdir -p external - if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains targetting Win64/Personal Builds/mingw-builds/8.1.0/threads-posix/seh/x86_64-8.1.0-release-posix-seh-rt_v6-rev0.7z" - ARCH=64 - else - MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains targetting Win32/Personal Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z" - ARCH=32 - fi - curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.7z" - 7z x -y "external/mingw-${{ matrix.target.cpu }}.7z" -oexternal/ - mv external/mingw$ARCH external/mingw-${{ matrix.target.cpu }} - - - name: Install DLLs dependencies (Windows) + - name: Install DLL dependencies (Windows) if: > steps.windows-dlls-cache.outputs.cache-hit != 'true' && runner.os == 'Windows' - shell: bash run: | - mkdir -p external + mkdir external curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip - 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + 7z x external/windeps.zip -oexternal/dlls - name: Path to cached dependencies (Windows) if: > runner.os == 'Windows' - shell: bash run: | - echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH - echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - - name: Get latest nimbus-build-system commit hash - id: versions - shell: bash + - name: Derive environment variables run: | - getHash() { - git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 - } - nbsHash=$(getHash status-im/nimbus-build-system) - echo "::set-output name=nimbus_build_system::$nbsHash" - - - name: Restore prebuilt Nim from cache - id: nim-cache - uses: actions/cache@v2 - with: - path: NimBinaries - key: 'NimBinaries-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nimbus_build_system }}' - - - name: Build Nim and associated tools - shell: bash - run: | - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then PLATFORM=x64 else PLATFORM=x86 fi - if [[ '${{ matrix.target.os }}' == 'windows' ]]; then + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + + ncpu= + MAKE_CMD="make" + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS MAKE_CMD="mingw32-make" - else - MAKE_CMD="make" - fi - env MAKE="$MAKE_CMD -j2" ARCH_OVERRIDE=$PLATFORM CC=gcc bash build_nim.sh nim csources dist/nimble NimBinaries + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=$ncpu" >> $GITHUB_ENV + echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV - - name: Setup environment - shell: bash - run: echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH - - - name: Run nim-faststreams tests - shell: bash - working-directory: nim-faststreams + - name: Build Nim and Nimble run: | + curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \ + QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \ + bash build_nim.sh nim csources dist/nimble NimBinaries + echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH + + - name: Run tests + run: | + if [[ "${{ matrix.target.os }}" == "windows" ]]; then + # https://github.com/status-im/nimbus-eth2/issues/3121 + export NIMFLAGS="-d:nimRawSetjmp" + fi + nim --version + nimble --version nimble install -y --depsOnly - env TEST_LANG="${{ matrix.test_lang }}" nimble test + env TEST_LANG="c" nimble test + env TEST_LANG="cpp" nimble test diff --git a/faststreams.nimble b/faststreams.nimble index c238701..2554214 100644 --- a/faststreams.nimble +++ b/faststreams.nimble @@ -14,20 +14,18 @@ requires "nim >= 1.2.0", "unittest2" ### Helper functions -proc test(env, path: string) = +proc test(args, path: string) = # Compilation language is controlled by TEST_LANG - var lang = "c" - if existsEnv"TEST_LANG": - lang = getEnv"TEST_LANG" + let lang = getEnv("TEST_LANG", "c") - let common_args = "-r -f --hints:off --skipParentCfg --styleCheck:usages --styleCheck:error" + let common_args = "-r -f " & getEnv("NIMFLAGS") & " --hints:off --skipParentCfg --styleCheck:usages --styleCheck:error" - exec "nim " & lang & " " & env & + exec "nim " & lang & " " & args & " -d:asyncBackend=none " & common_args & " " & path - exec "nim " & lang & " " & env & + exec "nim " & lang & " " & args & " -d:asyncBackend=chronos " & common_args & " " & path # TODO std backend is broken / untested - # exec "nim " & lang & " " & env & + # exec "nim " & lang & " " & args & # " -d:asyncBackend=asyncdispatch " & common_args & " " & path task test, "Run all tests": diff --git a/faststreams/inputs.nim b/faststreams/inputs.nim index ed9fb7e..a36edb0 100644 --- a/faststreams/inputs.nim +++ b/faststreams/inputs.nim @@ -796,6 +796,9 @@ template readIntoExImpl(s: InputStream, var adjustedDst = offset(dst, totalBytesDrained) while true: + if s.vtable.isNil(): + break + let newBytesRead = awaiter s.vtable.readOp(s, adjustedDst, bytesDeficit) s.spanEndPos += newBytesRead @@ -808,7 +811,7 @@ template readIntoExImpl(s: InputStream, if bytesDeficit == 0: break - adjustedDst = offset(dst, newBytesRead) + adjustedDst = offset(adjustedDst, newBytesRead) dstLen - bytesDeficit diff --git a/tests/test_inputs.nim b/tests/test_inputs.nim index 74c8cdf..a2ae220 100644 --- a/tests/test_inputs.nim +++ b/tests/test_inputs.nim @@ -58,26 +58,27 @@ suite "input stream": var input = memFileInput("files" / "empty_file") template asciiTableFileTest(name: string, body: untyped) = - test name & " of ascii table with regular pageSize": - var input {.inject.} = fileInput(asciiTableFile) - try: - body - finally: - close input + suite name: + test name & " of ascii table with regular pageSize": + var input {.inject.} = fileInput(asciiTableFile) + try: + body + finally: + close input - test name & " of ascii table with pageSize = 10": - var input {.inject.} = fileInput(asciiTableFile, pageSize = 10) - try: - body - finally: - close input + test name & " of ascii table with pageSize = 10": + var input {.inject.} = fileInput(asciiTableFile, pageSize = 10) + try: + body + finally: + close input - test name & " of ascii table with pageSize = 1": - var input {.inject.} = fileInput(asciiTableFile, pageSize = 1) - try: - body - finally: - close input + test name & " of ascii table with pageSize = 1": + var input {.inject.} = fileInput(asciiTableFile, pageSize = 1) + try: + body + finally: + close input # TODO: fileInput with offset # - in the middle of the @@ -124,41 +125,42 @@ suite "input stream": check fileContents == asciiTableContents - test "missing file input": - const fileName = "there-is-no-such-faststreams-file" + suite "misc": + test "missing file input": + const fileName = "there-is-no-such-faststreams-file" - check not fileExists(fileName) - expect CatchableError: discard fileInput(fileName) + check not fileExists(fileName) + expect CatchableError: discard fileInput(fileName) - check not fileExists(fileName) - expect CatchableError: discard memFileInput(fileName) + check not fileExists(fileName) + expect CatchableError: discard memFileInput(fileName) - check not fileExists(fileName) + check not fileExists(fileName) - test "non-blocking reads": - let s = fileInput(asciiTableFile, pageSize = 100) - if s.readable(20): - s.withReadableRange(20, r): - check r.len.get == 20 - check r.totalUnconsumedBytes == 20 - check r.readAll.len == 20 + test "non-blocking reads": + let s = fileInput(asciiTableFile, pageSize = 100) + if s.readable(20): + s.withReadableRange(20, r): + check r.len.get == 20 + check r.totalUnconsumedBytes == 20 + check r.readAll.len == 20 - check s.readable + check s.readable - if s.readable(200): - s.withReadableRange(200, r): - check r.len.get == 200 - check r.totalUnconsumedBytes == 200 - check r.readAll.len == 200 + if s.readable(200): + s.withReadableRange(200, r): + check r.len.get == 200 + check r.totalUnconsumedBytes == 200 + check r.readAll.len == 200 - check s.readable + check s.readable - test "simple": - var input = repeat("1234 5678 90AB CDEF\n", 1000) - var stream = unsafeMemoryInput(input) + test "simple": + var input = repeat("1234 5678 90AB CDEF\n", 1000) + var stream = unsafeMemoryInput(input) - check: - (stream.read(4) == "1234".toOpenArrayByte(0, 3)) + check: + (stream.read(4) == "1234".toOpenArrayByte(0, 3)) template posTest(name: string, setup: untyped) = test name: diff --git a/tests/test_outputs.nim b/tests/test_outputs.nim index 4caebc3..86a3fbc 100644 --- a/tests/test_outputs.nim +++ b/tests/test_outputs.nim @@ -1,7 +1,7 @@ {.used.} import - os, unittest, random, strformat, + os, unittest2, random, strformat, stew/ranges/ptr_arith, ../faststreams, ../faststreams/textio @@ -20,12 +20,10 @@ proc repeat(b: byte, count: int): seq[byte] = result = newSeq[byte](count) for i in 0 ..< count: result[i] = b -const line = "123456789123456789123456789123456789\n\n\n\n\n" - proc randomBytes(n: int): seq[byte] = result.newSeq n for i in 0 ..< n: - result[i] = byte(rand(line)) + result[i] = byte(rand(int('1')..int('9'))) proc readAllAndClose(s: InputStream): seq[byte] = while s.readable: @@ -55,6 +53,8 @@ suite "output stream": streamWritingToExistingBuffer = unsafeMemoryOutput(buffer, bufferSize) teardown: + fileStream.close() + unbufferedFileStream.close() removeFile fileOutputPath removeFile unbufferedFileOutputPath dealloc buffer