From a2a249535112ec3f1497e14ac27c0e0c00bd60c1 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Fri, 19 Jun 2020 22:08:15 +0200 Subject: [PATCH] Github Action CI (without GMP) (#29) * Github Action CI (without GMP) * Deactivate MacOS, spurious failures: https://github.com/actions/virtual-environments/issues/841 * force install with nimble * Add badge * Don"t include Nim 1.2.x https://github.com/mratsim/constantine/pull/20#issuecomment-646327952 * Action branch mistake * Add back OSX? https://github.com/actions/virtual-environments/issues/841, https://github.com/actions/virtual-environments/issues/969 * fix MacOS target * comment out RDTSC on i386 * Add initialization canaries * Add more verbose output to debug windows failures * spurious windows i386 test * For now only activate Linux and mac * missed include --- .github/workflows/ci.yml | 199 +++++++++++++++++++++++ README.md | 3 +- benchmarks/platforms.nim | 2 +- constantine.nimble | 37 ++++- constantine/arithmetic/limbs_modular.nim | 2 +- tests/support/canaries.nim | 31 ++++ tests/t_bigints.nim | 97 ++++++----- 7 files changed, 323 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tests/support/canaries.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0eaec97 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,199 @@ +name: Constantine CI +on: [push, pull_request] + +jobs: + build: + strategy: + fail-fast: false + max-parallel: 20 + matrix: + branch: [devel] # [version-1-2, devel] - # 1.2 doesn't compile https://github.com/mratsim/constantine/pull/20#issuecomment-646327952 + target: + - os: linux + cpu: amd64 + TEST_LANG: c + - os: linux + cpu: amd64 + TEST_LANG: cpp + - os: linux + cpu: i386 + TEST_LANG: c + - os: linux + cpu: i386 + TEST_LANG: cpp + - os: macos + cpu: amd64 + TEST_LANG: c + - os: macos + cpu: amd64 + TEST_LANG: cpp + # TODO: + # 1. Modulo/reduce bug on 32-bit + # 2. ModInverse bug on all windows + # - os: windows + # cpu: amd64 + # TEST_LANG: c + # - os: windows + # cpu: amd64 + # TEST_LANG: cpp + # - os: windows + # cpu: i386 + # TEST_LANG: c + # - os: windows + # cpu: i386 + # TEST_LANG: cpp + include: + - target: + os: linux + builder: ubuntu-18.04 + - target: + os: macos + builder: macos-10.15 + # - target: + # os: windows + # builder: windows-2019 + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.target.TEST_LANG }} (${{ matrix.branch }})' + runs-on: ${{ matrix.builder }} + steps: + - name: Checkout constantine + uses: actions/checkout@v2 + with: + path: constantine + + # - name: Install dependencies (Linux amd64) + # if: runner.os == 'Linux' && matrix.target.cpu == 'amd64' + # run: | + # sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ + # --no-install-recommends -yq + + - name: Install dependencies (Linux i386) + if: runner.os == 'Linux' && matrix.target.cpu == 'i386' + run: | + sudo dpkg --add-architecture i386 + sudo apt-fast update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ + --no-install-recommends -yq gcc-multilib g++-multilib \ + libssl-dev:i386 + mkdir -p external/bin + cat << EOF > external/bin/gcc + #!/bin/bash + exec $(which gcc) -m32 "\$@" + EOF + cat << EOF > external/bin/g++ + #!/bin/bash + exec $(which g++) -m32 "\$@" + EOF + chmod 755 external/bin/gcc external/bin/g++ + echo '::add-path::${{ github.workspace }}/external/bin' + # - name: Install dependencies (macOS) + # if: runner.os == 'macOS' + # run: brew install + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + shell: bash + run: | + mkdir external + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + arch=64 + else + arch=32 + fi + curl -L "https://nim-lang.org/download/mingw$arch-6.3.0.7z" -o "external/mingw$arch.7z" + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x "external/mingw$arch.7z" -oexternal/ + 7z x external/windeps.zip -oexternal/dlls + echo '::add-path::${{ github.workspace }}'"/external/mingw$arch/bin" + echo '::add-path::${{ github.workspace }}'"/external/dlls" + - name: Setup environment + shell: bash + run: echo '::add-path::${{ github.workspace }}/nim/bin' + + - name: Get latest Nim commit hash + id: versions + shell: bash + run: | + getHash() { + git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 + } + nimHash=$(getHash nim-lang/Nim '${{ matrix.branch }}') + csourcesHash=$(getHash nim-lang/csources) + echo "::set-output name=nim::$nimHash" + echo "::set-output name=csources::$csourcesHash" + - name: Restore prebuilt Nim from cache + id: nim-cache + uses: actions/cache@v1 + with: + path: nim + key: 'nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nim }}' + + - name: Restore prebuilt csources from cache + if: steps.nim-cache.outputs.cache-hit != 'true' + id: csources-cache + uses: actions/cache@v1 + with: + path: csources/bin + key: 'csources-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.csources }}' + + - name: Checkout Nim csources + if: > + steps.csources-cache.outputs.cache-hit != 'true' && + steps.nim-cache.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: nim-lang/csources + path: csources + ref: ${{ steps.versions.outputs.csources }} + + - name: Checkout Nim + if: steps.nim-cache.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: nim-lang/Nim + path: nim + ref: ${{ steps.versions.outputs.nim }} + + - name: Build Nim and associated tools + if: steps.nim-cache.outputs.cache-hit != 'true' + shell: bash + run: | + ncpu= + ext= + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS + ext=.exe + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + if [[ ! -e csources/bin/nim$ext ]]; then + make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.target.cpu }}' + else + echo 'Using prebuilt csources' + fi + cp -v csources/bin/nim$ext nim/bin + cd nim + nim c koch + ./koch boot -d:release + ./koch tools -d:release + # clean up to save cache space + rm koch + rm -rf nimcache + rm -rf dist + rm -rf .git + - name: Install test dependencies + shell: bash + run: | + nimble refresh + nimble install -y gmp stew + - name: Run constantine tests (without GMP) + shell: bash + run: | + cd constantine + nimble test_no_gmp diff --git a/README.md b/README.md index 0cf98f5..a6de283 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) ![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)\ -[![Build Status: Travis](https://img.shields.io/travis/com/mratsim/constantine/master?label=Travis%20%28Linux%20x86_64%2FARM64,%20MacOS%20x86_64%29)](https://travis-ci.com/mratsim/constantine) +![Github Actions CI](https://github.com/mratsim/constantine/workflows/Constantine%20CI/badge.svg)\ +[![Build Status: Travis](https://img.shields.io/travis/com/mratsim/constantine/master?label=Travis%20%28Linux%20x86_64%2FARM64,%20MacOS%20x86_64%29)](https://travis-ci.com/mratsim/constantine)\ [![Build Status: Azure](https://img.shields.io/azure-devops/build/numforge/07a2a7a5-995a-45d3-acd5-f5456fe7b04d/4?label=Azure%20%28Linux%2064-bit%2C%20Windows%2032%2F64-bit%2C%20MacOS%2064-bit%29)](https://dev.azure.com/numforge/Constantine/_build?definitionId=4&branchName=master) This library provides constant-time implementation of elliptic curve cryptography. diff --git a/benchmarks/platforms.nim b/benchmarks/platforms.nim index a9dd92a..0669fb9 100644 --- a/benchmarks/platforms.nim +++ b/benchmarks/platforms.nim @@ -6,7 +6,7 @@ # * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -when defined(i386) or defined(amd64): +when defined(amd64): # TODO defined(i386) but it seems lie RDTSC call is misconfigured import platforms/x86 export getTicks, cpuName diff --git a/constantine.nimble b/constantine.nimble index 9480e41..79da9b0 100644 --- a/constantine.nimble +++ b/constantine.nimble @@ -63,6 +63,12 @@ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[ ("tests/t_ec_sage_bls12_381.nim", false) ] +# For temporary (hopefully) investigation that can only be reproduced in CI +const useDebug = [ + "tests/t_bigints.nim" +] + + # Helper functions # ---------------------------------------------------------------- @@ -109,11 +115,17 @@ task test, "Run all tests": # -d:testingCurves is configured in a *.nim.cfg for convenience for td in testDesc: - test "", td.path + if td.path in useDebug: + test "-d:debugConstantine", td.path + else: + test "", td.path if sizeof(int) == 8: # 32-bit tests on 64-bit arch for td in testDesc: - test "-d:Constantine32", td.path + if td.path in useDebug: + test "-d:Constantine32 -d:debugConstantine", td.path + else: + test "-d:Constantine32", td.path # Benchmarks compile and run # ignore Windows 32-bit for the moment @@ -130,12 +142,19 @@ task test_no_gmp, "Run tests that don't require GMP": # -d:testingCurves is configured in a *.nim.cfg for convenience for td in testDesc: if not td.useGMP: - test "", td.path + if td.path in useDebug: + test "-d:debugConstantine", td.path + else: + test "", td.path if sizeof(int) == 8: # 32-bit tests on 64-bit arch for td in testDesc: if not td.useGMP: - test "-d:Constantine32", td.path + if td.path in useDebug: + test "-d:Constantine32 -d:debugConstantine", td.path + else: + test "-d:Constantine32", td.path + # Benchmarks compile and run # ignore Windows 32-bit for the moment @@ -154,7 +173,10 @@ task test_parallel, "Run all tests in parallel (via GNU parallel)": exec "> " & buildParallel for td in testDesc: - test "", td.path, cmdFile + if td.path in useDebug: + test "-d:debugConstantine", td.path, cmdFile + else: + test "", td.path, cmdFile # cmdFile.close() # Execute everything in parallel with GNU parallel @@ -163,7 +185,10 @@ task test_parallel, "Run all tests in parallel (via GNU parallel)": exec "> " & buildParallel if sizeof(int) == 8: # 32-bit tests on 64-bit arch for td in testDesc: - test "-d:Constantine32", td.path, cmdFile + if td.path in useDebug: + test "-d:Constantine32 -d:debugConstantine", td.path, cmdFile + else: + test "-d:Constantine32", td.path, cmdFile # cmdFile.close() # Execute everything in parallel with GNU parallel exec "parallel --keep-order --group < " & buildParallel diff --git a/constantine/arithmetic/limbs_modular.nim b/constantine/arithmetic/limbs_modular.nim index a0ddc88..c0b0586 100644 --- a/constantine/arithmetic/limbs_modular.nim +++ b/constantine/arithmetic/limbs_modular.nim @@ -143,7 +143,7 @@ func steinsGCD*(v: var Limbs, a: Limbs, F, M: Limbs, bits: int, mp1div2: Limbs) # GCD exist (always true if a and M are relatively prime) doAssert bool b.isOne() or # or not (on prime fields iff input was zero) and no GCD fallback output is zero - (a.isZero() and v.isZero()) + v.isZero() # ############################################################ # diff --git a/tests/support/canaries.nim b/tests/support/canaries.nim new file mode 100644 index 0000000..79c2f9f --- /dev/null +++ b/tests/support/canaries.nim @@ -0,0 +1,31 @@ +# Constantine +# Copyright (c) 2018-2019 Status Research & Development GmbH +# Copyright (c) 2020-Present Mamy André-Ratsimbazafy +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + ../../constantine/arithmetic/bigints, + ../../constantine/config/[common, curves], + ../../constantine/elliptic/[ec_weierstrass_affine, ec_weierstrass_projective] + +# Canaries +# -------------------------------------------------------------- +# +# This file initializes a type with canary +# to detect initialization bugs that are silent +# when initialized from zero. + +when sizeof(SecretWord) == 8: + const Canary = SecretWord(0xAAFACADEAAFACADE'u64) +else: + const Canary = SecretWord(0xAAFACADE'u32) + +func canary*(T: typedesc): T = + when T is BigInt: + for i in 0 ..< result.limbs.len: + result.limbs[0] = Canary + else: + {.error: "Not implemented".} diff --git a/tests/t_bigints.nim b/tests/t_bigints.nim index 11ae074..7136650 100644 --- a/tests/t_bigints.nim +++ b/tests/t_bigints.nim @@ -6,11 +6,16 @@ # * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -import std/unittest, - ../constantine/io/io_bigints, - ../constantine/arithmetic, - ../constantine/config/common, - ../constantine/primitives +import + # Standard library + std/unittest, + # Internal + ../constantine/io/io_bigints, + ../constantine/arithmetic, + ../constantine/config/[common, type_bigint], + ../constantine/primitives, + # Test utilities, + support/canaries echo "\n------------------------------------------------------\n" @@ -143,7 +148,7 @@ proc mainArith() = suite "Multi-precision multiplication" & " [" & $WordBitwidth & "-bit mode]": test "Same size operand into double size result": block: - var r: BigInt[256] + var r = canary(BigInt[256]) let a = BigInt[128].fromHex"0x12345678_FF11FFAA_00321321_CAFECAFE" let b = BigInt[128].fromHex"0xDEADBEEF_DEADBEEF_DEADBEEF_DEADBEEF" @@ -156,7 +161,7 @@ proc mainArith() = test "Different size into large result": block: - var r: BigInt[200] + var r = canary(BigInt[200]) let a = BigInt[29].fromHex"0x12345678" let b = BigInt[128].fromHex"0xDEADBEEF_DEADBEEF_DEADBEEF_DEADBEEF" @@ -183,7 +188,7 @@ proc mainArith() = suite "Multi-precision multiplication keeping only high words" & " [" & $WordBitwidth & "-bit mode]": test "Same size operand into double size result - discard first word": block: - var r: BigInt[256] + var r = canary(BigInt[256]) let a = BigInt[128].fromHex"0x12345678_FF11FFAA_00321321_CAFECAFE" let b = BigInt[128].fromHex"0xDEADBEEF_DEADBEEF_DEADBEEF_DEADBEEF" @@ -199,7 +204,7 @@ proc mainArith() = test "Same size operand into double size result - discard first 3 words": block: - var r: BigInt[256] + var r = canary(BigInt[256]) let a = BigInt[128].fromHex"0x12345678_FF11FFAA_00321321_CAFECAFE" let b = BigInt[128].fromHex"0xDEADBEEF_DEADBEEF_DEADBEEF_DEADBEEF" @@ -215,7 +220,7 @@ proc mainArith() = test "All lower words trigger a carry": block: - var r: BigInt[256] + var r = canary(BigInt[256]) let a = BigInt[256].fromHex"0xFFFFF000_FFFFF111_FFFFFFFA_FFFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF" let b = BigInt[256].fromHex"0xFFFFFFFF_FFFFF222_FFFFFFFB_FFFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF_FFFFFFFF" @@ -235,7 +240,7 @@ proc mainArith() = test "Different size into large result": block: - var r: BigInt[200] + var r = canary(BigInt[200]) let a = BigInt[29].fromHex"0x12345678" let b = BigInt[128].fromHex"0xDEADBEEF_DEADBEEF_DEADBEEF_DEADBEEF" @@ -273,46 +278,56 @@ proc mainArith() = let a = BigInt[7].fromUint(100'u32) let m = BigInt[4].fromUint(13'u8) - var r: BigInt[4] + var r = canary(BigInt[4]) r.reduce(a, m) - check: - bool(r == BigInt[4].fromUint(100'u8 mod 13)) + let expected = BigInt[4].fromUint(100'u8 mod 13) + doAssert bool(r == expected), + "\n r (low-level repr): " & $r & + "\n expected (ll repr): " & $expected block: # let a = BigInt[32].fromUint(100'u32) let m = BigInt[4].fromUint(13'u8) - var r: BigInt[4] + var r = canary(BigInt[4]) r.reduce(a, m) - check: - bool(r == BigInt[4].fromUint(100'u8 mod 13)) + let expected = BigInt[4].fromUint(100'u8 mod 13) + doAssert bool(r == expected), + "\n r (low-level repr): " & $r & + "\n expected (ll repr): " & $expected block: # let a = BigInt[64].fromUint(100'u32) let m = BigInt[4].fromUint(13'u8) - var r: BigInt[4] + var r = canary(BigInt[4]) r.reduce(a, m) - check: - bool(r == BigInt[4].fromUint(100'u8 mod 13)) + let expected = BigInt[4].fromUint(100'u8 mod 13) + doAssert bool(r == expected), + "\n r (low-level repr): " & $r & + "\n expected (ll repr): " & $expected test "2^64 mod 3": let a = BigInt[65].fromHex("0x1_00000000_00000000") let m = BigInt[8].fromUint(3'u8) - var r: BigInt[8] + var r = canary(BigInt[8]) r.reduce(a, m) - check: - bool(r == BigInt[8].fromUint(1'u8)) + let expected = BigInt[8].fromUint(1'u8) + doAssert bool(r == expected), + "\n r (low-level repr): " & $r & + "\n expected (ll repr): " & $expected test "1234567891234567890 mod 10": let a = BigInt[64].fromUint(1234567891234567890'u64) let m = BigInt[8].fromUint(10'u8) - var r: BigInt[8] + var r = canary(BigInt[8]) r.reduce(a, m) - check: - bool(r == BigInt[8].fromUint(0'u8)) + let expected = BigInt[8].fromUint(0'u8) + doAssert bool(r == expected), + "\n r (low-level repr): " & $r & + "\n expected (ll repr): " & $expected suite "Modular operations - small modulus - Stint specific failures highlighted by property-based testing" & " [" & $WordBitwidth & "-bit mode]": # Vectors taken from Stint - https://github.com/status-im/nim-stint @@ -323,11 +338,13 @@ proc mainArith() = let a = BigInt[56].fromUint(u) let m = BigInt[48].fromUint(v) - var r: BigInt[48] + var r = canary(BigInt[48]) r.reduce(a, m) - check: - bool(r == BigInt[48].fromUint(u mod v)) + let expected = BigInt[48].fromUint(u mod v) + doAssert bool(r == expected), + "\n r (low-level repr): " & $r & + "\n expected (ll repr): " & $expected test "Modulo: 15080397990160655 mod 600432699691": let u = 15080397990160655'u64 @@ -336,11 +353,13 @@ proc mainArith() = let a = BigInt[54].fromUint(u) let m = BigInt[40].fromUint(v) - var r: BigInt[40] + var r = canary(BigInt[40]) r.reduce(a, m) - check: - bool(r == BigInt[40].fromUint(u mod v)) + let expected = BigInt[40].fromUint(u mod v) + doAssert bool(r == expected), + "\n r (low-level repr): " & $r & + "\n expected (ll repr): " & $expected proc mainNeg() = suite "Conditional negation" & " [" & $WordBitwidth & "-bit mode]": @@ -501,7 +520,7 @@ proc mainModularInverse() = mp1div2.shiftRight(1) let expected = BigInt[16].fromUint(1969'u16) - var r {.noInit.}: BigInt[16] + var r = canary(BigInt[16]) r.invmod(a, M, mp1div2) @@ -516,7 +535,7 @@ proc mainModularInverse() = mp1div2.shiftRight(1) let expected = BigInt[381].fromUint(1969'u16) - var r {.noInit.}: BigInt[381] + var r = canary(BigInt[381]) r.invmod(a, M, mp1div2) @@ -532,7 +551,7 @@ proc mainModularInverse() = mp1div2.shiftRight(1) let expected = BigInt[16].fromUint(106'u16) - var r {.noInit.}: BigInt[16] + var r = canary(BigInt[16]) r.invmod(a, M, mp1div2) @@ -547,7 +566,7 @@ proc mainModularInverse() = mp1div2.shiftRight(1) let expected = BigInt[381].fromUint(106'u16) - var r {.noInit.}: BigInt[381] + var r = canary(BigInt[381]) r.invmod(a, M, mp1div2) @@ -563,7 +582,7 @@ proc mainModularInverse() = let expected = BigInt[381].fromHex("0x0636759a0f3034fa47174b2c0334902f11e9915b7bd89c6a2b3082b109abbc9837da17201f6d8286fe6203caa1b9d4c8") - var r {.noInit.}: BigInt[381] + var r = canary(BigInt[381]) r.invmod(a, M, mp1div2) check: bool(r == expected) @@ -578,7 +597,7 @@ proc mainModularInverse() = discard mp1div2.add(SecretWord 1) let expected = BigInt[16].fromUint(0'u16) - var r {.noInit.}: BigInt[16] + var r = canary(BigInt[16]) r.invmod(a, M, mp1div2) @@ -593,7 +612,7 @@ proc mainModularInverse() = discard mp1div2.add(SecretWord 1) let expected = BigInt[381].fromUint(0'u16) - var r {.noInit.}: BigInt[381] + var r = canary(BigInt[381]) r.invmod(a, M, mp1div2)