codex-storage
/
constantine
mirror of https://github.com/codex-storage/constantine.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CI: enable GMP tests on Windows and Linux 32-bit and fix caching (#204) 

* Try to compile with GMP on windows and 32-bit linux

* remove leftover msys shell

* Don't use GMP Mersenne Twister, bad randomness and untested Nim wrapper

* properly cache nim

* fix path after cache

* run pacman in msys2 env

* rework msys2 ... again

* shell compat for file clearing

* shell compat try-again for file clearing

* force bash for clearing parallel builds on windows

* Use nimscript directly (why didn't it work last time?)

* Avoid IO redirection to support any shell

* Avoid IO redirection v2 to support any shell

* add debug data

* add debug again

* Introduce pararun, a parallel test runner to remove need of GNU parallel

* pararun: style
This commit is contained in:
Mamy Ratsimbazafy 2022-09-15 09:33:34 +02:00 committed by GitHub
parent 094445482b
commit 962e7ccf49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 431 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
.github/workflows/ci.yml vendored
View File

@ -117,10 +117,9 @@ jobs:
run: | run: |
echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH
- name: Restore Nim from cache - name: Restore Nim from cache
if: > if: matrix.nim_version != 'devel'
steps.nim-compiler-cache.outputs.cache-hit != 'true' &&
matrix.nim_version != 'devel'
id: nim-compiler-cache id: nim-compiler-cache
uses: actions/cache@v2 uses: actions/cache@v2
with: with:
@ -128,20 +127,28 @@ jobs:
key: 'nim-${{ matrix.target.cpu }}-${{ matrix.nim_version }}' key: 'nim-${{ matrix.target.cpu }}-${{ matrix.nim_version }}'
- name: Setup Nim - name: Setup Nim
if: steps.nim-compiler-cache.outputs.cache-hit != 'true'
uses: alaviss/setup-nim@0.1.1 uses: alaviss/setup-nim@0.1.1
with: with:
path: 'nim' path: 'nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}'
version: ${{ matrix.nim_version }} version: ${{ matrix.nim_version }}
architecture: ${{ matrix.target.cpu }} architecture: ${{ matrix.target.cpu }}
add-to-path: false
- name: Install dependencies (Linux amd64) - name: Path to cached Nim
shell: bash
run: |
echo '${{ github.workspace }}'"/nim-${{ matrix.nim_version }}-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
echo '${{ github.workspace }}'"/.nimble/bin" >> $GITHUB_PATH
- name: Install test dependencies (Linux amd64)
if: runner.os == 'Linux' && matrix.target.cpu == 'amd64' if: runner.os == 'Linux' && matrix.target.cpu == 'amd64'
run: | run: |
sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
--no-install-recommends -yq \ --no-install-recommends -yq \
libgmp-dev libgmp-dev
- name: Install dependencies (Linux i386) - name: Install test dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386' if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: | run: |
sudo dpkg --add-architecture i386 sudo dpkg --add-architecture i386
@ -160,70 +167,51 @@ jobs:
#!/bin/bash #!/bin/bash
exec $(which g++) -m32 "\$@" exec $(which g++) -m32 "\$@"
EOF EOF
chmod 755 external/bin/gcc external/bin/g++ chmod 755 external/bin/{gcc,g++}
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: Install dependencies (macOS) - name: Install test dependencies (macOS)
if: runner.os == 'macOS' if: runner.os == 'macOS'
run: brew install gmp parallel run: brew install gmp
- name: Setup MSYS2 (Windows) - name: Setup MSYS2 (Windows)
if: runner.os == 'Windows' if: runner.os == 'Windows'
uses: msys2/setup-msys2@v2 uses: msys2/setup-msys2@v2
with: with:
path-type: inherit path-type: inherit
update: true update: false
install: base-devel git mingw-w64-x86_64-toolchain install: base-devel git mingw-w64-x86_64-toolchain
- name: Install dependencies (Windows) - name: Install test dependencies (Windows)
if: runner.os == 'Windows' if: runner.os == 'Windows'
shell: msys2 {0} shell: msys2 {0}
run: | run: |
pacman -S --needed --noconfirm mingw-w64-x86_64-gmp pacman -S --needed --noconfirm mingw-w64-x86_64-gmp
pacman -S --needed --noconfirm parallel nimble refresh --verbose -y
nimble refresh -y nimble install --verbose -y gmp stew jsony asynctools
nimble install -y gmp stew jsony
- name: Install test dependencies - name: Install test dependencies
if: runner.os != 'Windows' if: runner.os != 'Windows'
shell: bash shell: bash
run: | run: |
nimble refresh -y nimble refresh --verbose -y
nimble install -y gmp stew jsony nimble install --verbose -y gmp stew jsony asynctools
- name: Run Constantine tests (with Assembler & with GMP) - name: Run Constantine tests (UNIX with Assembler)
if: (runner.os == 'Linux' || runner.os == 'macOS') && matrix.target.BACKEND == 'ASM' && matrix.target.cpu != 'i386' if: runner.os != 'Windows' && matrix.target.BACKEND == 'ASM'
shell: bash shell: bash
run: | run: |
export UCPU="$cpu"
cd constantine cd constantine
nimble test_parallel nimble test_parallel --verbose
- name: Run Constantine tests (no Assembler & with GMP) - name: Run Constantine tests (UNIX no Assembler)
if: (runner.os == 'Linux' || runner.os == 'macOS') && matrix.target.BACKEND == 'NO_ASM' && matrix.target.cpu != 'i386' if: runner.os != 'Windows' && matrix.target.BACKEND == 'NO_ASM'
shell: bash shell: bash
run: | run: |
export UCPU="$cpu"
cd constantine cd constantine
nimble test_parallel_no_assembler nimble test_parallel_no_asm --verbose
- name: Run Constantine tests (without GMP) - name: Run Constantine tests (Windows no Assembler)
if: runner.os == 'Linux' && matrix.target.BACKEND == 'ASM' && matrix.target.cpu == 'i386' if: runner.os == 'Windows' && matrix.target.BACKEND == 'NO_ASM'
shell: bash
run: |
export UCPU="$cpu"
cd constantine
nimble test_parallel_no_gmp
- name: Run Constantine tests (without Assembler or GMP)
if: runner.os == 'Linux' && matrix.target.BACKEND == 'NO_ASM' && matrix.target.cpu == 'i386'
shell: bash
run: |
export UCPU="$cpu"
cd constantine
nimble test_parallel_no_gmp_no_assembler
- name: Run Constantine tests (Windows - without Assembler or GMP)
# TODO, why aren't GMP or parallel in path?
if: runner.os == 'Windows'
shell: msys2 {0} shell: msys2 {0}
run: | run: |
export UCPU="$cpu"
cd constantine cd constantine
nimble test_no_gmp_no_assembler nimble test_parallel_no_asm --verbose

287
constantine.nimble
View File

@ -21,6 +21,24 @@ const buildParallel = "test_parallel.txt"
# Code refactoring requires re-enabling the full suite. # Code refactoring requires re-enabling the full suite.
# Basic primitives should stay on to catch compiler regressions. # Basic primitives should stay on to catch compiler regressions.
const testDesc: seq[tuple[path: string, useGMP: bool]] = @[ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[
# Hashing vs OpenSSL
# ----------------------------------------------------------
("tests/t_hash_sha256_vs_openssl.nim", true), # skip OpenSSL tests on Windows
# Ciphers
# ----------------------------------------------------------
("tests/t_cipher_chacha20.nim", false),
# Message Authentication Code
# ----------------------------------------------------------
("tests/t_mac_poly1305.nim", false),
("tests/t_mac_hmac_sha256.nim", false),
# KDF
# ----------------------------------------------------------
("tests/t_kdf_hkdf.nim", false),
# Primitives # Primitives
# ---------------------------------------------------------- # ----------------------------------------------------------
("tests/math/t_primitives.nim", false), ("tests/math/t_primitives.nim", false),
@ -188,29 +206,12 @@ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[
# ---------------------------------------------------------- # ----------------------------------------------------------
("tests/math/t_fr.nim", false), ("tests/math/t_fr.nim", false),
# Hashing vs OpenSSL
# ----------------------------------------------------------
("tests/t_hash_sha256_vs_openssl.nim", true), # skip OpenSSL tests on Windows
# Hashing to elliptic curves # Hashing to elliptic curves
# ---------------------------------------------------------- # ----------------------------------------------------------
("tests/t_hash_to_field.nim", false), ("tests/t_hash_to_field.nim", false),
("tests/t_hash_to_curve_random.nim", false), ("tests/t_hash_to_curve_random.nim", false),
("tests/t_hash_to_curve.nim", false), ("tests/t_hash_to_curve.nim", false),
# Ciphers
# ----------------------------------------------------------
("tests/t_cipher_chacha20.nim", false),
# Message Authentication Code
# ----------------------------------------------------------
("tests/t_mac_poly1305.nim", false),
("tests/t_mac_hmac_sha256.nim", false),
# KDF
# ----------------------------------------------------------
("tests/t_kdf_hkdf.nim", false),
# Protocols # Protocols
# ---------------------------------------------------------- # ----------------------------------------------------------
("tests/t_ethereum_evm_precompiles.nim", false), ("tests/t_ethereum_evm_precompiles.nim", false),
@ -218,6 +219,26 @@ const testDesc: seq[tuple[path: string, useGMP: bool]] = @[
("tests/t_ethereum_eip2333_bls12381_key_derivation.nim", false), ("tests/t_ethereum_eip2333_bls12381_key_derivation.nim", false),
] ]
const benchDesc = [
"bench_fp",
"bench_fp_double_precision",
"bench_fp2",
"bench_fp6",
"bench_fp12",
"bench_ec_g1",
"bench_ec_g2",
"bench_pairing_bls12_377",
"bench_pairing_bls12_381",
"bench_pairing_bn254_nogami",
"bench_pairing_bn254_snarks",
"bench_summary_bls12_377",
"bench_summary_bls12_381",
"bench_summary_bn254_nogami",
"bench_summary_bn254_snarks",
"bench_sha256",
"bench_hash_to_curve"
]
# For temporary (hopefully) investigation that can only be reproduced in CI # For temporary (hopefully) investigation that can only be reproduced in CI
const useDebug = [ const useDebug = [
"tests/math/t_bigints.nim", "tests/math/t_bigints.nim",
@ -256,13 +277,11 @@ else:
# ---------------------------------------------------------------- # ----------------------------------------------------------------
proc clearParallelBuild() = proc clearParallelBuild() =
exec "> " & buildParallel # Support clearing from non POSIX shell like CMD, Powershell or MSYS2
if fileExists(buildParallel):
rmFile(buildParallel)
proc test(flags, path: string, commandFile = false) = template setupCommand(): untyped {.dirty.} =
# commandFile should be a "file" but Nimscript doesn't support IO
if not dirExists "build":
mkDir "build"
# Compilation language is controlled by WEAVE_TEST_LANG
var lang = "c" var lang = "c"
if existsEnv"TEST_LANG": if existsEnv"TEST_LANG":
lang = getEnv"TEST_LANG" lang = getEnv"TEST_LANG"
@ -280,36 +299,55 @@ proc test(flags, path: string, commandFile = false) =
" --nimcache:nimcache/" & path & " " & " --nimcache:nimcache/" & path & " " &
path path
if not commandFile: proc test(cmd: string) =
echo "\n==============================================================================================" echo "\n=============================================================================================="
echo "Running [flags:", flags, "] ", path echo "Running '", cmd, "'"
echo "==============================================================================================" echo "=============================================================================================="
exec command exec cmd
else:
exec "echo \'" & command & "\' >> " & buildParallel
exec "echo \"------------------------------------------------------\""
proc buildBench(benchName: string, compiler = "", useAsm = true, run = false) = proc testBatch(commands: var string, flags, path: string) =
if not dirExists "build": setupCommand()
mkDir "build" commands &= command & '\n'
template setupBench(): untyped {.dirty.} =
let runFlag = if run: " -r " let runFlag = if run: " -r "
else: " " else: " "
var lang = " c "
if existsEnv"TEST_LANG":
lang = getEnv"TEST_LANG"
var cc = "" var cc = ""
if compiler != "": if compiler != "":
cc = "--cc:" & compiler cc = "--cc:" & compiler
elif existsEnv"CC":
cc = " --cc:" & getEnv"CC"
if not useAsm: if not useAsm:
cc &= " -d:CttASM=false" cc &= " -d:CttASM=false"
exec "nim c " & cc & let command = "nim " & lang & cc &
" -d:danger --verbosity:0 -o:build/bench/" & benchName & "_" & compiler & "_" & (if useAsm: "useASM" else: "noASM") & " -d:danger --verbosity:0 -o:build/bench/" & benchName & "_" & compiler & "_" & (if useAsm: "useASM" else: "noASM") &
" --nimcache:nimcache/" & benchName & "_" & compiler & "_" & (if useAsm: "useASM" else: "noASM") & " --nimcache:nimcache/" & benchName & "_" & compiler & "_" & (if useAsm: "useASM" else: "noASM") &
runFlag & "--hints:off --warnings:off benchmarks/" & benchName & ".nim" runFlag & "--hints:off --warnings:off benchmarks/" & benchName & ".nim"
proc runBench(benchName: string, compiler = "", useAsm = true) = proc runBench(benchName: string, compiler = "", useAsm = true) =
buildBench(benchName, compiler, useAsm, run = true) if not dirExists "build":
mkDir "build"
let run = true
setupBench()
exec command
proc runTests(requireGMP: bool, dumpCmdFile = false, test32bit = false, testASM = true) = proc buildBenchBatch(commands: var string, benchName: string, compiler = "", useAsm = true) =
let run = false
let compiler = ""
setupBench()
commands &= command & '\n'
proc addTestSet(cmdFile: var string, requireGMP: bool, test32bit = false, testASM = true) =
if not dirExists "build":
mkDir "build"
echo "Found " & $testDesc.len & " tests to run."
for td in testDesc: for td in testDesc:
if not(td.useGMP and not requireGMP): if not(td.useGMP and not requireGMP):
var flags = "" var flags = ""
@ -321,29 +359,15 @@ proc runTests(requireGMP: bool, dumpCmdFile = false, test32bit = false, testASM
flags &= " -d:debugConstantine" flags &= " -d:debugConstantine"
if td.path notin skipSanitizers: if td.path notin skipSanitizers:
flags &= sanitizers flags &= sanitizers
test flags, td.path, dumpCmdFile
cmdFile.testBatch(flags, td.path)
proc buildAllBenches(useAsm = true) = proc addBenchSet(cmdFile: var string, useAsm = true) =
echo "\n\n------------------------------------------------------\n" if not dirExists "build":
echo "Building benchmarks to ensure they stay relevant ..." mkDir "build"
buildBench("bench_fp", useAsm = useAsm) echo "Found " & $benchDesc.len & " benches to compile. (compile-only to ensure they stay relevant)"
buildBench("bench_fp_double_precision", useAsm = useAsm) for bd in benchDesc:
buildBench("bench_fp2", useAsm = useAsm) cmdFile.buildBenchBatch(bd, useASM = useASM)
buildBench("bench_fp6", useAsm = useAsm)
buildBench("bench_fp12", useAsm = useAsm)
buildBench("bench_ec_g1", useAsm = useAsm)
buildBench("bench_ec_g2", useAsm = useAsm)
buildBench("bench_pairing_bls12_377", useAsm = useAsm)
buildBench("bench_pairing_bls12_381", useAsm = useAsm)
buildBench("bench_pairing_bn254_nogami", useAsm = useAsm)
buildBench("bench_pairing_bn254_snarks", useAsm = useAsm)
buildBench("bench_summary_bls12_377", useAsm = useAsm)
buildBench("bench_summary_bls12_381", useAsm = useAsm)
buildBench("bench_summary_bn254_nogami", useAsm = useAsm)
buildBench("bench_summary_bn254_snarks", useAsm = useAsm)
buildBench("bench_sha256", useAsm = useAsm)
buildBench("bench_hash_to_curve", useAsm = useAsm)
echo "All benchmarks compile successfully."
proc genBindings(bindingsName, prefixNimMain: string) = proc genBindings(bindingsName, prefixNimMain: string) =
proc compile(libName: string, flags = "") = proc compile(libName: string, flags = "") =
@ -379,6 +403,9 @@ proc genHeaders(bindingsName: string) =
" --nimcache:nimcache/bindings/" & bindingsName & "_header" & " --nimcache:nimcache/bindings/" & bindingsName & "_header" &
" bindings/" & bindingsName & ".nim" " bindings/" & bindingsName & ".nim"
proc genParallelCmdRunner() =
exec "nim c --verbosity:0 --hints:off --warnings:off -d:release --out:build/pararun --nimcache:nimcache/pararun helpers/pararun.nim"
# Tasks # Tasks
# ---------------------------------------------------------------- # ----------------------------------------------------------------
@ -390,123 +417,79 @@ task bindings, "Generate Constantine bindings":
task test, "Run all tests": task test, "Run all tests":
# -d:testingCurves is configured in a *.nim.cfg for convenience # -d:testingCurves is configured in a *.nim.cfg for convenience
runTests(requireGMP = true) var cmdFile: string
cmdFile.addTestSet(requireGMP = true, testASM = true)
cmdFile.addBenchSet(useASM = true) # Build (but don't run) benches to ensure they stay relevant
for cmd in cmdFile.splitLines():
exec cmd
# if sizeof(int) == 8: # 32-bit tests on 64-bit arch task test_no_asm, "Run all tests (no assembly)":
# runTests(requireGMP = true, test32bit = true)
# Ensure benchmarks stay relevant. Ignore Windows 32-bit at the moment
if not defined(windows) or not (existsEnv"UCPU" or getEnv"UCPU" == "i686"):
buildAllBenches()
task test_no_assembler, "Run all tests":
# -d:testingCurves is configured in a *.nim.cfg for convenience # -d:testingCurves is configured in a *.nim.cfg for convenience
runTests(requireGMP = true, testASM = false) var cmdFile: string
cmdFile.addTestSet(requireGMP = true, testASM = false)
# if sizeof(int) == 8: # 32-bit tests on 64-bit arch cmdFile.addBenchSet(useASM = false) # Build (but don't run) benches to ensure they stay relevant
# runTests(requireGMP = true, test32bit = true) for cmd in cmdFile.splitLines():
exec cmd
# Ensure benchmarks stay relevant. Ignore Windows 32-bit at the moment
if not defined(windows) or not (existsEnv"UCPU" or getEnv"UCPU" == "i686"):
buildAllBenches(useASM = false)
task test_no_gmp, "Run tests that don't require GMP": task test_no_gmp, "Run tests that don't require GMP":
# -d:testingCurves is configured in a *.nim.cfg for convenience # -d:testingCurves is configured in a *.nim.cfg for convenience
runTests(requireGMP = false) var cmdFile: string
cmdFile.addTestSet(requireGMP = false, testASM = true)
cmdFile.addBenchSet(useASM = true) # Build (but don't run) benches to ensure they stay relevant
for cmd in cmdFile.splitLines():
exec cmd
# if sizeof(int) == 8: # 32-bit tests on 64-bit arch task test_no_gmp_no_asm, "Run tests that don't require GMP using a pure Nim backend":
# runTests(requireGMP = true, test32bit = true)
# Ensure benchmarks stay relevant. Ignore Windows 32-bit at the moment
if not defined(windows) or not (existsEnv"UCPU" or getEnv"UCPU" == "i686"):
buildAllBenches()
task test_no_gmp_no_assembler, "Run tests that don't require GMP using a pure Nim backend":
# -d:testingCurves is configured in a *.nim.cfg for convenience # -d:testingCurves is configured in a *.nim.cfg for convenience
runTests(requireGMP = false, testASM = false) var cmdFile: string
cmdFile.addTestSet(requireGMP = false, testASM = false)
# if sizeof(int) == 8: # 32-bit tests on 64-bit arch cmdFile.addBenchSet(useASM = false) # Build (but don't run) benches to ensure they stay relevant
# runTests(requireGMP = true, test32bit = true) for cmd in cmdFile.splitLines():
exec cmd
# Ensure benchmarks stay relevant. Ignore Windows 32-bit at the moment
if not defined(windows) or not (existsEnv"UCPU" or getEnv"UCPU" == "i686"):
buildAllBenches()
task test_parallel, "Run all tests in parallel (via GNU parallel)": task test_parallel, "Run all tests in parallel (via GNU parallel)":
# -d:testingCurves is configured in a *.nim.cfg for convenience # -d:testingCurves is configured in a *.nim.cfg for convenience
clearParallelBuild() clearParallelBuild()
runTests(requireGMP = true, dumpCmdFile = true) genParallelCmdRunner()
exec "parallel --keep-order --group < " & buildParallel
# if sizeof(int) == 8: # 32-bit tests on 64-bit arch var cmdFile: string
# clearParallelBuild() cmdFile.addTestSet(requireGMP = true, testASM = true)
# runTests(requireGMP = true, dumpCmdFile = true, test32bit = true) cmdFile.addBenchSet(useASM = true) # Build (but don't run) benches to ensure they stay relevant
# exec "parallel --keep-order --group < " & buildParallel writeFile(buildParallel, cmdFile)
exec "build/pararun " & buildParallel
# Now run the benchmarks task test_parallel_no_asm, "Run all tests (without macro assembler) in parallel (via GNU parallel)":
#
# Benchmarks compile
# ignore Windows 32-bit for the moment
# Ensure benchmarks stay relevant. Ignore Windows 32-bit at the moment
if not defined(windows) or not (existsEnv"UCPU" or getEnv"UCPU" == "i686"):
buildAllBenches()
task test_parallel_no_assembler, "Run all tests (without macro assembler) in parallel (via GNU parallel)":
# -d:testingCurves is configured in a *.nim.cfg for convenience # -d:testingCurves is configured in a *.nim.cfg for convenience
clearParallelBuild() clearParallelBuild()
runTests(requireGMP = true, dumpCmdFile = true, testASM = false) genParallelCmdRunner()
exec "parallel --keep-order --group < " & buildParallel
# if sizeof(int) == 8: # 32-bit tests on 64-bit arch var cmdFile: string
# clearParallelBuild() cmdFile.addTestSet(requireGMP = true, testASM = false)
# runTests(requireGMP = true, dumpCmdFile = true, test32bit = true, testASM = false) cmdFile.addBenchSet(useASM = false)
# exec "parallel --keep-order --group < " & buildParallel writeFile(buildParallel, cmdFile)
exec "build/pararun " & buildParallel
# Now run the benchmarks
#
# Benchmarks compile
# ignore Windows 32-bit for the moment
# Ensure benchmarks stay relevant. Ignore Windows 32-bit at the moment
if not defined(windows) or not (existsEnv"UCPU" or getEnv"UCPU" == "i686"):
buildAllBenches(useASM = false)
task test_parallel_no_gmp, "Run all tests in parallel (via GNU parallel)": task test_parallel_no_gmp, "Run all tests in parallel (via GNU parallel)":
# -d:testingCurves is configured in a *.nim.cfg for convenience # -d:testingCurves is configured in a *.nim.cfg for convenience
clearParallelBuild() clearParallelBuild()
runTests(requireGMP = false, dumpCmdFile = true) genParallelCmdRunner()
exec "parallel --keep-order --group < " & buildParallel
# if sizeof(int) == 8: # 32-bit tests on 64-bit arch var cmdFile: string
# clearParallelBuild() cmdFile.addTestSet(requireGMP = false, testASM = true)
# runTests(requireGMP = false, dumpCmdFile = true, test32bit = true) cmdFile.addBenchSet(useASM = true) # Build (but don't run) benches to ensure they stay relevant
# exec "parallel --keep-order --group < " & buildParallel writeFile(buildParallel, cmdFile)
exec "build/pararun " & buildParallel
# Now run the benchmarks task test_parallel_no_gmp_no_asm, "Run all tests in parallel (via GNU parallel)":
#
# Benchmarks compile
# ignore Windows 32-bit for the moment
# Ensure benchmarks stay relevant. Ignore Windows 32-bit at the moment
if not defined(windows) or not (existsEnv"UCPU" or getEnv"UCPU" == "i686"):
buildAllBenches()
task test_parallel_no_gmp_no_assembler, "Run all tests in parallel (via GNU parallel)":
# -d:testingCurves is configured in a *.nim.cfg for convenience # -d:testingCurves is configured in a *.nim.cfg for convenience
clearParallelBuild() clearParallelBuild()
runTests(requireGMP = false, dumpCmdFile = true, testASM = false) genParallelCmdRunner()
exec "parallel --keep-order --group < " & buildParallel
# if sizeof(int) == 8: # 32-bit tests on 64-bit arch var cmdFile: string
# clearParallelBuild() cmdFile.addTestSet(requireGMP = false, testASM = false)
# runTests(requireGMP = false, dumpCmdFile = true, test32bit = true, testASM = false) cmdFile.addBenchSet(useASM = false) # Build (but don't run) benches to ensure they stay relevant
# exec "parallel --keep-order --group < " & buildParallel writeFile(buildParallel, cmdFile)
exec "build/pararun " & buildParallel
# Now run the benchmarks
#
# Benchmarks compile
# ignore Windows 32-bit for the moment
# Ensure benchmarks stay relevant. Ignore Windows 32-bit at the moment
if not defined(windows) or not (existsEnv"UCPU" or getEnv"UCPU" == "i686"):
buildAllBenches(useASM = false)
# Finite field 𝔽p # Finite field 𝔽p
# ------------------------------------------ # ------------------------------------------

151
helpers/pararun.nim Normal file
View File

@ -0,0 +1,151 @@
# Constantine
# Copyright (c) 2018-2019 Status Research & Development GmbH
# Copyright (c) 2020-Present Mamy André-Ratsimbazafy
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
std/[os, strutils, cpuinfo, strformat, deques, terminal],
std/[asyncfutures, asyncdispatch],
asynctools/[asyncproc, asyncpipe, asyncsync]
# Pararun is a parallel shell command runner
# ------------------------------------------
# Usage: pararun <file-with-1-command-per-line> <numWorkers
# AsyncSemaphore
# ----------------------------------------------------------------
type AsyncSemaphore = ref object
waiters: Deque[Future[void]]
slots, max: int
proc new(_: type AsyncSemaphore, max: int): AsyncSemaphore =
## Initialize an AsyncSemaphore that can release up to max items
AsyncSemaphore(
waiters: default(Deque[Future[void]]),
slots: max,
max: max)
proc acquire(s: AsyncSemaphore) {.async.} =
doAssert s.slots in {0..s.max}
if s.slots == 0:
let waiter = newFuture[void]("AsyncSemaphore.acquire")
s.waiters.addLast(waiter)
await waiter
s.slots -= 1
doAssert s.slots in {0..s.max}
proc release(s: AsyncSemaphore) =
doAssert s.slots in {0..s.max-1}
s.slots += 1
if s.waiters.len > 0:
let waiter = s.waiters.popFirst()
waiter.complete()
doAssert s.slots in {0..s.max}
# Task runner
# ----------------------------------------------------------------
type WorkQueue = ref object
sem: AsyncSemaphore
cmdQueue: Deque[string]
outputQueue: AsyncQueue[tuple[cmd: string, p: AsyncProcess]]
lineBuf: string
proc releaseOnProcessExit(sem: AsyncSemaphore, p: AsyncProcess) {.async.} =
# TODO: addProcess callback on exit is cleaner but locks the AsyncPipe "readInto"
#
# p.processID.addProcess do (fd: AsyncFD) -> bool:
# sem.release()
#
# see also: https://forum.nim-lang.org/t/5565
while p.running():
await sleepAsync(10)
sem.release()
proc enqueuePendingCommands(wq: WorkQueue) {.async.} =
while wq.cmdQueue.len > 0:
await wq.sem.acquire()
let cmd = wq.cmdQueue.popFirst()
let p = cmd.startProcess(
options = {poStdErrToStdOut, poUsePath, poEvalCommand}
)
asyncCheck wq.sem.releaseOnProcessExit(p)
wq.outputQueue.putNoWait((cmd, p))
proc flushCommandsOutput(wq: WorkQueue) {.async.} =
while true:
let (cmd, p) = await wq.outputQueue.get()
echo '\n', '='.repeat(80)
echo "||\n|| Running: ", cmd ,"\n||"
echo '='.repeat(80)
while true:
let charsRead = await p.outputHandle.readInto(wq.lineBuf[0].addr, wq.lineBuf.len)
if charsRead == 0:
break
let charsWritten = stdout.writeBuffer(wq.lineBuf[0].addr, charsRead)
doAssert charsRead == charsWritten
if wq.cmdQueue.len == 0 and wq.outputQueue.len == 0:
return
proc runCommands(commandFile: string, numWorkers: int) =
# State
# -----
let wq = WorkQueue(
sem: AsyncSemaphore.new(numWorkers),
cmdQueue: initDeque[string](),
outputQueue: newAsyncQueue[tuple[cmd: string, p: AsyncProcess]](),
lineBuf: newString(max(80, terminalWidth()))
)
# Parse the file
# --------------
for cmd in lines(commandFile):
if cmd.len == 0: continue
wq.cmdQueue.addLast(cmd)
echo "Found ", wq.cmdQueue.len, " commands to run"
# Run the commands
# ----------------
asyncCheck wq.enqueuePendingCommands()
waitFor wq.flushCommandsOutput()
# Main
# ----------------------------------------------------------------
proc main() =
var commandFile: string
var numWorkers = countProcessors()
if paramCount() == 0:
let exeName = getAppFilename().extractFilename()
echo &"Usage: {exeName} <file-with-commands-1-per-line> <numWorkers: {numWorkers}>"
if paramCount() >= 1:
commandFile = paramStr(1)
if paramCount() == 2:
numWorkers = paramStr(2).parseInt()
if paramCount() > 2:
let exeName = getAppFilename().extractFilename()
echo &"Usage: {exeName} <file-with-commands-1-per-line> <numThreads: {numWorkers}>"
quit 1
runCommands(commandFile, numWorkers)
when isMainModule:
main()

12
helpers/prng_unsafe.nim
View File

@ -158,9 +158,19 @@ func random_unsafe(rng: var RngState, a: var Limbs) =
for i in 0 ..< a.len: for i in 0 ..< a.len:
a[i] = SecretWord(rng.next()) a[i] = SecretWord(rng.next())
template clearExtraBitsOverMSB(a: var BigInt) =
## If we do bit manipulation at the word level,
## for example a 381-bit BigInt stored in a 384-bit buffer
## we need to clear the upper 3-bit
when a.bits != a.limbs.len * WordBitWidth:
const posExtraBits = a.bits - (a.limbs.len-1) * WordBitWidth
const mask = (One shl posExtraBits) - One
a.limbs[^1] = a.limbs[^1] and mask
func random_unsafe(rng: var RngState, a: var BigInt) = func random_unsafe(rng: var RngState, a: var BigInt) =
## Initialize a standalone BigInt ## Initialize a standalone BigInt
rng.random_unsafe(a.limbs) rng.random_unsafe(a.limbs)
a.clearExtraBitsOverMSB()
func random_unsafe(rng: var RngState, a: var FF) = func random_unsafe(rng: var RngState, a: var FF) =
## Initialize a Field element ## Initialize a Field element
@ -193,6 +203,7 @@ func random_highHammingWeight(rng: var RngState, a: var BigInt) =
## with high Hamming weight ## with high Hamming weight
## to have a higher probability of triggering carries ## to have a higher probability of triggering carries
rng.random_highHammingWeight(a.limbs) rng.random_highHammingWeight(a.limbs)
a.clearExtraBitsOverMSB()
func random_highHammingWeight(rng: var RngState, a: var FF) = func random_highHammingWeight(rng: var RngState, a: var FF) =
## Recursively initialize a BigInt (part of a field) or Field element ## Recursively initialize a BigInt (part of a field) or Field element
@ -236,6 +247,7 @@ func random_long01Seq(rng: var RngState, a: var BigInt) =
a.unmarshal(buf, bigEndian) a.unmarshal(buf, bigEndian)
else: else:
a.unmarshal(buf, littleEndian) a.unmarshal(buf, littleEndian)
a.clearExtraBitsOverMSB()
func random_long01Seq(rng: var RngState, a: var Limbs) = func random_long01Seq(rng: var RngState, a: var Limbs) =
## Initialize standalone limbs ## Initialize standalone limbs

51
tests/math/t_bigints_mod_vs_gmp.nim
View File

@ -14,7 +14,9 @@ import
# Internal # Internal
../../constantine/math/io/io_bigints, ../../constantine/math/io/io_bigints,
../../constantine/math/arithmetic, ../../constantine/math/arithmetic,
../../constantine/platforms/primitives ../../constantine/platforms/primitives,
# Test utilities
../../helpers/prng_unsafe
echo "\n------------------------------------------------------\n" echo "\n------------------------------------------------------\n"
# We test up to 1024-bit, more is really slow # We test up to 1024-bit, more is really slow
@ -82,15 +84,11 @@ const # https://gmplib.org/manual/Integer-Import-and-Export.html
GMP_LeastSignificantWordFirst = -1'i32 GMP_LeastSignificantWordFirst = -1'i32
proc main() = proc main() =
var gmpRng: gmp_randstate_t var rng: RngState
gmp_randinit_mt(gmpRng)
# The GMP seed varies between run so that
# test coverage increases as the library gets tested.
# This requires to dump the seed in the console or the function inputs
# to be able to reproduce a bug
let seed = uint32(getTime().toUnix() and (1'i64 shl 32 - 1)) # unixTime mod 2^32 let seed = uint32(getTime().toUnix() and (1'i64 shl 32 - 1)) # unixTime mod 2^32
echo "GMP seed: ", seed rng.seed(seed)
gmp_randseed_ui(gmpRng, seed) echo "\n------------------------------------------------------\n"
echo "test_bigints_mod_vs_gmp xoshiro512** seed: ", seed
var a, m, r: mpz_t var a, m, r: mpz_t
mpz_init(a) mpz_init(a)
@ -101,34 +99,25 @@ proc main() =
# echo "--------------------------------------------------------------------------------" # echo "--------------------------------------------------------------------------------"
echo "Testing: random dividend (" & align($aBits, 4) & "-bit) -- random modulus (" & align($mBits, 4) & "-bit)" echo "Testing: random dividend (" & align($aBits, 4) & "-bit) -- random modulus (" & align($mBits, 4) & "-bit)"
# Generate random value in the range 0 ..< 2^aBits # Build the bigints
mpz_urandomb(a, gmpRng, aBits) let aTest = rng.random_unsafe(BigInt[aBits])
# Generate random modulus and ensure the MSB is set var mTest = rng.random_unsafe(BigInt[mBits])
mpz_urandomb(m, gmpRng, mBits) # Ensure modulus MSB is set
mpz_setbit(m, mBits-1) mTest.setBit(mBits-1)
# discard gmp_printf(" -- %#Zx mod %#Zx\n", a.addr, m.addr)
######################################################### #########################################################
# Conversion buffers # Conversion to GMP
const aLen = (aBits + 7) div 8 const aLen = (aBits + 7) div 8
const mLen = (mBits + 7) div 8 const mLen = (mBits + 7) div 8
var aBuf: array[aLen, byte] var aBuf: array[aLen, byte]
var mBuf: array[mLen, byte] var mBuf: array[mLen, byte]
var aW, mW: csize # Word written by GMP aBuf.marshal(aTest, bigEndian)
mBuf.marshal(mTest, bigEndian)
discard mpz_export(aBuf[0].addr, aW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, a) mpz_import(a, aLen, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, aBuf[0].addr)
discard mpz_export(mBuf[0].addr, mW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, m) mpz_import(m, mLen, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, mBuf[0].addr)
# Since the modulus is using all bits, it's we can test for exact amount copy
doAssert aLen >= aW, "Expected at most " & $aLen & " bytes but wrote " & $aW & " for " & toHex(aBuf) & " (big-endian)"
doAssert mLen == mW, "Expected " & $mLen & " bytes but wrote " & $mW & " for " & toHex(mBuf) & " (big-endian)"
# Build the bigint
let aTest = BigInt[aBits].unmarshal(aBuf.toOpenArray(0, aW-1), bigEndian)
let mTest = BigInt[mBits].unmarshal(mBuf.toOpenArray(0, mW-1), bigEndian)
######################################################### #########################################################
# Modulus # Modulus
@ -139,8 +128,12 @@ proc main() =
######################################################### #########################################################
# Check # Check
{.push warnings: off.} # deprecated csize
var aW, mW, rW: csize # Words written by GMP
{.pop.}
var rGMP: array[mLen, byte] var rGMP: array[mLen, byte]
var rW: csize # Word written by GMP
discard mpz_export(rGMP[0].addr, rW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r) discard mpz_export(rGMP[0].addr, rW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r)
var rConstantine: array[mLen, byte] var rConstantine: array[mLen, byte]

53
tests/math/t_bigints_mul_high_words_vs_gmp.nim
View File

@ -14,7 +14,9 @@ import
# Internal # Internal
../../constantine/math/io/io_bigints, ../../constantine/math/io/io_bigints,
../../constantine/math/arithmetic, ../../constantine/math/arithmetic,
../../constantine/platforms/abstractions ../../constantine/platforms/abstractions,
# Test utilities
../../helpers/prng_unsafe
echo "\n------------------------------------------------------\n" echo "\n------------------------------------------------------\n"
# We test up to 1024-bit, more is really slow # We test up to 1024-bit, more is really slow
@ -51,15 +53,11 @@ const # https://gmplib.org/manual/Integer-Import-and-Export.html
GMP_LeastSignificantWordFirst {.used.} = -1'i32 GMP_LeastSignificantWordFirst {.used.} = -1'i32
proc main() = proc main() =
var gmpRng: gmp_randstate_t var rng: RngState
gmp_randinit_mt(gmpRng)
# The GMP seed varies between run so that
# test coverage increases as the library gets tested.
# This requires to dump the seed in the console or the function inputs
# to be able to reproduce a bug
let seed = uint32(getTime().toUnix() and (1'i64 shl 32 - 1)) # unixTime mod 2^32 let seed = uint32(getTime().toUnix() and (1'i64 shl 32 - 1)) # unixTime mod 2^32
echo "GMP seed: ", seed rng.seed(seed)
gmp_randseed_ui(gmpRng, seed) echo "\n------------------------------------------------------\n"
echo "test_bigints_mod_vs_gmp xoshiro512** seed: ", seed
var r, a, b: mpz_t var r, a, b: mpz_t
mpz_init(r) mpz_init(r)
@ -74,36 +72,23 @@ proc main() =
"-bit) * b (", align($bBits, 4), "-bit) (full mul bits: ", align($(aBits+bBits), 4), "-bit) * b (", align($bBits, 4), "-bit) (full mul bits: ", align($(aBits+bBits), 4),
"), r large enough? ", wordsRequired(rBits) >= wordsRequired(aBits+bBits) - wordsStartIndex "), r large enough? ", wordsRequired(rBits) >= wordsRequired(aBits+bBits) - wordsStartIndex
# Generate random value in the range 0 ..< 2^aBits # Build the bigints
mpz_urandomb(a, gmpRng, aBits) let aTest = rng.random_unsafe(BigInt[aBits])
# Generate random modulus and ensure the MSB is set var bTest = rng.random_unsafe(BigInt[bBits])
mpz_urandomb(b, gmpRng, bBits)
mpz_setbit(r, aBits+bBits)
# discard gmp_printf(" -- %#Zx mod %#Zx\n", a.addr, m.addr)
######################################################### #########################################################
# Conversion buffers # Conversion to GMP
const aLen = (aBits + 7) div 8 const aLen = (aBits + 7) div 8
const bLen = (bBits + 7) div 8 const bLen = (bBits + 7) div 8
var aBuf: array[aLen, byte] var aBuf: array[aLen, byte]
var bBuf: array[bLen, byte] var bBuf: array[bLen, byte]
{.push warnings: off.} # deprecated csize aBuf.marshal(aTest, bigEndian)
var aW, bW: csize # Word written by GMP bBuf.marshal(bTest, bigEndian)
{.pop.}
discard mpz_export(aBuf[0].addr, aW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, a) mpz_import(a, aLen, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, aBuf[0].addr)
discard mpz_export(bBuf[0].addr, bW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, b) mpz_import(b, bLen, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, bBuf[0].addr)
# Since the modulus is using all bits, it's we can test for exact amount copy
doAssert aLen >= aW, "Expected at most " & $aLen & " bytes but wrote " & $aW & " for " & toHex(aBuf) & " (big-endian)"
doAssert bLen >= bW, "Expected at most " & $bLen & " bytes but wrote " & $bW & " for " & toHex(bBuf) & " (big-endian)"
# Build the bigint
let aTest = BigInt[aBits].unmarshal(aBuf.toOpenArray(0, aW-1), bigEndian)
let bTest = BigInt[bBits].unmarshal(bBuf.toOpenArray(0, bW-1), bigEndian)
######################################################### #########################################################
# Multiplication + drop low words # Multiplication + drop low words
@ -124,11 +109,13 @@ proc main() =
######################################################### #########################################################
# Check # Check
{.push warnings: off.} # deprecated csize
var aW, bW, rW: csize # Word written by GMP
{.pop.}
const rLen = numWords * WordBitWidth const rLen = numWords * WordBitWidth
var rGMP: array[rLen, byte] var rGMP: array[rLen, byte]
{.push warnings: off.} # deprecated csize
var rW: csize # Word written by GMP
{.pop.}
discard mpz_export(rGMP[0].addr, rW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r) discard mpz_export(rGMP[0].addr, rW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r)
var rConstantine: array[rLen, byte] var rConstantine: array[rLen, byte]

53
tests/math/t_bigints_mul_vs_gmp.nim
View File

@ -14,7 +14,9 @@ import
# Internal # Internal
../../constantine/math/io/io_bigints, ../../constantine/math/io/io_bigints,
../../constantine/math/arithmetic, ../../constantine/math/arithmetic,
../../constantine/platforms/abstractions ../../constantine/platforms/abstractions,
# Test utilities
../../helpers/prng_unsafe
echo "\n------------------------------------------------------\n" echo "\n------------------------------------------------------\n"
# We test up to 1024-bit, more is really slow # We test up to 1024-bit, more is really slow
@ -48,15 +50,11 @@ const # https://gmplib.org/manual/Integer-Import-and-Export.html
GMP_LeastSignificantWordFirst {.used.} = -1'i32 GMP_LeastSignificantWordFirst {.used.} = -1'i32
proc main() = proc main() =
var gmpRng: gmp_randstate_t var rng: RngState
gmp_randinit_mt(gmpRng)
# The GMP seed varies between run so that
# test coverage increases as the library gets tested.
# This requires to dump the seed in the console or the function inputs
# to be able to reproduce a bug
let seed = uint32(getTime().toUnix() and (1'i64 shl 32 - 1)) # unixTime mod 2^32 let seed = uint32(getTime().toUnix() and (1'i64 shl 32 - 1)) # unixTime mod 2^32
echo "GMP seed: ", seed rng.seed(seed)
gmp_randseed_ui(gmpRng, seed) echo "\n------------------------------------------------------\n"
echo "test_bigints_mod_vs_gmp xoshiro512** seed: ", seed
var r, a, b: mpz_t var r, a, b: mpz_t
mpz_init(r) mpz_init(r)
@ -67,36 +65,23 @@ proc main() =
# echo "--------------------------------------------------------------------------------" # echo "--------------------------------------------------------------------------------"
echo "Testing: random mul r (", align($rBits, 4), "-bit) <- a (", align($aBits, 4), "-bit) * b (", align($bBits, 4), "-bit) (full mul bits: ", align($(aBits+bBits), 4), "), r large enough? ", rBits >= aBits+bBits echo "Testing: random mul r (", align($rBits, 4), "-bit) <- a (", align($aBits, 4), "-bit) * b (", align($bBits, 4), "-bit) (full mul bits: ", align($(aBits+bBits), 4), "), r large enough? ", rBits >= aBits+bBits
# Generate random value in the range 0 ..< 2^aBits # Build the bigints
mpz_urandomb(a, gmpRng, aBits) let aTest = rng.random_unsafe(BigInt[aBits])
# Generate random modulus and ensure the MSB is set var bTest = rng.random_unsafe(BigInt[bBits])
mpz_urandomb(b, gmpRng, bBits)
mpz_setbit(r, aBits+bBits)
# discard gmp_printf(" -- %#Zx mod %#Zx\n", a.addr, m.addr)
######################################################### #########################################################
# Conversion buffers # Conversion to GMP
const aLen = (aBits + 7) div 8 const aLen = (aBits + 7) div 8
const bLen = (bBits + 7) div 8 const bLen = (bBits + 7) div 8
var aBuf: array[aLen, byte] var aBuf: array[aLen, byte]
var bBuf: array[bLen, byte] var bBuf: array[bLen, byte]
{.push warnings: off.} # deprecated csize aBuf.marshal(aTest, bigEndian)
var aW, bW: csize # Word written by GMP bBuf.marshal(bTest, bigEndian)
{.pop.}
discard mpz_export(aBuf[0].addr, aW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, a) mpz_import(a, aLen, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, aBuf[0].addr)
discard mpz_export(bBuf[0].addr, bW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, b) mpz_import(b, bLen, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, bBuf[0].addr)
# Since the modulus is using all bits, it's we can test for exact amount copy
doAssert aLen >= aW, "Expected at most " & $aLen & " bytes but wrote " & $aW & " for " & toHex(aBuf) & " (big-endian)"
doAssert bLen >= bW, "Expected at most " & $bLen & " bytes but wrote " & $bW & " for " & toHex(bBuf) & " (big-endian)"
# Build the bigint
let aTest = BigInt[aBits].unmarshal(aBuf.toOpenArray(0, aW-1), bigEndian)
let bTest = BigInt[bBits].unmarshal(bBuf.toOpenArray(0, bW-1), bigEndian)
######################################################### #########################################################
# Multiplication # Multiplication
@ -114,11 +99,13 @@ proc main() =
######################################################### #########################################################
# Check # Check
{.push warnings: off.} # deprecated csize
var aW, bW, rW: csize # Word written by GMP
{.pop.}
const rLen = numWords * WordBitWidth const rLen = numWords * WordBitWidth
var rGMP: array[rLen, byte] var rGMP: array[rLen, byte]
{.push warnings: off.} # deprecated csize
var rW: csize # Word written by GMP
{.pop.}
discard mpz_export(rGMP[0].addr, rW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r) discard mpz_export(rGMP[0].addr, rW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r)
var rConstantine: array[rLen, byte] var rConstantine: array[rLen, byte]

79
tests/math/t_finite_fields_vs_gmp.nim
View File

@ -15,7 +15,9 @@ import
../../constantine/platforms/abstractions, ../../constantine/platforms/abstractions,
../../constantine/math/io/[io_bigints, io_fields], ../../constantine/math/io/[io_bigints, io_fields],
../../constantine/math/arithmetic, ../../constantine/math/arithmetic,
../../constantine/math/config/curves ../../constantine/math/config/curves,
# Test utilities
../../helpers/prng_unsafe
echo "\n------------------------------------------------------\n" echo "\n------------------------------------------------------\n"
@ -45,36 +47,32 @@ const # https://gmplib.org/manual/Integer-Import-and-Export.html
# Factor common things in proc to avoid generating 100k+ lines of C code # Factor common things in proc to avoid generating 100k+ lines of C code
proc binary_prologue[C: static Curve, N: static int]( proc binary_prologue[C: static Curve, N: static int](
gmpRng: var gmp_randstate_t, rng: var RngState,
a, b, p: var mpz_t, a, b, p: var mpz_t,
aTest, bTest: var Fp[C], aTest, bTest: var Fp[C],
aBuf, bBuf: var array[N, byte]) = aBuf, bBuf: var array[N, byte]) =
const bits = C.getCurveBitwidth()
# Generate random value in the range 0 ..< 2^(bits-1) # Build the field elements
mpz_urandomb(a, gmpRng, uint bits) aTest = rng.random_unsafe(Fp[C])
mpz_urandomb(b, gmpRng, uint bits) bTest = rng.random_unsafe(Fp[C])
# Set modulus to curve modulus # Set modulus to curve modulus
let err = mpz_set_str(p, Curve(C).Mod.toHex(), 0) let err = mpz_set_str(p, Curve(C).Mod.toHex(), 0)
doAssert err == 0, "Error on prime for curve " & $Curve(C) doAssert err == 0, "Error on prime for curve " & $Curve(C)
######################################################### #########################################################
# Conversion buffers # Conversion to GMP
const len = (bits + 7) div 8 const aLen = (C.getCurveBitwidth() + 7) div 8
const bLen = (C.getCurveBitwidth() + 7) div 8
static: doAssert N >= len var aBuf: array[aLen, byte]
var bBuf: array[bLen, byte]
var aW, bW: csize # Word written by GMP aBuf.marshal(aTest, bigEndian)
discard mpz_export(aBuf[0].addr, aW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, a) bBuf.marshal(bTest, bigEndian)
discard mpz_export(bBuf[0].addr, bW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, b)
# Since the modulus is using all bits, it's we can test for exact amount copy mpz_import(a, aLen, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, aBuf[0].addr)
doAssert len >= aW, "Expected at most " & $len & " bytes but wrote " & $aW & " for " & toHex(aBuf) & " (big-endian)" mpz_import(b, bLen, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, bBuf[0].addr)
doAssert len >= bW, "Expected at most " & $len & " bytes but wrote " & $bW & " for " & toHex(bBuf) & " (big-endian)"
# Build the bigint
aTest = Fp[C].fromBig BigInt[bits].unmarshal(aBuf.toOpenArray(0, aW-1), bigEndian)
bTest = Fp[C].fromBig BigInt[bits].unmarshal(bBuf.toOpenArray(0, bW-1), bigEndian)
proc binary_epilogue[C: static Curve, N: static int]( proc binary_epilogue[C: static Curve, N: static int](
r, a, b: mpz_t, r, a, b: mpz_t,
@ -85,8 +83,12 @@ proc binary_epilogue[C: static Curve, N: static int](
######################################################### #########################################################
# Check # Check
{.push warnings: off.} # deprecated csize
var aW, bW, rW: csize # Word written by GMP
{.pop.}
var rGMP: array[N, byte] var rGMP: array[N, byte]
var rW: csize # Word written by GMP
discard mpz_export(rGMP[0].addr, rW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r) discard mpz_export(rGMP[0].addr, rW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, r)
var rConstantine: array[N, byte] var rConstantine: array[N, byte]
@ -95,7 +97,6 @@ proc binary_epilogue[C: static Curve, N: static int](
# Note: in bigEndian, GMP aligns left while constantine aligns right # Note: in bigEndian, GMP aligns left while constantine aligns right
doAssert rGMP.toOpenArray(0, rW-1) == rConstantine.toOpenArray(N-rW, N-1), block: doAssert rGMP.toOpenArray(0, rW-1) == rConstantine.toOpenArray(N-rW, N-1), block:
# Reexport as bigEndian for debugging # Reexport as bigEndian for debugging
var aW, bW: csize
discard mpz_export(aBuf[0].unsafeAddr, aW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, a) discard mpz_export(aBuf[0].unsafeAddr, aW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, a)
discard mpz_export(bBuf[0].unsafeAddr, bW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, b) discard mpz_export(bBuf[0].unsafeAddr, bW.addr, GMP_MostSignificantWordFirst, 1, GMP_WordNativeEndian, 0, b)
"\nModular " & operation & " on curve " & $C & " with operands\n" & "\nModular " & operation & " on curve " & $C & " with operands\n" &
@ -112,7 +113,7 @@ proc binary_epilogue[C: static Curve, N: static int](
# #
# ############################################################ # ############################################################
proc addTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curve) = proc addTests(rng: var RngState, a, b, p, r: var mpz_t, C: static Curve) =
# echo "Testing: random modular addition on ", $C # echo "Testing: random modular addition on ", $C
const const
@ -121,7 +122,7 @@ proc addTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curv
var var
aTest, bTest{.noInit.}: Fp[C] aTest, bTest{.noInit.}: Fp[C]
aBuf, bBuf: array[bufLen, byte] aBuf, bBuf: array[bufLen, byte]
binary_prologue(gmpRng, a, b, p, aTest, bTest, aBuf, bBuf) binary_prologue(rng, a, b, p, aTest, bTest, aBuf, bBuf)
mpz_add(r, a, b) mpz_add(r, a, b)
mpz_mod(r, r, p) mpz_mod(r, r, p)
@ -135,7 +136,7 @@ proc addTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curv
binary_epilogue(r, a, b, rTest, aBuf, bBuf, "Addition (with result)") binary_epilogue(r, a, b, rTest, aBuf, bBuf, "Addition (with result)")
binary_epilogue(r, a, b, r2Test, aBuf, bBuf, "Addition (in-place)") binary_epilogue(r, a, b, r2Test, aBuf, bBuf, "Addition (in-place)")
proc subTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curve) = proc subTests(rng: var RngState, a, b, p, r: var mpz_t, C: static Curve) =
# echo "Testing: random modular substraction on ", $C # echo "Testing: random modular substraction on ", $C
const const
@ -144,7 +145,7 @@ proc subTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curv
var var
aTest, bTest{.noInit.}: Fp[C] aTest, bTest{.noInit.}: Fp[C]
aBuf, bBuf: array[bufLen, byte] aBuf, bBuf: array[bufLen, byte]
binary_prologue(gmpRng, a, b, p, aTest, bTest, aBuf, bBuf) binary_prologue(rng, a, b, p, aTest, bTest, aBuf, bBuf)
mpz_sub(r, a, b) mpz_sub(r, a, b)
mpz_mod(r, r, p) mpz_mod(r, r, p)
@ -163,7 +164,7 @@ proc subTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curv
binary_epilogue(r, a, b, r2Test, aBuf, bBuf, "Substraction (in-place)") binary_epilogue(r, a, b, r2Test, aBuf, bBuf, "Substraction (in-place)")
binary_epilogue(r, a, b, r3Test, aBuf, bBuf, "Substraction (result aliasing)") binary_epilogue(r, a, b, r3Test, aBuf, bBuf, "Substraction (result aliasing)")
proc mulTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curve) = proc mulTests(rng: var RngState, a, b, p, r: var mpz_t, C: static Curve) =
# echo "Testing: random modular multiplication on ", $C # echo "Testing: random modular multiplication on ", $C
const const
@ -172,7 +173,7 @@ proc mulTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curv
var var
aTest, bTest{.noInit.}: Fp[C] aTest, bTest{.noInit.}: Fp[C]
aBuf, bBuf: array[bufLen, byte] aBuf, bBuf: array[bufLen, byte]
binary_prologue(gmpRng, a, b, p, aTest, bTest, aBuf, bBuf) binary_prologue(rng, a, b, p, aTest, bTest, aBuf, bBuf)
mpz_mul(r, a, b) mpz_mul(r, a, b)
mpz_mod(r, r, p) mpz_mod(r, r, p)
@ -186,7 +187,7 @@ proc mulTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curv
binary_epilogue(r, a, b, rTest, aBuf, bBuf, "Multiplication (with result)") binary_epilogue(r, a, b, rTest, aBuf, bBuf, "Multiplication (with result)")
binary_epilogue(r, a, b, r2Test, aBuf, bBuf, "Multiplication (in-place)") binary_epilogue(r, a, b, r2Test, aBuf, bBuf, "Multiplication (in-place)")
proc invTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curve) = proc invTests(rng: var RngState, a, b, p, r: var mpz_t, C: static Curve) =
# We use the binary prologue epilogue but the "b" parameter is actual unused # We use the binary prologue epilogue but the "b" parameter is actual unused
# echo "Testing: random modular inversion on ", $C # echo "Testing: random modular inversion on ", $C
@ -196,7 +197,7 @@ proc invTests(gmpRng: var gmp_randstate_t, a, b, p, r: var mpz_t, C: static Curv
var var
aTest, bTest{.noInit.}: Fp[C] aTest, bTest{.noInit.}: Fp[C]
aBuf, bBuf: array[bufLen, byte] aBuf, bBuf: array[bufLen, byte]
binary_prologue(gmpRng, a, b, p, aTest, bTest, aBuf, bBuf) binary_prologue(rng, a, b, p, aTest, bTest, aBuf, bBuf)
let exist = mpz_invert(r, a, p) let exist = mpz_invert(r, a, p)
doAssert exist != 0 doAssert exist != 0
@ -227,15 +228,11 @@ macro randomTests(numTests: static int, curveSym, body: untyped): untyped =
`body` `body`
template testSetup {.dirty.} = template testSetup {.dirty.} =
var gmpRng: gmp_randstate_t var rng: RngState
gmp_randinit_mt(gmpRng)
# The GMP seed varies between run so that
# test coverage increases as the library gets tested.
# This requires to dump the seed in the console or the function inputs
# to be able to reproduce a bug
let seed = uint32(getTime().toUnix() and (1'i64 shl 32 - 1)) # unixTime mod 2^32 let seed = uint32(getTime().toUnix() and (1'i64 shl 32 - 1)) # unixTime mod 2^32
echo "GMP seed: ", seed rng.seed(seed)
gmp_randseed_ui(gmpRng, seed) echo "\n------------------------------------------------------\n"
echo "test_finite_fields_vs_gmp** seed: ", seed
var a, b, p, r: mpz_t var a, b, p, r: mpz_t
mpz_init(a) mpz_init(a)
@ -247,25 +244,25 @@ proc mainMul() =
testSetup() testSetup()
echo "Testing modular multiplications vs GMP" echo "Testing modular multiplications vs GMP"
randomTests(24, curve): randomTests(24, curve):
mulTests(gmpRng, a, b, p, r, curve) mulTests(rng, a, b, p, r, curve)
proc mainAdd() = proc mainAdd() =
testSetup() testSetup()
echo "Testing modular additions vs GMP" echo "Testing modular additions vs GMP"
randomTests(24, curve): randomTests(24, curve):
addTests(gmpRng, a, b, p, r, curve) addTests(rng, a, b, p, r, curve)
proc mainSub() = proc mainSub() =
testSetup() testSetup()
echo "Testing modular substractions vs GMP" echo "Testing modular substractions vs GMP"
randomTests(24, curve): randomTests(24, curve):
subTests(gmpRng, a, b, p, r, curve) subTests(rng, a, b, p, r, curve)
proc mainInv() = proc mainInv() =
testSetup() testSetup()
echo "Testing modular inversions vs GMP" echo "Testing modular inversions vs GMP"
randomTests(24, curve): randomTests(24, curve):
invTests(gmpRng, a, b, p, r, curve) invTests(rng, a, b, p, r, curve)
mainMul() mainMul()

4
tests/t_hash_sha256_vs_openssl.nim
View File

@ -43,8 +43,8 @@ proc SHA256_OpenSSL[T: byte|char](
# -------------------------------------------------------------------- # --------------------------------------------------------------------
echo "\n------------------------------------------------------\n" echo "\n------------------------------------------------------\n"
const SmallSizeIters = 128 const SmallSizeIters = 64
const LargeSizeIters = 10 const LargeSizeIters = 1
proc sanityABC = proc sanityABC =
var bufCt: array[32, byte] var bufCt: array[32, byte]