Merge #864: Add support for Cirrus CI

cc2a5451dc ci: Refactor Nix shell files (Jonas Nick)
2480e55c8f ci: Remove support for Travis CI (Tim Ruffing)
2b359f1c1d ci: Enable simple cache for brewing valgrind on macOS (Tim Ruffing)
8c02e465c5 ci: Add support for Cirrus CI (Tim Ruffing)

Pull request description:

ACKs for top commit:
  sipa:
    ACK cc2a5451dc. Tested by introducing bugs: #883, #884, #885, #886, #887.
  jonasnick:
    ACK cc2a5451dc

Tree-SHA512: c9e8a891c9bda48b3fc307c2a85d2e4aa180531d084edd778d41c034769661627538ab397efac3abfc1a71c2f0730a45350dd212d499fe475c90a2a1b3c61ac8

.cirrus.yml

@@ -0,0 +1,191 @@
+env:
+  WIDEMUL: auto
+  BIGNUM: auto
+  STATICPRECOMPUTATION: yes
+  ECMULTGENPRECISION: auto
+  ASM: no
+  BUILD: check
+  WITH_VALGRIND: yes
+  RUN_VALGRIND: no
+  EXTRAFLAGS:
+  HOST:
+  ECDH: no
+  RECOVERY: no
+  SCHNORRSIG: no
+  EXPERIMENTAL: no
+  CTIMETEST: yes
+  BENCH: yes
+  ITERS: 2
+  # We only need the top commit
+  CIRRUS_CLONE_DEPTH: 1
+
+cat_logs_snippet: &CAT_LOGS
+  always:
+    test_logs_script:
+      - cat tests.log || true
+      - cat exhaustive_tests.log || true
+      - cat valgrind_ctime_test.log || true
+      - cat bench.log || true
+  on_failure:
+    debug_output_script:
+      - cat config.log || true
+      - cat test_env.log || true
+      - env
+
+task:
+  name: "x86_64: Linux (Alpine Linux, Nix Shell)"
+  container:
+    dockerfile: ci/linux-nixos.Dockerfile
+    # Reduce number of CPUs to be able to do more builds in parallel.
+    cpu: 1
+    # More than enough for our scripts.
+    memory: 1G
+  matrix: &ENV_MATRIX
+    - env: {WIDEMUL:  int64,  RECOVERY: yes}
+    - env: {WIDEMUL:  int64,                 ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes}
+    - env: {WIDEMUL: int128}
+    - env: {WIDEMUL: int128,  RECOVERY: yes,            EXPERIMENTAL: yes, SCHNORRSIG: yes}
+    - env: {WIDEMUL: int128,                 ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes}
+    - env: {WIDEMUL: int128,  ASM: x86_64}
+    - env: {BIGNUM: no}
+    - env: {BIGNUM: no,       RECOVERY: yes,            EXPERIMENTAL: yes, SCHNORRSIG: yes}
+    - env: {BIGNUM: no,       STATICPRECOMPUTATION: no}
+    - env: {BUILD: distcheck, WITH_VALGRIND: no, CTIMETEST: no, BENCH: no}
+    - env: {CPPFLAGS: -DDETERMINISTIC}
+    - env: {CFLAGS: -O0, CTIMETEST: no}
+    - env:
+        CFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
+        LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
+        UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+        BIGNUM: no
+        ASM: x86_64
+        ECDH: yes
+        RECOVERY: yes
+        EXPERIMENTAL: yes
+        SCHNORRSIG: yes
+        CTIMETEST: no
+    - env: { ECMULTGENPRECISION: 2 }
+    - env: { ECMULTGENPRECISION: 8 }
+    - env:
+        RUN_VALGRIND: yes
+        BIGNUM: no
+        ASM: x86_64
+        ECDH: yes
+        RECOVERY: yes
+        EXPERIMENTAL: yes
+        SCHNORRSIG: yes
+        EXTRAFLAGS: "--disable-openssl-tests"
+        BUILD:
+  matrix:
+    - env:
+        CC: gcc
+    - env:
+        CC: clang
+  test_script:
+    - nix-shell ci/shell.nix --run ./ci/cirrus.sh
+  << : *CAT_LOGS
+
+task:
+  name: "i686: Linux (Alpine Linux, Nix Shell)"
+  container:
+    dockerfile: ci/linux-nixos.Dockerfile
+    cpu: 1
+    memory: 1G
+  env:
+    HOST: i686-linux-gnu
+    ECDH: yes
+    RECOVERY: yes
+    EXPERIMENTAL: yes
+    SCHNORRSIG: yes
+  matrix:
+    - env:
+        CC: gcc
+    - env:
+        CC: clang
+  matrix:
+    - env:
+        BIGNUM: gmp
+    - env:
+        BIGNUM: no
+  test_script:
+    - nix-shell ci/shell-i686.nix --run ./ci/cirrus.sh
+  << : *CAT_LOGS
+
+task:
+  name: "x86_64: macOS Catalina"
+  macos_instance:
+    image: catalina-base
+  env:
+    HOMEBREW_NO_AUTO_UPDATE: 1
+    HOMEBREW_NO_INSTALL_CLEANUP: 1
+  matrix:
+    << : *ENV_MATRIX
+  matrix:
+    - env:
+        CC: gcc-9
+    - env:
+        CC: clang
+  # Update Command Line Tools
+  # Uncomment this if the Command Line Tools on the CirrusCI macOS image are too old to brew valgrind.
+  # See https://apple.stackexchange.com/a/195963 for the implementation.
+  ## update_clt_script:
+  ##   - system_profiler SPSoftwareDataType
+  ##   - touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress
+  ##   - |-
+  ##     PROD=$(softwareupdate -l | grep "*.*Command Line" | tail -n 1 | awk -F"*" '{print $2}' | sed -e 's/^ *//' | sed 's/Label: //g' | tr -d '\n')
+  ##   # For debugging
+  ##   - softwareupdate -l && echo "PROD: $PROD"
+  ##   - softwareupdate -i "$PROD" --verbose
+  ##   - rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress
+  ##
+  brew_valgrind_pre_script:
+    - brew config
+    - brew tap --shallow LouisBrunner/valgrind
+    # Fetch valgrind source but don't build it yet.
+    - brew fetch --HEAD LouisBrunner/valgrind/valgrind
+  brew_valgrind_cache:
+    # This is $(brew --cellar valgrind) but command substition does not work here.
+    folder: /usr/local/Cellar/valgrind
+    # Rebuild cache if ...
+    fingerprint_script:
+      # ... macOS version changes:
+      - sw_vers
+      # ... brew changes:
+      - brew config
+      # ... valgrind changes:
+      - git -C "$(brew --cache)/valgrind--git" rev-parse HEAD
+    populate_script:
+      # If there's no hit in the cache, build and install valgrind.
+      - brew install --HEAD LouisBrunner/valgrind/valgrind
+  brew_valgrind_post_script:
+    # If we have restored valgrind from the cache, tell brew to create symlink to the PATH.
+    # If we haven't restored from cached (and just run brew install), this is a no-op.
+    - brew link valgrind
+  brew_script:
+    - brew install automake libtool gmp gcc@9
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
+
+task:
+  name: "s390x (big-endian): Linux (Debian QEMU)"
+  container:
+    dockerfile: ci/linux-debian-s390-qemu.Dockerfile
+    cpu: 1
+    memory: 1G
+  env:
+    QEMU_CMD: qemu-s390x
+    HOST: s390x-linux-gnu
+    BUILD:
+    WITH_VALGRIND: no
+    BIGNUM: no
+    ECDH: yes
+    RECOVERY: yes
+    EXPERIMENTAL: yes
+    SCHNORRSIG: yes
+    CTIMETEST: no
+  test_script:
+    # https://sourceware.org/bugzilla/show_bug.cgi?id=27008
+    - rm /etc/ld.so.cache
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS

.travis.yml

@@ -1,109 +0,0 @@
-language: c
-os:
-  - linux
-  - osx
-
-dist: bionic
-# Valgrind currently supports upto macOS 10.13, the latest xcode of that version is 10.1
-osx_image: xcode10.1
-addons:
-  apt:
-    packages:
-      - libgmp-dev
-      - valgrind
-      - libtool-bin
-compiler:
-  - clang
-  - gcc
-env:
-  global:
-    - WIDEMUL=auto  BIGNUM=auto  STATICPRECOMPUTATION=yes  ECMULTGENPRECISION=auto  ASM=no  BUILD=check  WITH_VALGRIND=yes RUN_VALGRIND=no EXTRAFLAGS=  HOST=  ECDH=no  RECOVERY=no SCHNORRSIG=no EXPERIMENTAL=no CTIMETEST=yes BENCH=yes ITERS=2
-  matrix:
-    - WIDEMUL=int64   RECOVERY=yes
-    - WIDEMUL=int64   ECDH=yes  EXPERIMENTAL=yes SCHNORRSIG=yes
-    - WIDEMUL=int128
-    - WIDEMUL=int128  RECOVERY=yes EXPERIMENTAL=yes SCHNORRSIG=yes
-    - WIDEMUL=int128  ECDH=yes EXPERIMENTAL=yes SCHNORRSIG=yes
-    - WIDEMUL=int128                    ASM=x86_64
-    - BIGNUM=no
-    - BIGNUM=no       RECOVERY=yes EXPERIMENTAL=yes SCHNORRSIG=yes
-    - BIGNUM=no       STATICPRECOMPUTATION=no
-    - BUILD=distcheck WITH_VALGRIND=no CTIMETEST=no BENCH=no
-    - CPPFLAGS=-DDETERMINISTIC
-    - CFLAGS=-O0 CTIMETEST=no
-    - CFLAGS="-fsanitize=undefined -fno-omit-frame-pointer" LDFLAGS="-fsanitize=undefined -fno-omit-frame-pointer" UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1" BIGNUM=no ASM=x86_64 ECDH=yes RECOVERY=yes EXPERIMENTAL=yes SCHNORRSIG=yes CTIMETEST=no
-    - ECMULTGENPRECISION=2
-    - ECMULTGENPRECISION=8
-    - RUN_VALGRIND=yes BIGNUM=no ASM=x86_64 ECDH=yes  RECOVERY=yes EXPERIMENTAL=yes SCHNORRSIG=yes EXTRAFLAGS="--disable-openssl-tests" BUILD=
-matrix:
-  fast_finish: true
-  include:
-    - compiler: clang
-      os: linux
-      env: HOST=i686-linux-gnu
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - libgmp-dev:i386
-            - valgrind
-            - libtool-bin
-            - libc6-dbg:i386
-    - compiler: clang
-      env: HOST=i686-linux-gnu
-      os: linux
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - valgrind
-            - libtool-bin
-            - libc6-dbg:i386
-    - compiler: gcc
-      env: HOST=i686-linux-gnu
-      os: linux
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - valgrind
-            - libtool-bin
-            - libc6-dbg:i386
-    - compiler: gcc
-      os: linux
-      env: HOST=i686-linux-gnu
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - libgmp-dev:i386
-            - valgrind
-            - libtool-bin
-            - libc6-dbg:i386
-    # S390x build (big endian system)
-    - compiler: gcc
-      env: HOST=s390x-unknown-linux-gnu ECDH=yes RECOVERY=yes EXPERIMENTAL=yes SCHNORRSIG=yes CTIMETEST=
-      arch: s390x
-
-# We use this to install macOS dependencies instead of the built in `homebrew` plugin,
-# because in xcode earlier than 11 they have a bug requiring updating the system which overall takes ~8 minutes.
-# https://travis-ci.community/t/macos-build-fails-because-of-homebrew-bundle-unknown-command/7296
-before_install:
- - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install gmp valgrind gcc@9; fi
-
-before_script: ./autogen.sh
-
-# travis auto terminates jobs that go for 10 minutes without printing to stdout, but travis_wait doesn't work well with forking programs like valgrind (https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received https://github.com/bitcoin-core/secp256k1/pull/750#issuecomment-623476860)
-script:
-  - function keep_alive() { while true; do echo -en "\a"; sleep 60; done }
-  - keep_alive &
-  - ./contrib/travis.sh
-  - kill %keep_alive
-
-after_script:
-    - cat ./tests.log
-    - cat ./exhaustive_tests.log
-    - cat ./valgrind_ctime_test.log
-    - cat ./bench.log
-    - $CC --version
-    - valgrind --version

README.md

@@ -1,7 +1,7 @@
 libsecp256k1
 ============
 
-[![Build Status](https://travis-ci.org/bitcoin-core/secp256k1.svg?branch=master)](https://travis-ci.org/bitcoin-core/secp256k1)
+[![Build Status](https://api.cirrus-ci.com/github/bitcoin-core/secp256k1.svg?branch=master)](https://cirrus-ci.com/github/bitcoin-core/secp256k1)
 
 Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1.
 

ci/cirrus.sh

@@ -3,14 +3,18 @@
 set -e
 set -x
 
-if [ "$HOST" = "i686-linux-gnu" ]
+export LC_ALL=C
-then
+
-    export CC="$CC -m32"
+env >> test_env.log
-fi
+
-if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$TRAVIS_COMPILER" = "gcc" ]
+$CC -v || true
-then
+valgrind --version || true
-    export CC="gcc-9"
+
-fi
+./autogen.sh
+
+# Nix doesn't store GNU file in /usr/bin, see https://lists.gnu.org/archive/html/bug-libtool/2015-09/msg00000.html .
+# The -i'' is necessary for macOS portability, see https://stackoverflow.com/a/4247319 .
+sed -i'' -e 's@/usr/bin/file@$(which file)@g' configure
 
 ./configure \
     --enable-experimental="$EXPERIMENTAL" \
@@ -25,6 +29,7 @@ if [ -n "$BUILD" ]
 then
     make -j2 "$BUILD"
 fi
+
 if [ "$RUN_VALGRIND" = "yes" ]
 then
     make -j2
@@ -32,16 +37,27 @@ then
     valgrind --error-exitcode=42 ./tests 16
     valgrind --error-exitcode=42 ./exhaustive_tests
 fi
+
+if [ -n "$QEMU_CMD" ]
+then
+    make -j2
+    $QEMU_CMD ./tests 16
+    $QEMU_CMD ./exhaustive_tests
+fi
+
 if [ "$BENCH" = "yes" ]
 then
+    # Using the local `libtool` because on macOS the system's libtool has nothing to do with GNU libtool
+    EXEC='./libtool --mode=execute'
+    if [ -n "$QEMU_CMD" ]
+    then
+       EXEC="$EXEC $QEMU_CMD"
+    fi
     if [ "$RUN_VALGRIND" = "yes" ]
     then
-        # Using the local `libtool` because on macOS the system's libtool has nothing to do with GNU libtool
+        EXEC="$EXEC valgrind --error-exitcode=42"
-        EXEC='./libtool --mode=execute valgrind --error-exitcode=42'
-    else
-        EXEC=
     fi
-    # This limits the iterations in the benchmarks below to ITER(set in .travis.yml) iterations.
+    # This limits the iterations in the benchmarks below to ITER iterations.
     export SECP256K1_BENCH_ITERS="$ITERS"
     {
         $EXEC ./bench_ecmult

ci/linux-debian-s390-qemu.Dockerfile

@@ -0,0 +1,6 @@
+FROM debian
+
+RUN dpkg --add-architecture s390x
+RUN apt-get update
+RUN apt-get install --no-install-recommends --no-upgrade -y make automake libtool
+RUN apt-get install --no-install-recommends --no-upgrade -y gcc-s390x-linux-gnu libc6-dev-s390x-cross qemu-user libc6:s390x

ci/linux-nixos.Dockerfile

@@ -0,0 +1,12 @@
+FROM nixos/nix
+
+COPY ci/shell.nix /tmp
+COPY ci/shell-i686.nix /tmp
+
+RUN nix-channel --remove nixpkgs
+RUN nix-channel --add https://nixos.org/channels/nixos-20.09 nixpkgs
+RUN nix-channel --update
+
+# Run dummy command "true" in the nix-shell just to get the packages prepared.
+RUN nix-shell /tmp/shell.nix --command true
+RUN nix-shell /tmp/shell-i686.nix --command true

ci/mkshell.nix

@@ -0,0 +1,12 @@
+{ pkgs }:
+
+with pkgs;
+
+mkShell {
+   buildInputs = [
+       bash file pkgconfig autoconf automake libtool gmp valgrind clang gcc
+   ];
+   shellHook = ''
+       echo Running nix-shell with nixpkgs version: $(nix eval --raw nixpkgs.lib.version)
+   '';
+}

ci/shell-i686.nix

@@ -0,0 +1,4 @@
+let
+  pkgs = (import <nixpkgs> {}).pkgsi686Linux;
+in
+import ./mkshell.nix { inherit pkgs; }

ci/shell.nix

@@ -0,0 +1,4 @@
+let
+  pkgs = (import <nixpkgs> {});
+in
+import ./mkshell.nix { inherit pkgs; }