Nightly tests (#240)

* Nightly notifications

* Publish discovered regression cases as a PR

* Nightly notifications

* Increase nightly tests runtime

* Extract discordnotify and params

* Groovy script fixes

* Remove unused iterations variable

* Cleanup jenkinsfiles and add verbosity

ci/Jenkinsfile.nightly.fuzzy

@@ -0,0 +1,61 @@
+pipeline {
+  agent {
+    dockerfile {
+      label 'linux'
+      dir 'ci'
+    }
+  }
+
+  parameters {
+    string(
+      name: 'PROPTEST_CASES',
+      description: 'Test cases to be executed',
+      defaultValue: params.PROPTEST_CASES ?: '500000'
+    )
+  }
+
+  environment {
+    /* Avoid cache poisoning by other jobs. */
+    GOCACHE = "${env.WORKSPACE_TMP}/go-build"
+    GOPATH  = "${env.WORKSPACE_TMP}/go"
+    PROPTEST_CASES = "${params.PROTEST_CASES}"
+  }
+
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(
+      numToKeepStr: '20',
+      daysToKeepStr: '30',
+    ))
+  }
+
+  stages {
+    stage('Fuzztest') {
+      steps {
+        sh 'cargo test --test fuzz_test'
+      }
+      post {
+        failure {
+          archiveArtifacts artifacts: '**/*.proptest-regressions'
+        }
+      }
+    }
+  }
+
+  post {
+    failure {
+      script {
+        def discord = load "${WORKSPACE}/ci/discord.groovy"
+        discord.sendMessage(header: 'Nightly Fuzztest Failed. Regression files archived as job artifacts')
+      }
+    }
+    success {
+      script {
+        def discord = load "${WORKSPACE}/ci/discord.groovy"
+        discord.sendMessage(header: 'Nightly Fuzztest Passed')
+      }
+    }
+    cleanup { cleanWs() }
+  }
+}
+

ci/Jenkinsfile.nightly.integration

@@ -0,0 +1,75 @@
+pipeline {
+  agent {
+    dockerfile {
+      label 'linux'
+      dir 'ci'
+    }
+  }
+
+  parameters {
+    string(
+      name: 'ITERATIONS',
+      description: 'Number of repeated integration test runs',
+      defaultValue: params.ITERATIONS ?: '1000'
+    )
+  }
+
+  environment {
+    /* Avoid cache poisoning by other jobs. */
+    GOCACHE = "${env.WORKSPACE_TMP}/go-build"
+    GOPATH  = "${env.WORKSPACE_TMP}/go"
+    RUST_BACKTRACE = 1
+  }
+
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(
+      numToKeepStr: '20',
+      daysToKeepStr: '30',
+    ))
+  }
+
+  stages {
+    stage('Build') {
+      steps {
+        /* Node binary is required for integration tests */
+        sh 'cargo build'
+      }
+    }
+
+    stage('Integration tests') {
+      steps {
+        script {
+          int iterations = params.ITERATIONS.toInteger()
+
+          for (int i = 0; i < iterations; i++) {
+            echo "Running iteration ${i + 1} of ${iterations}"
+
+            def result = sh(script: 'cargo test ten_nodes_happy', returnStatus: true)
+
+            if (result != 0) {
+              error("Test failed on iteration ${i + 1}")
+              break
+            }
+          }
+        }
+      }
+    }
+  }
+  post {
+    failure {
+      script {
+        def discord = load "${WORKSPACE}/ci/discord.groovy"
+        discord.sendMessage(header: 'Nightly Integration Tests Failed')
+      }
+    }
+    success {
+      script {
+        def discord = load "${WORKSPACE}/ci/discord.groovy"
+        discord.sendMessage(header: 'Nightly Integration Tests Passed')
+      }
+    }
+    cleanup { cleanWs() }
+  }
+}
+

ci/discord.groovy

@@ -0,0 +1,32 @@
+def sendMessage(Map args=[:]) {
+  def opts = [
+    header: args.header ?: 'Build succeeded',
+    title:  args.title  ?: "${env.JOB_NAME}#${env.BUILD_NUMBER}",
+    cred:   args.cred   ?: 'nomos-node-discord-commits-webhook',
+  ]
+  def repo = [
+    url: GIT_URL.minus('.git'),
+    branch: GIT_BRANCH.minus('origin/'),
+    commit: GIT_COMMIT.take(8),
+  ]
+  withCredentials([
+    string(
+      credentialsId: opts.cred,
+      variable: 'DISCORD_WEBHOOK',
+    ),
+  ]) {
+    discordSend(
+      link: env.BUILD_URL,
+      result: currentBuild.currentResult,
+      webhookURL: env.DISCORD_WEBHOOK,
+      title: opts.title,
+      description: """
+      ${opts.header}
+      Branch: [`${repo.branch}`](${repo.url}/commits/${repo.branch})
+      Commit: [`${repo.commit}`](${repo.url}/commit/${repo.commit})
+      """,
+    )
+  }
+}
+
+return this