Add `canary-test` make target to test beta fleet mailservers. Closes #1274

.travis.yml

@@ -12,6 +12,9 @@ jobs:
     before_script: make dep-ensure
     script:
       - make lint
+  - stage: Test with deployed mailservers
+    script:
+      - make canary-test
   - stage: Test unit and integration
     script:
       - make test-unit

Makefile

@@ -283,6 +283,9 @@ test-e2e: ##@tests Run e2e tests
 test-e2e-race: gotest_extraflags=-race
 test-e2e-race: test-e2e ##@tests Run e2e tests with -race flag
 
+canary-test: node-canary
+	_assets/scripts/canary_test_mailservers.sh ./config/cli/fleet-eth.beta.json
+
 lint-install:
 	@# The following installs a specific version of golangci-lint, which is appropriate for a CI server to avoid different results from build to build
 	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b $(GOPATH)/bin v1.10.2
@@ -291,7 +294,7 @@ lint:
 	@echo "lint"
 	@golangci-lint run ./...
 
-ci: lint mock dep-ensure test-unit test-e2e ##@tests Run all linters and tests at once
+ci: lint mock dep-ensure canary-test test-unit test-e2e ##@tests Run all linters and tests at once
 
 clean: ##@other Cleanup
 	rm -fr build/bin/*

_assets/ci/Jenkinsfile

@@ -30,10 +30,11 @@ node('linux') {
         println(gitSHA)
     }
 
-    // TODO(adam): enable when unit tests start passing
-    // stage('Test') {
-    //     sh 'make ci'
-    // }
+    stage('Test') {
+        // TODO(adam): enable when unit tests start passing
+        //sh 'make ci'
+        sh 'make canary-test'
+    }
 
     stage('Build') {
         sh 'go get github.com/status-im/xgo'

_assets/scripts/canary_test_mailservers.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+json_path='.ClusterConfig.TrustedMailServers'
+mailservers=$(jq -r "${json_path} | .[]" $1)
+count=$(jq -r "${json_path} | length" $1)
+
+echo "Will test ${count} mailservers..."
+failed_count=0
+
+while read -r mailserver; do
+  echo "Testing $mailserver ..."
+  ./build/bin/node-canary -log=ERROR -log-without-color=true -mailserver $mailserver || failed_count=$((failed_count + 1))
+done <<< "$mailservers"
+
+if [ $failed_count -gt 0 ]; then
+  echo "${failed_count}/${count} mailservers failed the test"
+  exit 1
+else
+  echo "All mailservers replied correctly"
+fi

_assets/scripts/install_deps.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 
 if [ -x "$(command -v apt)" ]; then
-  apt install -y protobuf-compiler
+  apt install -y protobuf-compiler jq
 fi
 
 if [ -x "$(command -v pacman)" ]; then
-  pacman -Sy protobuf --noconfirm
+  pacman -Sy protobuf jq --noconfirm
 fi
 
 if [ -x "$(command -v brew)" ]; then
-  brew install protobuf
+  brew install protobuf jq
 fi