Visual tests setup (#14329)

feat: configuration setup for visual tests

Co-authored-by: Erik Seppanen <esep@protonmail.com>

.detoxrc.js

@@ -0,0 +1,37 @@
+module.exports = {
+  "testRunner": "jest",
+  "testRegex": "\\.visual\\.js$",
+  "runner-config": "visual-test/config.json",
+  "devices": {
+    "simulator": {
+      "type": "ios.simulator",
+      "device": {
+        "type": "iPhone 11 Pro"
+      }
+    }
+  },
+  "apps": {
+    "ios.release": {
+      "name": "StatusIm",
+      "type": "ios.app",
+      "binaryPath": "ios/build/Build/Products/Release-iphonesimulator/StatusIm.app",
+      "build": "make release-ios"
+    },
+    "ios.debug": {
+      "name": "StatusIm",
+      "type": "ios.app",
+      "binaryPath": process.env.TEST_BINARY_PATH,
+      "build": "make run-ios SIMULATOR='iPhone 11 Pro'"
+    }
+  },
+  "configurations": {
+    "ios.sim.release": {
+      "device": "simulator",
+      "app": "ios.release"
+    },
+    "ios.sim.debug": {
+      "device": "simulator",
+      "app": "ios.debug"
+    }
+  }
+}

.gitignore

@@ -175,3 +175,6 @@ test/appium/tests/users.py
 ##node bindings
 /bin/
 /lib/
+
+## visual tests
+/artifacts

Makefile

@@ -48,7 +48,7 @@ export _NIX_GCROOTS = /nix/var/nix/gcroots/per-user/$(USER)/status-mobile
 # Defines which variables will be kept for Nix pure shell, use semicolon as divider
 export _NIX_KEEP ?= TMPDIR,BUILD_ENV,STATUS_GO_SRC_OVERRIDE
 
-# Useful for Andoird release builds
+# Useful for Android release builds
 TMP_BUILD_NUMBER := $(shell ./scripts/version/gen_build_no.sh | cut -c1-10)
 
 # MacOS root is read-only, read nix/README.md for details
@@ -313,6 +313,16 @@ test: ##@test Run tests once in NodeJS
 	yarn shadow-cljs compile test && \
 	node --require ./test-resources/override.js target/test/test.js
 
+
+run-visual-test-ios: export TARGET := clojure
+run-visual-test-ios: XCODE_DERIVED_DATA := $(HOME)/Library/Developer/Xcode/DerivedData
+run-visual-test-ios: APPLICATION_NAME := $(shell ls $(XCODE_DERIVED_DATA) | grep -E '\bStatusIm-')
+run-visual-test-ios: export TEST_BINARY_PATH := $(XCODE_DERIVED_DATA)/$(APPLICATION_NAME)/Build/Products/Debug-iphonesimulator/StatusIm.app
+run-visual-test-ios: ##@test Run tests once in NodeJS
+	yarn install
+	detox build --configuration ios.sim.debug && \
+	detox test --configuration ios.sim.debug 
+
 #--------------
 # Other
 #--------------

package.json

@@ -84,7 +84,10 @@
     "@babel/preset-env": "7.1.0",
     "@babel/register": "7.0.0",
     "@mapbox/node-pre-gyp": "^1.0.9",
-    "jest": "^25.1.0",
+    "@types/jest": "^28.1.6",
+    "detox": "^19.9.1",
+    "jest-image-snapshot": "^5.1.0",
+    "jest": "^28.1.3",
     "nodemon": "^2.0.16",
     "nyc": "^14.1.1",
     "process": "0.11.10",

src/quo/components/button/view.cljs

@@ -56,7 +56,7 @@
 
 (defn button [{:keys [on-press disabled type theme before after
                       haptic-feedback haptic-type on-long-press on-press-start
-                      accessibility-label loading border-radius style]
+                      accessibility-label loading border-radius style test-ID]
                :or   {theme           :main
                       type            :primary
                       haptic-feedback true
@@ -92,7 +92,7 @@
                                   {:on-press-start (fn []
                                                      (optional-haptic)
                                                      (on-press-start))}))
-     [rn/view {:style (merge (style-container type) style)}
+     [rn/view {:test-ID test-ID :style (merge (style-container type) style)}
       (when before
         [rn/view
          [icons/icon before {:color icon-color}]])

src/quo2/__visual_tests__/button.e2e.js

@@ -0,0 +1,44 @@
+const waitToNavigate = duration => new Promise(resolve => setTimeout(() => resolve(), duration));
+
+const SUPER_SECRET_PASSWORD = 'password'
+
+const loginToHomePage = async () => {
+    if (device.getPlatform() === 'ios') {
+        await device.setStatusBar({ time: '12:34', dataNetwork: 'wifi', wifiBars: '3', batteryState: 'charging', batteryLevel: '100' });
+    }
+    await device.reloadReactNative();
+    await element(by.id('terms-of-service')).tap();
+    await waitToNavigate(400);
+    await element(by.id('get-started')).tap();
+    await waitToNavigate(300);
+    await element(by.id('generate-keys')).tap();
+    await waitToNavigate(200);
+    await element(by.text('Next')).tap();
+    await waitToNavigate(700);
+    await element(by.text('Next')).tap();
+    await waitToNavigate(700);
+    await element(by.id('password-placeholder')).typeText(SUPER_SECRET_PASSWORD);
+    await element(by.id('confirm-password-placeholder')).typeText(SUPER_SECRET_PASSWORD);
+    await element(by.text('Next')).tap();
+    await waitToNavigate(200);
+    await element(by.id("browser-stack")).tap();
+    await waitToNavigate(400);
+    await element(by.text("Quo2.0 Preview")).tap();
+    await waitToNavigate(400);
+}
+
+describe('Default Renders', () => {
+    beforeAll(async () => loginToHomePage())
+    beforeEach(async () => {
+    });
+    afterEach(async () => {
+        await element(by.id("back-button")).tap();
+        await waitToNavigate(200);
+    });
+
+    it(`button page should match image render`, async () => {
+        await element(by.id(`quo2-:button`)).tap();
+        await waitToNavigate(200);
+        const res = await jestExpect(`button`).toMatchImageSnapshot();
+    })
+});

src/quo2/components/buttons/button.cljs

@@ -186,7 +186,7 @@
   (let [pressed (reagent/atom false)]
     (fn [{:keys [on-press disabled type size before after above width
                  override-theme override-background-color
-                 on-long-press accessibility-label icon icon-no-color style]
+                 on-long-press accessibility-label icon icon-no-color style test-ID]
           :or   {type :primary
                  size 40}}
          children]
@@ -197,7 +197,8 @@
             state                (cond disabled :disabled @pressed :pressed :else :default)
             icon-size            (when (= 24 size) 12)
             icon-secondary-color (or icon-secondary-color icon-color)]
-        [rn/touchable-without-feedback (merge {:disabled            disabled
+        [rn/touchable-without-feedback (merge {:test-ID test-ID
+                                               :disabled            disabled
                                                :accessibility-label accessibility-label}
                                               (when on-press
                                                 {:on-press (fn []

src/quo2/components/navigation/bottom_nav_tab.cljs

@@ -26,7 +26,7 @@
     :icon-color-anim        reanimated shared value
   "
   [{:keys [icon new-notifications? notification-indicator counter-label
-           on-press pass-through? icon-color-anim accessibility-label]}]
+           on-press pass-through? icon-color-anim accessibility-label test-ID]}]
   [:f>
    (fn []
      (let [icon-animated-style       (reanimated/apply-animations-to-style
@@ -40,7 +40,8 @@
                                        :height           40
                                        :border-radius    10})]
        [rn/touchable-without-feedback
-        {:on-press            on-press
+        {:test-ID test-ID
+         :on-press            on-press
          :on-press-in         #(toggle-background-color background-color false pass-through?)
          :on-press-out        #(toggle-background-color background-color true pass-through?)
          :accessibility-label accessibility-label}

src/status_im/ui/screens/keycard/recovery/views.cljs

@@ -234,7 +234,8 @@
                                   :height 110}}])]]]
       [react/view {:margin-bottom 50}
        [quo/button
-        {:on-press #(re-frame/dispatch [:keycard.recovery.no-key.ui/generate-key-pressed])}
+        {:test-id :generate-new-key
+         :on-press #(re-frame/dispatch [:keycard.recovery.no-key.ui/generate-key-pressed])}
         (i18n/label :t/generate-new-key)]
        [quo/button
         {:type     :secondary

src/status_im/ui/screens/onboarding/intro/views.cljs

@@ -157,7 +157,7 @@
                  :margin-bottom   24}
      [quo/checkbox {:value     @tos-accepted
                     :on-change #(swap! tos-accepted not)}]
-     [rn/touchable-opacity {:on-press #(swap! tos-accepted not)}
+     [rn/touchable-opacity {:test-ID :terms-of-service :on-press #(swap! tos-accepted not)}
       [react/nested-text {:style {:margin-left 12}}
        (i18n/label :t/accept-status-tos-prefix)
        [{:style               (merge {:color colors/blue}
@@ -167,7 +167,8 @@
         " "
         (i18n/label :t/terms-of-service)]]]]
     [react/view {:style {:margin-bottom 24}}
-     [quo/button {:disabled (not @tos-accepted)
+     [quo/button {:test-ID :get-started
+                  :disabled (not @tos-accepted)
                   :on-press #(do (re-frame/dispatch [:init-root :onboarding])
                                  ;; clear atom state for next use
                                  (reset! tos-accepted false)

src/status_im/ui/screens/onboarding/keys/views.cljs

@@ -83,7 +83,8 @@
        [react/view {:style {:align-items :center}}
         [react/view {:style (assoc styles/bottom-button :margin-bottom 16)}
          [quo/button
-          {;:disabled            existing-account?
+          {:test-ID :generate-keys
+           ;:disabled            existing-account?
            :on-press            #(re-frame/dispatch [:generate-and-derive-addresses])
            :accessibility-label :onboarding-next-button}
           (i18n/label :t/generate-a-key)]]

src/status_im/ui/screens/onboarding/password/views.cljs

@@ -44,7 +44,8 @@
            (i18n/label :intro-wizard-title-alt4)]
           [rn/view
            [rn/view {:style {:padding 16}}
-            [quo/text-input {:secure-text-entry   true
+            [quo/text-input {:test-ID :password-placeholder
+                             :secure-text-entry   true
                              :auto-capitalize     :none
                              :auto-focus          true
                              :show-cancel         false
@@ -56,7 +57,8 @@
                                                      (some-> ^js @confirm-ref .focus))}]]
            [rn/view {:style {:padding 16
                              :opacity (if-not valid-password 0.33 1)}}
-            [quo/text-input {:secure-text-entry   true
+            [quo/text-input {:test-ID :confirm-password-placeholder
+                             :secure-text-entry   true
                              :get-ref             #(reset! confirm-ref %)
                              :auto-capitalize     :none
                              :show-cancel         false

src/status_im/ui2/screens/quo2_preview/main.cljs

@@ -227,7 +227,8 @@
                  (for [{:keys [name]} (val category)]
                    ^{:key name}
                    [quo2-button/button
-                    {:style {:margin-vertical 8}
+                    {:test-ID (str "quo2-" name)
+                     :style {:margin-vertical 8}
                      :on-press #(re-frame/dispatch [:navigate-to name])}
                     (clojure.core/name name)])]) (sort screens-categories))]])]))
 

src/status_im2/contexts/shell/bottom_tabs.cljs

@@ -38,7 +38,8 @@
 
 (defn bottom-tab [icon stack-id shared-values]
   [bottom-nav-tab/bottom-nav-tab
-   {:icon                icon
+   {:test-ID stack-id
+    :icon                icon
     :icon-color-anim     (get
                           shared-values
                           (get constants/tabs-icon-color-keywords stack-id))

src/status_im2/navigation/roots.cljs

@@ -18,6 +18,7 @@
    :title                {:color colors/neutral-100}
    :rightButtonColor     colors/neutral-100
    :background           {:color colors/white}
+   :backButton {:testID :back-button}
    ;; TODO adjust colors and icons with quo2
    ;;:backButton
    #_{:icon  (icons/icon-source :main-icons/arrow-left)

visual-test/config.json

@@ -0,0 +1,13 @@
+{
+    "globalSetup": "./global-setup.js",
+    "globalTeardown": "./global-teardown.js",
+    "setupFilesAfterEnv": ["./setup.js"],
+    "maxWorkers": 1,
+    "testEnvironment": "./environment",
+    "testRunner": "jest-circus/runner",
+    "testTimeout": 120000,
+    "testRegex": "\\.e2e\\.js$",
+    "roots": ["../src/"],
+    "reporters": ["detox/runners/jest/streamlineReporter"],
+    "verbose": true
+}

visual-test/environment.js

@@ -0,0 +1,23 @@
+const {
+    DetoxCircusEnvironment,
+    SpecReporter,
+    WorkerAssignReporter,
+  } = require('detox/runners/jest-circus');
+  
+  class CustomDetoxEnvironment extends DetoxCircusEnvironment {
+    constructor(config, context) {
+      super(config, context);
+  
+      // Can be safely removed, if you are content with the default value (=300000ms)
+      this.initTimeout = 300000;
+  
+      // This takes care of generating status logs on a per-spec basis. By default, Jest only reports at file-level.
+      // This is strictly optional.
+      this.registerListeners({
+        SpecReporter,
+        WorkerAssignReporter,
+      });
+    }
+  }
+  
+  module.exports = CustomDetoxEnvironment;

visual-test/global-setup.js

@@ -0,0 +1,40 @@
+const fs = require('fs-extra');
+const { execSync } = require('child_process');
+const detox = require('detox');
+
+async function globalSetup() {
+  const config = resolveSelectedConfiguration() || {};
+  downloadTestButlerAPKIfNeeded(config);
+  await detox.globalInit();
+}
+
+function downloadTestButlerAPKIfNeeded(config) {
+  if (isAndroidConfig(config)) {
+    downloadTestButlerAPK();
+  }
+}
+
+function downloadTestButlerAPK() {
+  const version = '2.2.1';
+  const artifactUrl = `https://repo1.maven.org/maven2/com/linkedin/testbutler/test-butler-app/${version}/test-butler-app-${version}.apk`;
+  const filePath = `./cache/test-butler-app.apk`;
+
+  fs.ensureDirSync('./cache');
+  if (!fs.existsSync(filePath)) {
+    console.log(`\nDownloading Test-Butler APK v${version}...`);
+    execSync(`curl -f -o ${filePath} ${artifactUrl}`);
+  }
+}
+
+function resolveSelectedConfiguration() {
+  const { configurations } = require('../.detoxrc');
+  const configName = process.env.DETOX_CONFIGURATION; 
+  return configurations[configName];
+}
+
+// TODO eventually, this should be made available by Detox more explicitly
+function isAndroidConfig(config) {
+  return [config.type, process.env.DETOX_CONFIGURATION, config.device].some(s => `${s}`.includes('android'));
+}
+
+module.exports = globalSetup;

visual-test/global-teardown.js

@@ -0,0 +1,7 @@
+const detox = require('detox');
+
+async function globalTeardown() {
+  await detox.globalCleanup();
+}
+
+module.exports = globalTeardown;

visual-test/setup.js

@@ -0,0 +1,41 @@
+const { configureToMatchImageSnapshot } = require('jest-image-snapshot');
+const fs = require('fs');
+const path = require('path');
+const kebabCase = require('lodash/kebabCase');
+const {expect} = require('expect');
+
+
+const toMatchImage = configureToMatchImageSnapshot({
+  comparisonMethod: 'ssim', failureThreshold: 0.002, failureThresholdType: 'percent'
+});
+
+
+expect.extend({ toMatchImage });
+
+expect.extend({
+  async toMatchImageSnapshot(screenName) {
+    const platform = await device.getPlatform();
+    const deviceName = await device.name.split(' ').slice(1).join ('') ;
+    const deviceType = 'iPhone 11 Pro' ;
+
+    const SNAPSHOTS_DIR = `__image_snapshots__/${platform}/${deviceType}`;
+
+    const { testPath, currentTestName } = this;
+
+    const customSnapshotsDir = path.join(path.dirname(testPath), SNAPSHOTS_DIR);
+    const customSnapshotIdentifier = kebabCase(`${path.basename(testPath)}-${currentTestName}-${screenName}`)
+
+    const tempPath = await device.takeScreenshot(screenName);
+    const image = fs.readFileSync(tempPath);
+    expect(image).toMatchImage({ customSnapshotIdentifier, customSnapshotsDir });
+
+    return { pass: true }
+  },
+});
+
+global.jestExpect = expect
+
+
+beforeAll(async () => {
+    await device.launchApp();
+  });