From a5bb95cd18b95b580e8e8d0ff037e96a408f34a0 Mon Sep 17 00:00:00 2001 From: Dmitri Akatov Date: Tue, 7 Nov 2023 10:33:59 +0000 Subject: [PATCH] add translation linting to the "make lint" pipeline. (#17820) --- Makefile | 1 + nix/shells.nix | 2 +- scripts/lint_translations.clj | 98 +++++++++++++++++++ .../tests/non_func/test_translations.py | 41 -------- 4 files changed, 100 insertions(+), 42 deletions(-) create mode 100755 scripts/lint_translations.clj delete mode 100644 test/appium/tests/non_func/test_translations.py diff --git a/Makefile b/Makefile index cd282740a7..98fd46e302 100644 --- a/Makefile +++ b/Makefile @@ -315,6 +315,7 @@ lint: ##@test Run code style checks sh scripts/lint-direct-require-component-outside-quo.sh && \ clj-kondo --config .clj-kondo/config.edn --cache false --fail-level error --lint src $(if $(filter $(CLJ_LINTER_PRINT_WARNINGS),true),,| grep -v ': warning: ') && \ ALL_CLOJURE_FILES=$(call find_all_clojure_files) && \ + scripts/lint_translations.clj && \ zprint '{:search-config? true}' -sfc $$ALL_CLOJURE_FILES && \ sh scripts/lint-trailing-newline.sh && \ node_modules/.bin/prettier --write . diff --git a/nix/shells.nix b/nix/shells.nix index 8f9cd776a6..19814c1a22 100644 --- a/nix/shells.nix +++ b/nix/shells.nix @@ -21,7 +21,7 @@ let buildInputs = with pkgs; [ clojure flock maven openjdk # lint specific utilities - clj-kondo zprint clojure-lsp ripgrep + babashka clj-kondo clojure-lsp ripgrep zprint ]; # CLASSPATH from clojure deps with 'src' appended to find local sources. shellHook = with pkgs; '' diff --git a/scripts/lint_translations.clj b/scripts/lint_translations.clj new file mode 100755 index 0000000000..3e1866365d --- /dev/null +++ b/scripts/lint_translations.clj @@ -0,0 +1,98 @@ +#!/usr/bin/env bb + +(ns lint-translations + (:require [babashka.pods :as pods])) + +(pods/load-pod 'clj-kondo/clj-kondo "2023.09.07") +(require '[pod.borkdude.clj-kondo :as kondo]) +(require '[cheshire.core :as json]) +(require '[clojure.set :as set]) + +(def src-paths ["src"]) + +(def translation-file "translations/en.json") + +;; set the following to true when solving https://github.com/status-im/status-mobile/issues/17811 +(def flag-show-non-namespaced-translation-keys false) +(def flag-show-non-namespaced-translation-keys-occurrences false) ;; this makes the output super verbose! + +;; set the following to true when solving https://github.com/status-im/status-mobile/issues/17813 +;; and keep it permanently on after #17811 and #17813 have both been solved +(def flag-show-unused-translation-keys false) + +(def flag-show-missing-translation-keys true) + +(defn- safe-name + [x] + (when x (name x))) + +(defn- ->keyword + [analysis-keyword] + (keyword (safe-name (:ns analysis-keyword)) (:name analysis-keyword))) + +(defn- report-issues + [incorrect-usages] + (doseq [incorrect-usage incorrect-usages] + (->> incorrect-usage + ((juxt :filename :row :reason ->keyword)) + (apply format "%s:%s %s %s") + println))) + +(defn- extract-translation-keys + [file] + (-> file slurp json/parse-string keys)) + +(def ^:private probably-unused-warning + (format "Probably unused translation key in %s:" translation-file)) + +(defn -main + [& _args] + (println "Linting translations...") + (let [result (kondo/run! + {:lint src-paths + :config {:output {:analysis {:keywords true}}}}) + all-keywords (get-in result [:analysis :keywords]) + used-translations (filter (comp (partial = 't) :ns) all-keywords) + file-translation-keys (apply sorted-set (extract-translation-keys translation-file)) + missing-translations (remove (comp file-translation-keys :name) used-translations) + used-translation-keys (set (map :name used-translations)) + possibly-unused-translation-keys (set/difference file-translation-keys used-translation-keys) + non-namespaced-translations (filter + (fn [kw] + (and (not (:ns kw)) + (possibly-unused-translation-keys (:name kw)))) + all-keywords) + unused-translation-keys (set/difference possibly-unused-translation-keys + (set (map :name non-namespaced-translations)))] + + ;; TODO (2023-11-06 akatov): delete the following once #17811 and #17813 have both been solved + (doseq [k (apply sorted-set (map :name non-namespaced-translations))] + (when flag-show-non-namespaced-translation-keys + (println "Probably non-namespaced key" k)) + (when flag-show-non-namespaced-translation-keys-occurrences + (->> non-namespaced-translations + (filter #(= k (:name %))) + (map #(assoc % :reason "Possibly non-namespaced translation key")) + report-issues))) + + (when flag-show-unused-translation-keys + (run! #(println probably-unused-warning %) unused-translation-keys)) + + (when flag-show-missing-translation-keys + (report-issues (map #(assoc % :reason "Undefined Translation Key") missing-translations))) + + (if (and + (or (not flag-show-missing-translation-keys) + (empty? missing-translations)) + (or (not flag-show-unused-translation-keys) + (empty? possibly-unused-translation-keys)) + (or (not flag-show-non-namespaced-translation-keys) + (not flag-show-non-namespaced-translation-keys-occurrences) + (empty? unused-translation-keys))) + 0 + 1))) + +(when (= *file* (System/getProperty "babashka.file")) + (->> *command-line-args* + (apply -main) + System/exit)) diff --git a/test/appium/tests/non_func/test_translations.py b/test/appium/tests/non_func/test_translations.py deleted file mode 100644 index fe258e4ff7..0000000000 --- a/test/appium/tests/non_func/test_translations.py +++ /dev/null @@ -1,41 +0,0 @@ -from itertools import chain - -import json -import os -import pytest -from tests import marks -from tests.base_test_case import NoDeviceTestCase - - -class TestTranslations(NoDeviceTestCase): - - @marks.testrail_id(6223) - @marks.skip - # skipped: no need to launch it on daily basis - def test_find_unused_translations(self): - directory = os.sep.join(__file__.split(os.sep)[:-5]) - with open(os.path.join(directory, 'translations/en.json'), 'r') as f: - data = set(json.load(f).keys()) - result = [] - paths = ['src/status_im', 'components/src', 'src'] - for root, dirs, files in chain.from_iterable(os.walk(os.path.join(directory, path)) for path in paths): - dirs[:] = [d for d in dirs if d not in ['test', 'translations']] - for file in [file for file in files if file.endswith('.cljs')]: - with open(os.path.join(root, file), "r") as source: - try: - content = source.read() - for key_name in data: - if key_name in content: - result.append(key_name) - except UnicodeDecodeError: - pass - unused = data - set(result) - recheck = [i for i in unused if i[-1].isdigit()] - error = '' - if recheck: - error += 'Translations to recheck: \n %s' % recheck - unused -= set(recheck) - if unused: - error += '\nUnused translations: \n %s' % unused - if error: - pytest.fail(error)