refactor and simplify mkFilter, use more restrictive rules

This should reduce rebuilds of status-react-npm-gradle-modules

Signed-off-by: Jakub Sokołowski <jakub@status.im>

nix/mobile/android/jsbundle/default.nix

@@ -20,12 +20,22 @@ in stdenv.mkDerivation {
       filter =
         # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
         mkFilter {
-          dirRootsToInclude = [ 
+          include = [ 
-            "components/src" "react-native/src/cljsjs" "react-native/src/mobile" "src" "env/prod" "prod" # Taken from project.clj :profiles :prod :cljsbuild :builds :android :source-paths
+            "src/.*" "prod/.*" "env/prod/.*"
-            "resources" "status-modules/cljs" "status-modules/resources" "scripts/version"
+            "components/src/.*" 
+            "react-native/src" 
+            "react-native/src/cljsjs/.*"
+            "react-native/src/mobile/.*"
+            "status-modules/cljs/.*"
+            "status-modules/resources/.*"
+            "build.clj" "externs.js"
+            "project.clj" "prepare-modules.js"
+            "resources/js/.*"
+            "resources/config/.*"
+          ];
+          exclude = [
+            "resources/images/.*"
           ];
-          dirsToExclude = [ ".git" ".svn" "CVS" ".hg" ".gradle" "build" "intermediates" "libs" "obj" ];
-          filesToInclude = [ "build.clj" "externs.js" "project.clj" "prepare-modules.js" "VERSION" "BUILD_NUMBER"];
           root = path;
         };
     };

nix/mobile/android/maven-and-npm-deps/default.nix

@@ -37,14 +37,14 @@ let
             filter =
               # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
               mkFilter {
-                dirRootsToInclude = [
-                  "android" "mobile/js_files" "resources"
-                  "translations" "status-modules"
-                ];
-                dirsToExclude = [ ".git" ".svn" "CVS" ".hg" ".gradle" "build" "intermediates" "libs" "obj" ];
-                filesToInclude = [ ".babelrc" ];
-                filesToExclude = [ "VERSION" "android/gradlew" ];
                 root = path;
+                include = [
+                  "android/.*" "translations/.*" "status-modules/.*"
+                  "resources/.*" "mobile/js_files/.*" ".babelrc"
+                ];
+                exclude = [
+                  ".*.keystore" "node_modules"
+                ];
               };
           };
         phases = [ "unpackPhase" "patchPhase" "installPhase" "fixupPhase" ];

nix/mobile/android/targets/release-android.nix

@@ -44,20 +44,13 @@ in stdenv.mkDerivation {
       filter =
         # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
         mkFilter {
-          dirRootsToInclude = [
+          root = path;
-            "mobile/js_files"
+          include = [
-            "modules/react-native-status/android"
+            "mobile/js_files.*" "resources.*"
-            "resources"
+            "modules/react-native-status/android.*"
-          ];
-          dirsToExclude = [
-            ".git" ".svn" "CVS" ".hg" ".gradle"
-            "build" "intermediates" "libs" "obj"
-          ];
-          filesToInclude = [
             envFileName "VERSION" ".watchmanconfig"
             "status-go-version.json" "react-native.config.js"
           ];
-          root = path;
         };
     };
   nativeBuildInputs = [ bash gradle unzip ] ++ lib.optionals stdenv.isDarwin [ file gnumake patchedWatchman ];

nix/mobile/ios/default.nix

@@ -12,15 +12,16 @@ let
 
   src =
     let path = ./../../..;
-    in builtins.path { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
+    # We use builtins.path so that we can name the resulting derivation,
+    # otherwise the name would be taken from the checkout directory, which is outside of our control
+    in builtins.path {
       inherit path;
       name = "status-react-source-npm-deps";
       filter =
-        # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
+        # Keep this filter as restrictive as possible in order to avoid 
+        # unnecessary rebuilds and limit closure size
         mkFilter {
-          dirRootsToInclude = [ "mobile/js_files" ];
+          include = [ ".babelrc" "mobile/js_files.*" ];
-          dirsToExclude = [ ".git" ".svn" "CVS" ".hg" ];
-          filesToInclude = [ ".babelrc" ];
           root = path;
         };
     };

nix/status-go/nimbus/default.nix

@@ -12,10 +12,11 @@ let
     filter =
       # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
       mkFilter {
-        dirRootsToInclude = [ "nix" "wrappers" "vendor" ];
-        dirsToExclude = [ ".git" ".svn" "CVS" ".hg" ".vscode" ".dependabot" ".github" "examples" "docs" ];
-        filesToInclude = [ "Makefile" "nim.cfg" "nimbus.nimble" "default.nix" ];
         root = path;
+        include = [
+          "nix.*" "wrappers.*" "vendor.*"
+          "Makefile" "nim.cfg" "nimbus.nimble" "default.nix"
+        ];
       };
   };
 

nix/tools/mkFilter.nix

@@ -1,44 +1,59 @@
 { lib }:
 
-# This Nix expression allows filtering a local directory by specifying dirRootsToInclude, dirsToExclude and filesToInclude.
+# This Nix expression allows filtering a local directory by
-# It also filters out symlinks to result folders created by nix-build, as well as backup/swap/generated files
+# specifying dirRootsToInclude, dirsToExclude and filesToInclude.
+# It also filters out symlinks to result folders created by nix-build,
+# as well as backup/swap/generated files.
 
 let
   inherit (lib)
-    any compare compareLists elem elemAt hasPrefix length min splitString take;
+    any range flatten length sublist cleanSourceFilter 
-
+    splitString hasPrefix removePrefix concatStringsSep;
-  isPathAllowed = allowedPath: path:
+  inherit (builtins) map match;
-    let
-      count = min (length allowedPathElements) (length pathElements);
-      pathElements = splitString "/" path;
-      allowedPathElements = splitString "/" allowedPath;
-      pathElementsSubset = take count pathElements;
-      allowedPathElementsSubset = take count allowedPathElements;
-    in (compareLists compare allowedPathElementsSubset pathElementsSubset) == 0;
 
   mkFilter = {
-    dirRootsToInclude, # Relative paths of directories to include
+    # primary path under which all files are included, unless excluded
-    dirsToExclude ? [ ], # Base names of directories to exclude
+    root,
-    filesToInclude ? [ ], # Relative path of files to include
+    # list of regex expressions to match files to include/exclude
-    filesToExclude ? [ ], # Relative path of files to exclude
+    include ? [ ], exclude ? [ ], # has precedence over include
-    root }:
+    }:
     let
-      allPathRootsAllowed = (length dirRootsToInclude) == 0;
+      # removes superfluous slashes from the path
-      # this removes superfluous slashes from the path
       cleanRoot = "${toString (/. + root)}/";
-    in
+    in path: type:
-      path: type:
     let
-        baseName = baseNameOf (toString path);
+      # unpack path: "x/y/.*" => ["x" "x/y" "x/y/.*"]
-        subpath = elemAt (splitString cleanRoot path) 1;
+      unpackPath = path:
-        spdir = elemAt (splitString "/" subpath) 0;
+        let
+          tokens = splitString "/" path;
+          perms = range 1 (length tokens);
+          subPaths = builtins.map (x: sublist 0 x tokens) perms;
+        in builtins.map (x: concatStringsSep "/" x) subPaths;
+      # accept subdirs from regexes
+      includeSubdirs = regexes: flatten (map unpackPath regexes);
+      # checks all regexes in a list against str
+      matchesRegexes = str: regexes: (map (r: (match r str)) regexes);
+      # match returns empty list on match
+      isMatch = x: x == [ ];
+      # path relative to search root
+      relPath = removePrefix cleanRoot path;
+      # check if any of the regexes match the relative path
+      checkRegexes = regexes: any isMatch (matchesRegexes relPath regexes);
 
-      in lib.cleanSourceFilter path type && (
+      # main check methods
-        (type != "directory" && (elem spdir filesToInclude) && !(elem spdir filesToExclude)) ||
+      isRootSubdir = hasPrefix cleanRoot path;
-        # check if any part of the directory path is described in dirRootsToInclude
+      isIncluded = checkRegexes (includeSubdirs include);
-        ((allPathRootsAllowed || (any (dirRootToInclude: isPathAllowed dirRootToInclude subpath) dirRootsToInclude)) && ! (
+      isExcluded = checkRegexes exclude;
-          # Filter out version control software files/directories
+      isSCV = !cleanSourceFilter path type;
-          (type == "directory" && (elem baseName dirsToExclude))
+
-      )));
+    in
+      if !isRootSubdir then
+        # everything outside of root is excluded
+        false
+      else if isExcluded || isSCV then
+        # isExcluded has precedence over isIncluded
+        false
+      else
+        isIncluded;
 
 in mkFilter

nix/tools/mkFilter_test.nix

@@ -1,56 +1,66 @@
 # This file is an example of the syntax required to call a Nix function
 # and serves to test mkFilter.nix.
 # 
-# nix-instantiate --strict --json --eval ./mkFilter_test.nix
+# nix-instantiate --strict --json --eval ./mkFilter_test.nix | jq
 # [
 #   {
-#     "expected": true,
+#     "pass": true,
-#     "path": "/home/pedro/src/github.com/status-im/status-react/android/1",
+#     "path": "/ABS/PATH/android/subdir",
-#     "result": true
 #   },
 #   {
-#     "expected": true,
+#     "pass": true,
-#     "path": "/home/pedro/src/github.com/status-im/status-react/ios",
+#     "path": "/ABS/PATH/ios",
-#     "result": false
 #   }
+#   ...
 # ]
 
-{ pkgs ? import <nixpkgs> { },
-  lib ? pkgs.stdenv.lib }:
-
 let
+  pkgs = import <nixpkgs> { };
+  lib = pkgs.stdenv.lib;
   mkFilter = pkgs.callPackage ./mkFilter.nix { inherit lib; };
-  absPath = "/ABS/PROJECT/PATH";
+  absPath = "/ABS/PATH";
   filter = mkFilter {
-    dirRootsToInclude = [ "android" ];
-    # dirsToExclude ? [],  # Base names of directories to exclude
-    # filesToInclude ? [], # Relative path of files to include
-    # filesToExclude ? [], # Relative path of files to exclude
     root = absPath;
+    include = [ "android" ".*included.*" "sub/folder/.*" ];
+    exclude = [ ".*excluded.*" ];
   };
   tests = [
-    {
+    { path = "/WRONG/ABS/PATH"; type = "directory";
-      a = {
+      expected = false; }
-        path = "${absPath}/android/1";
+    { path = "${absPath}/.git"; type = "directory";
-        type = "directory";
+      expected = false; }
-      };
+    { path = "${absPath}/included"; type = "file";
-      e = true;
+      expected = true; }
-    }
+    { path = "${absPath}/android/included"; type = "directory";
-    {
+      expected = true; }
-      a = {
+    { path = "${absPath}/sub"; type = "file";
-        path = "${absPath}/ios";
+      expected = true; }
-        type = "directory";
+    { path = "${absPath}/sub/folder"; type = "file";
-      };
+      expected = true; }
-      e = false;
+    { path = "${absPath}/sub/folder/file"; type = "file";
-    }
+      expected = true; }
+    { path = "${absPath}/sub/folder/xyz/file"; type = "file";
+      expected = true; }
+    { path = "${absPath}/android/sub/included"; type = "directory";
+      expected = true; }
+    { path = "${absPath}/android/included/excluded"; type = "directory";
+      expected = false; }
+    { path = "${absPath}/android/excluded"; type = "directory";
+      expected = false; }
+    { path = "${absPath}/ios/included"; type = "directory";
+      expected = true; }
+    { path = "${absPath}/ios/subfile"; type = "file";
+      expected = false; }
   ];
-  boolToString = b: if b then "true" else "false";
+  # make paths absolute
-
+  testsAbs = builtins.map (
+    t: t // { path = "${absPath}/${t.path}"; }
+  ) tests;
 in builtins.map (
     t: let
-      rval = (filter t.a.path t.a.type);
+      rval = (filter t.path t.type);
     in {
-      path = t.a.path;
+      path = t.path;
-      pass = t.e == rval;
+      pass = t.expected == rval;
     }
   ) tests

scripts/generate-keystore.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -euf pipefail
+set -ef pipefail
 
 GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
 source "${GIT_ROOT}/scripts/colors.sh"

scripts/release-android.sh

@@ -56,7 +56,6 @@ if [[ "$OS" =~ Darwin ]]; then
     " --option" "extra-sandbox-paths" "${KEYSTORE_PATH} ${SECRETS_FILE_PATH} ${WATCHMAN_SOCKFILE}"
   )
 else
-  echo wtf
   nixOpts+=(
     "--option" "extra-sandbox-paths" "${KEYSTORE_PATH} ${SECRETS_FILE_PATH}"
   )