nix: refactor Gradle deps to be more generic

Refactoring the derivation that fetches all the POMs, JARs,
and AARs in order to make it more generic and easier to extend.
The main change is adding `files` key in `deps.json` which contains
a dict of all the files reletad to given package.

This way we can more easily include other files that might be available
for download, like AARs with ASC suffix, or `nodeps` JARs.

This is also necessary for the React Native upgrade:
https://github.com/status-im/status-mobile/pull/15203

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2023-02-28 18:22:17 +01:00
parent ff3ba6c0f0
commit 20a0cc01f6
No known key found for this signature in database
GPG Key ID: FE65CD384D5BF7B4
10 changed files with 8527 additions and 7748 deletions

View File

@ -24,6 +24,7 @@ Generating scripts:
- `get_deps.sh` - Calls Gradle to get all the dependencies of sub-projects.
- `gradle_parser.awk` - An AWK script that parses above Gradle output.
- `url2json.sh` - Converts the list of URLs into a format consumable by Nix.
- `add_package.sh` - Allows for adding a missing package manually.
Finally we have the Nix derivation in `default.nix` which produces a derivation with all of the Gradle project dependencies:
```
@ -66,9 +67,14 @@ You can use the `add_package.sh` script to add missing Gradle dependencies:
> make shell TARGET=gradle
Configuring Nix shell for target 'gradle'...
[nix-shell:~/status-mobile]$ nix/deps/gradle/add_package.sh com.android.tools.build:gradle:3.4.0
[nix-shell:~/work/status-mobile]$ nix/deps/gradle/add_package.sh com.android.tools.build:gradle:3.5.3
Changes made:
nix/deps/gradle/deps.urls | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
Regenerating Nix files...
Found 548 direct dependencies...
Successfully added: com.android.tools.build:gradle:3.4.0
Successfully added: com.android.tools.build:gradle:3.5.3
NOTICE: Running 'make nix-update-gradle' in a new shell is recommended.
```
This may take a bit longer than normal `make nix-update-gradle` but should add the missing package.
Keep in mind that the changes made by this script do not affect the already spawned shell.

View File

@ -1,6 +1,10 @@
#!/usr/bin/env bash
set -Eeu
# This script allows for adding a package by providing a Maven full name.
# Such name consists of 3 sections separated by the colon character.
# Example: com.android.tools.build:gradle:3.5.3
set -Eeuo pipefail
if [[ $# -ne 1 ]]; then
echo "Usage: add_package.sh <package>" >&2
@ -18,12 +22,15 @@ source "${GIT_ROOT}/scripts/colors.sh"
echo "${1}" | go-maven-resolver >> nix/deps/gradle/deps.urls
# Remove duplicates and sort.
sort -uo nix/deps/gradle/deps.urls nix/deps/gradle/deps.urls
sort -uVo nix/deps/gradle/deps.urls nix/deps/gradle/deps.urls
echo -e "${GRN}Changes made:${RST}" >&2
git diff --stat nix/deps/gradle/deps.urls
echo
# Re-generate dependencies JSON.
"${GIT_ROOT}/nix/deps/gradle/generate.sh" gen_deps_json
# Re-generate dependencies list.
"${GIT_ROOT}/nix/deps/gradle/generate.sh" gen_deps_list
echo -e "${GRN}Successfully added:${RST} ${BLD}${1}${RST}" >&2
echo
echo -e "${YLW}NOTICE:${RST} Running '${BLD}make nix-update-gradle${RST}' in a new shell is recommended."

View File

@ -1,82 +1,44 @@
{ stdenv, lib, pkgs, fetchurl, writeShellScriptBin }:
let
inherit (lib)
removeSuffix optionalString splitString concatMapStrings
attrByPath attrValues last makeOverridable importJSON;
inherit (lib) concatStrings concatMapStrings mapAttrsToList makeOverridable;
inherit (pkgs) aapt2;
deps = importJSON ./deps.json;
# some .jar files have an `-aot` suffix that doesn't work for .pom files
getPOM = jarUrl: "${removeSuffix "-aot" jarUrl}.pom";
deps = lib.importJSON ./deps.json;
script = writeShellScriptBin "create-local-maven-repo" (''
mkdir -p $out
cd $out
'' +
# TODO: Generalize this section to not repeat the same code.
(concatMapStrings (dep:
let
url = "${dep.host}/${dep.path}";
pom = {
sha1 = attrByPath [ "pom" "sha1" ] "" dep;
sha256 = attrByPath [ "pom" "sha256" ] "" dep;
};
pom-download = optionalString (pom.sha256 != "") (
fetchurl { url = getPOM url; inherit (pom) sha256; }
);
jar = {
sha1 = attrByPath [ "jar" "sha1" ] "" dep;
sha256 = attrByPath [ "jar" "sha256" ] "" dep;
};
jar-download = optionalString (jar.sha256 != "") (
fetchurl { url = "${url}.${dep.type}"; inherit (jar) sha256; }
);
nodeps = {
sha1 = attrByPath [ "nodeps" "sha1" ] "" dep;
sha256 = attrByPath [ "nodeps" "sha256" ] "" dep;
};
nodeps-download = optionalString (nodeps.sha256 != "") (
fetchurl { url = "${url}-nodeps.jar"; inherit (nodeps) sha256; }
);
fileName = last (splitString "/" dep.path);
directory = removeSuffix fileName dep.path;
in
''
mkdir -p ${directory}
${optionalString (pom-download != "") ''
ln -s "${pom-download}" "${getPOM dep.path}"
''}
${optionalString (pom.sha1 != "") ''
echo "${pom.sha1}" > "${getPOM dep.path}.sha1"
''}
${optionalString (jar-download != "") ''
ln -s "${jar-download}" "${dep.path}.${dep.type}"
''}
${optionalString (jar.sha1 != "") ''
echo "${jar.sha1}" > "${dep.path}.${dep.type}.sha1"
''}
${optionalString (nodeps-download != "") ''
ln -s "${nodeps-download}" "${dep.path}-nodeps.jar"
''}
${optionalString (nodeps.sha1 != "") ''
echo "${nodeps.sha1}" > "${dep.path}.${dep.type}.sha1"
''}
# For each dependency in deps.json.
(concatMapStrings (dep: concatStrings
# And for each file in the 'files' dict of given dependency.
(mapAttrsToList (filename: hashes: let
# Download given file(POM, JAR, nodeps JAR, or AAR).
download = fetchurl {
url = "${dep.repo}/${dep.path}/${filename}";
inherit (hashes) sha256;
};
# And symlink it in the correct folder along with SHA1.
in ''
mkdir -p "${dep.path}"
ln -s "${download}" "${dep.path}/${filename}"
echo "${hashes.sha1}" > "${dep.path}/${filename}.sha1"
'')
deps));
dep.files)
) deps)
);
in makeOverridable stdenv.mkDerivation {
name = "status-mobile-maven-deps";
buildInputs = [ aapt2 ];
buildInputs = [ pkgs.aapt2 ];
phases = [ "buildPhase" "patchPhase" ];
buildPhase = "${script}/bin/create-local-maven-repo";
# Patched AAPT2
# Replace AAPT2 package only with our patched version from overlay.
patchPhase = ''
aapt2_dir=$out/com/android/tools/build/aapt2/${aapt2.version}
aapt2_dir=$out/com/android/tools/build/aapt2/${pkgs.aapt2.version}
mkdir -p $aapt2_dir
ln -sf ${aapt2}/* $aapt2_dir
ln -sf ${pkgs.aapt2}/* $aapt2_dir
'';
}

File diff suppressed because it is too large Load Diff

View File

@ -636,7 +636,7 @@ https://repo.maven.apache.org/maven2/org/bouncycastle/bcprov-jdk15on/1.56/bcprov
https://repo.maven.apache.org/maven2/org/bouncycastle/bcprov-jdk15on/1.60/bcprov-jdk15on-1.60.pom
https://repo.maven.apache.org/maven2/org/bouncycastle/bcprov-jdk15on/1.70/bcprov-jdk15on-1.70.pom
https://repo.maven.apache.org/maven2/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.pom
https://repo.maven.apache.org/maven2/org/checkerframework/checker-qual/3.31.0/checker-qual-3.31.0.pom
https://repo.maven.apache.org/maven2/org/checkerframework/checker-qual/3.32.0/checker-qual-3.32.0.pom
https://repo.maven.apache.org/maven2/org/codehaus/codehaus-parent/4/codehaus-parent-4.pom
https://repo.maven.apache.org/maven2/org/codehaus/groovy/groovy-all/2.4.15/groovy-all-2.4.15.pom
https://repo.maven.apache.org/maven2/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.pom

View File

@ -1,18 +1,16 @@
#!/usr/bin/env bash
# Stop on any failures, including in pipes
set -e
set -o pipefail
# This script takes care of generating/updating the maven-sources.nix file
# representing the offline Maven repo containing the dependencies
# required to build the project
set -Eeo pipefail
if [[ -z "${IN_NIX_SHELL}" ]]; then
echo "Remember to call 'make shell'!"
exit 1
fi
# This script takes care of generating/updating the maven-sources.nix file
# representing the offline Maven repo containing the dependencies
# required to build the project
GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
source "${GIT_ROOT}/scripts/colors.sh"
@ -27,41 +25,43 @@ ulimit -n 16384
# Generate list of Gradle sub-projects.
function gen_proj_list() {
${CUR_DIR}/get_projects.sh | sort -u -o ${PROJ_LIST}
echo -e "Found ${GRN}$(wc -l < ${PROJ_LIST})${RST} sub-projects..."
"${CUR_DIR}/get_projects.sh" | sort -u -o "${PROJ_LIST}"
echo -e "Found ${GRN}$(wc -l < "${PROJ_LIST}")${RST} sub-projects..."
}
# Check each sub-project in parallel, the ":" is for local deps.
function gen_deps_list() {
PROJECTS=$(cat ${PROJ_LIST})
${CUR_DIR}/get_deps.sh ":" ${PROJECTS[@]} | sort -uV -o ${DEPS_LIST}
echo -e "${CLR}Found ${GRN}$(wc -l < ${DEPS_LIST})${RST} direct dependencies..."
PROJECTS=$(cat "${PROJ_LIST}")
# WARNING: The ${PROJECTS[@]} needs to remain unquoted to expand correctly.
# shellcheck disable=SC2068
"${CUR_DIR}/get_deps.sh" ":" ${PROJECTS[@]} | sort -uV -o "${DEPS_LIST}"
echo -e "${CLR}Found ${GRN}$(wc -l < "${DEPS_LIST}")${RST} direct dependencies..."
}
# Find download URLs for each dependency.
function gen_deps_urls() {
cat ${DEPS_LIST} | go-maven-resolver | sort -uV -o ${DEPS_URLS}
echo -e "${CLR}Found ${GRN}$(wc -l < ${DEPS_URLS})${RST} dependency URLs..."
go-maven-resolver < "${DEPS_LIST}" | sort -uV -o "${DEPS_URLS}"
echo -e "${CLR}Found ${GRN}$(wc -l < "${DEPS_URLS}")${RST} dependency URLs..."
}
# Generate the JSON that Nix will consume.
function gen_deps_json() {
# Open the Nix attribute set.
echo -n "[" > ${DEPS_JSON}
echo -n "[" > "${DEPS_JSON}"
# Format URLs into a Nix consumable file.
URLS=$(cat ${DEPS_URLS})
URLS=$(cat "${DEPS_URLS}")
# Avoid rate limiting by using 4 of the available threads.
parallel --will-cite --keep-order --jobs 4 \
"${CUR_DIR}/url2json.sh" \
::: ${URLS} \
>> ${DEPS_JSON}
::: "${URLS}" \
>> "${DEPS_JSON}"
# Drop tailing comma on last object, stupid JSON
sed -i '$ s/},/}/' ${DEPS_JSON}
sed -i '$ s/},/}/' "${DEPS_JSON}"
# Close the Nix attribute set
echo "]" >> ${DEPS_JSON}
echo "]" >> "${DEPS_JSON}"
}
# ------------------------------------------------------------------------------
@ -74,7 +74,7 @@ cd "${GIT_ROOT}/android"
./gradlew --stop >/dev/null
# A way to run a specific stage of generation
if [[ -n "${1}" ]] && type ${1} > /dev/null; then
if [[ -n "${1}" ]] && type "${1}" > /dev/null; then
${1}; exit 0
elif [[ -n "${1}" ]]; then
echo "No such function: ${1}"; exit 1
@ -86,6 +86,6 @@ gen_deps_list
gen_deps_urls
gen_deps_json
REL_DEPS_JSON=$(realpath --relative-to=${PWD} ${DEPS_JSON})
REL_DEPS_JSON=$(realpath --relative-to="${PWD}" "${DEPS_JSON}")
echo -e "${CLR}Generated Nix deps file: ${REL_DEPS_JSON#../}"
echo -e "${GRN}Done${RST}"

View File

@ -1,12 +1,19 @@
#!/usr/bin/env bash
set -Eeu
# This script generates a list of dependencies for the main project and its
# sub-projects defined using Gradle config files. It parses Gradle output of
# 'dependencies' and 'buildEnvironment` tasks using AWK.
set -Eeuo pipefail
GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
# Gradle needs to be run in 'android' subfolder
# Gradle needs to be run in 'android' subfolder.
cd "${GIT_ROOT}/android"
AWK_SCRIPT="${GIT_ROOT}/nix/deps/gradle/gradle_parser.awk"
# Show Gradle log in case of failure.
GRADLE_LOG_FILE='/tmp/gradle.log'
function show_gradle_log() { cat "${GRADLE_LOG_FILE}" >&2; }
trap show_gradle_log ERR
# Run the gradle command for a project:
# - ':buildEnvironment' to get build tools
@ -20,16 +27,11 @@ for i in "${!DEPS[@]}"; do
NORMAL_DEPS[${i}]="${DEPS[${i}]}:dependencies"
done
# And clean up the output by:
# - keep only lines that start with \--- or +---
# - drop lines that end with (*) or (n) but don't start with (+)
# - drop lines that refer to a project
# - drop entries starting with `status-im:` like `status-go`
# - drop entries that aren't just the name of the dependency
# - extract the package name and version, ignoring version range indications,
# such as in `com.google.android.gms:play-services-ads:[15.0.1,16.0.0) -> 15.0.1`
# And clean up the output using AWK script.
AWK_SCRIPT="${GIT_ROOT}/nix/deps/gradle/gradle_parser.awk"
./gradlew --no-daemon --console plain \
"${BUILD_DEPS[@]}" \
"${NORMAL_DEPS[@]}" \
| awk -f ${AWK_SCRIPT}
| tee "${GRADLE_LOG_FILE}" \
| awk -f "${AWK_SCRIPT}"

View File

@ -1,12 +1,22 @@
#!/usr/bin/env bash
set -Eeu
# This script generates a list of Gradle sub-projects by parsing the output
# of Gradle 'projects' task using grep and sed. It is necessary in order to
# collect list of dependencies for main project and its sub-projects.
set -Eeuo pipefail
GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
# Gradle needs to be run in 'android' subfolder
# Gradle needs to be run in 'android' subfolder.
cd "${GIT_ROOT}/android"
# Show Gradle log in case of failure.
GRADLE_LOG_FILE='/tmp/gradle.log'
function show_gradle_log() { cat "${GRADLE_LOG_FILE}" >&2; }
trap show_gradle_log ERR
# Print all our sub-projects
./gradlew projects --no-daemon --console plain 2>&1 \
| tee "${GRADLE_LOG_FILE}" \
| grep "Project ':" \
| sed -E "s;^.--- Project '\:([@_a-zA-Z0-9\-]+)';\1;"

View File

@ -26,6 +26,7 @@ react-native-lottie-splash-screen
react-native-mail
react-native-navigation
react-native-nfc-manager
react-native-orientation-locker
react-native-permissions
react-native-randombytes
react-native-reanimated

View File

@ -6,8 +6,6 @@
# a local Maven repository.
#
CUR_DIR=$(cd "${BASH_SOURCE%/*}" && pwd)
# This defines URLs of Maven repos we know about and use.
declare -a REPOS=(
"https://repo.maven.apache.org/maven2"
@ -37,105 +35,87 @@ function match_repo_url() {
}
function pom_has_nodeps_jar() {
grep '<shadedClassifierName>nodeps</shadedClassifierName>' $1 \
2>&1 >/dev/null
grep '<shadedClassifierName>nodeps</shadedClassifierName>' "${1}" \
>/dev/null 2>&1
}
function fetch_and_template_file() {
local FILENAME="${1}"
local OBJ_URL OBJ_NIX_FETCH_OUT OBJ_NAME OBJ_PATH
OBJ_URL="${REPO_URL}/${PKG_PATH}/${FILENAME}"
if ! OBJ_NIX_FETCH_OUT=$(nix_fetch "${OBJ_URL}"); then
echo " ! Failed to fetch: ${OBJ_URL}" >&2
exit 1
fi
OBJ_NAME="${FILENAME}"
OBJ_PATH=$(get_nix_path "${OBJ_NIX_FETCH_OUT}")
echo -n ",
\"${OBJ_NAME}\": {
\"sha1\": \"$(get_sha1 "${OBJ_PATH}")\",
\"sha256\": \"$(get_nix_sha "${OBJ_NIX_FETCH_OUT}")\"
}"
}
if [[ -z "${1}" ]]; then
echo "Required argument not given!" >&2
echo "Required POM URL argument not given!" >&2
exit 1
fi
POM_URL=${1}
# Drop the POM extension
OBJ_REL_URL=${POM_URL%.pom}
# Drop the POM extension.
PKG_URL_NO_EXT="${POM_URL%.pom}"
# Name of package without extension.
PKG_NAME="$(basename "${PKG_URL_NO_EXT}")"
echo -en "${CLR} - Nix entry for: ${1##*/}\r" >&2
REPO_URL=$(match_repo_url "${OBJ_REL_URL}")
REPO_URL=$(match_repo_url "${PKG_URL_NO_EXT}")
if [[ -z "${REPO_URL}" ]]; then
echo "\r\n ? REPO_URL: ${REPO_URL}" >&2
echo " ! Repo URL not found: %s" "${REPO_URL}" >&2
exit 1
fi
# Get the relative path without full URL
OBJ_REL_NAME="${OBJ_REL_URL#${REPO_URL}/}"
PKG_PATH="${PKG_URL_NO_EXT#"${REPO_URL}/"}"
PKG_PATH="$(dirname "${PKG_PATH}")"
# Both JARs and AARs have a POM
POM_NIX_FETCH_OUT=$(nix_fetch "${OBJ_REL_URL}.pom")
POM_NIX_FETCH_OUT=$(nix_fetch "${PKG_URL_NO_EXT}.pom")
POM_PATH=$(get_nix_path "${POM_NIX_FETCH_OUT}")
POM_NAME=$(basename "${PKG_URL_NO_EXT}.pom")
if [[ -z "${POM_PATH}" ]]; then
echo " ! Failed to fetch: ${OBJ_REL_URL}.pom" >&2
echo " ! Failed to fetch: ${PKG_URL_NO_EXT}.pom" >&2
exit 1
fi
POM_SHA256=$(get_nix_sha "${POM_NIX_FETCH_OUT}")
POM_SHA1=$(get_sha1 "${POM_PATH}")
# Identify packaging type, JAR, AAR, or just POM.
OBJ_TYPE_RAW=$(grep -oP '<packaging>\K[^<]+' "${POM_PATH}")
# Bundle is a JAR made using maven-bundle-plugin.
case "${OBJ_TYPE_RAW}" in
''|'bundle') OBJ_TYPE=jar;;
'aar.asc') OBJ_TYPE=aar;;
*) OBJ_TYPE="${OBJ_TYPE_RAW}"
esac
# Identify packaging type, JAR, AAR, bundle, or just POM.
OBJ_TYPE=$(grep -oP '<packaging>\K[^<]+' "${POM_PATH}")
# Some deps are Eclipse plugins, and we don't need those.
if [[ "${OBJ_TYPE}" == "eclipse-plugin" ]]; then
exit 0
fi
# Some deps are just POMs, in which case there is no JAR to fetch.
if [[ "${OBJ_TYPE}" != "pom" ]]; then
OBJ_NIX_FETCH_OUT=$(nix_fetch "${OBJ_REL_URL}.${OBJ_TYPE}")
# If type was a JAR or other, not getting one is an error.
if [[ ${?} -ne 0 ]]; then
# POMs without packaging type defined can still include JARs.
if [[ "${OBJ_TYPE_RAW}" != "" ]]; then
echo " ! Failed to fetch: ${OBJ_REL_URL}.${OBJ_TYPE}" >&2
exit 1
fi
else
OBJ_PATH=$(get_nix_path "${OBJ_NIX_FETCH_OUT}")
OBJ_SHA256=$(get_nix_sha "${OBJ_NIX_FETCH_OUT}")
OBJ_SHA1=$(get_sha1 "${OBJ_PATH}")
fi
fi
# Some deps include nodeps JARs which include.
if pom_has_nodeps_jar "${POM_PATH}"; then
NODEPS_NIX_FETCH_OUT=$(nix_fetch "${OBJ_REL_URL}-nodeps.jar")
if [[ ${?} -eq 0 ]]; then
NODEPS_PATH=$(get_nix_path "${NODEPS_NIX_FETCH_OUT}")
NODEPS_SHA256=$(get_nix_sha "${NODEPS_NIX_FETCH_OUT}")
NODEPS_SHA1=$(get_sha1 "${NODEPS_PATH}")
fi
fi
# Format into a Nix attrset entry
echo -ne "
{
\"path\": \"${OBJ_REL_NAME}\",
\"host\": \"${REPO_URL}\",
\"type\": \"${OBJ_TYPE}\","
if [[ -n "${POM_SHA256}" ]]; then
echo -n "
\"pom\": {
\"sha1\": \"${POM_SHA1}\",
\"sha256\": \"${POM_SHA256}\"
}";[[ -n "${OBJ_SHA256}" ]] && echo -n ","
fi
if [[ -n "${OBJ_SHA256}" ]]; then
echo -n "
\"jar\": {
\"sha1\": \"${OBJ_SHA1}\",
\"sha256\": \"${OBJ_SHA256}\"
}";[[ -n "${NODEPS_SHA256}" ]] && echo -n ","
fi
if [[ -n "${NODEPS_SHA256}" ]]; then
echo -n "
\"nodeps\": {
\"sha1\": \"${NODEPS_SHA1}\",
\"sha256\": \"${NODEPS_SHA256}\"
}"
fi
echo -e "\n },"
\"path\": \"${PKG_PATH}\",
\"repo\": \"${REPO_URL}\",
\"files\": {
\"${POM_NAME}\": {
\"sha1\": \"${POM_SHA1}\",
\"sha256\": \"${POM_SHA256}\"
}"
# Some deps are just POMs, in which case there is no JAR to fetch.
[[ "${OBJ_TYPE}" == "" ]] && fetch_and_template_file "${PKG_NAME}.jar"
[[ "${OBJ_TYPE}" == "jar" ]] && fetch_and_template_file "${PKG_NAME}.jar"
[[ "${OBJ_TYPE}" == "bundle" ]] && fetch_and_template_file "${PKG_NAME}.jar"
[[ "${OBJ_TYPE}" =~ aar* ]] && fetch_and_template_file "${PKG_NAME}.aar"
[[ "${OBJ_TYPE}" == "aar.asc" ]] && fetch_and_template_file "${PKG_NAME}.${OBJ_TYPE}"
pom_has_nodeps_jar "${POM_PATH}" && fetch_and_template_file "${PKG_NAME}-nodeps.jar"
echo -e '\n }\n },'