diff --git a/tools/sync-nimble-lock.sh b/tools/sync-nimble-lock.sh new file mode 100755 index 000000000..b55826327 --- /dev/null +++ b/tools/sync-nimble-lock.sh @@ -0,0 +1,322 @@ +#!/usr/bin/env bash +# +# sync-nimble-lock.sh +# +# Cross-check git-URL pinned `requires` in waku.nimble against nimble.lock and +# sync the lock entry for any pin that CHANGED relative to a git base ref +# (default: HEAD) -- and ONLY those entries. No other package is touched. +# +# It does NOT run `nimble lock` (which rewrites the whole file and churns +# unrelated packages). Instead it computes the package sha1 checksum itself, +# reproducing nimble's algorithm exactly (src/nimblepkg/checksums.nim): +# +# files = `git ls-files` in the package's git checkout at the pinned rev +# files.sort() # lexicographic +# sha1 = SHA1 over, for each existing regular file (in sorted order): +# update(relative_path_string) +# if symlink: update(symlink_target_string) +# else: update(file_bytes) # 8192-byte chunks +# +# For each changed pin it updates exactly three fields of the matching lock +# entry, preserving all formatting and every other entry byte-for-byte: +# version = "#" + (commit or tag) +# vcsRevision = git rev-parse of the ref (resolves tags) +# checksums.sha1 = the self-computed checksum +# +# The `dependencies` array is intentionally left untouched (see NOTE below). +# +# Usage: +# tools/sync-nimble-lock.sh # dry-run; exit 1 if drift +# tools/sync-nimble-lock.sh --apply # update nimble.lock +# tools/sync-nimble-lock.sh --base origin/master # compare against a ref +# +# Exit codes: 0 = in sync / applied, 1 = drift (dry-run), 2 = usage/tooling error +# +# Portable across macOS (bash 3.2, BSD tools) and Linux: all logic is in +# python3; bash only parses args and checks tools. Requires: git, python3. +# +# NOTE on `dependencies`: a version bump can in principle change a package's +# direct dependency set. Reproducing nimble's dependency-name normalization +# without running nimble is fragile, and the user-requested scope is +# version/vcsRevision/sha1. If a bumped dependency added/removed a `requires`, +# update its lock `dependencies` array by hand. The script warns when the +# bumped package's own .nimble `requires` count differs from the lock entry. + +set -euo pipefail + +APPLY=0 +BASE="HEAD" + +usage() { sed -n '2,55p' "$0" | sed 's/^#\{0,1\} \{0,1\}//'; } + +while [ $# -gt 0 ]; do + case "$1" in + --apply) APPLY=1 ;; + --base) shift; [ $# -gt 0 ] || { echo "error: --base needs a ref" >&2; exit 2; }; BASE="$1" ;; + --base=*) BASE="${1#*=}" ;; + -h|--help) usage; exit 0 ;; + *) echo "error: unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +command -v python3 >/dev/null 2>&1 || { echo "error: python3 is required" >&2; exit 2; } +command -v git >/dev/null 2>&1 || { echo "error: git is required" >&2; exit 2; } + +ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { echo "error: not in a git repo" >&2; exit 2; } + +export SYNC_ROOT="$ROOT" SYNC_APPLY="$APPLY" SYNC_BASE="$BASE" SYNC_PKGCACHE="${HOME}/.nimble/pkgcache" + +exec python3 - <<'PYEOF' +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +ROOT = os.environ["SYNC_ROOT"] +APPLY = os.environ["SYNC_APPLY"] == "1" +BASE = os.environ["SYNC_BASE"] +PKGCACHE = os.environ["SYNC_PKGCACHE"] + +NIMBLE_FILE = os.path.join(ROOT, "waku.nimble") +LOCK_FILE = os.path.join(ROOT, "nimble.lock") + +REQ_RE = re.compile(r'requires\s+"(https?://[^"#]+)#([^"]+)"') +COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +NEAR_HASH_RE = re.compile(r"^[0-9a-fx]{38,42}$") # catches the leading-`x` typo + + +def fail(msg): + sys.stderr.write("error: %s\n" % msg) + sys.exit(2) + + +def warn(msg): + sys.stderr.write("warning: %s\n" % msg) + + +def norm_url(url): + u = url.rstrip("/") + return u[:-4] if u.endswith(".git") else u + + +def git(args, cwd=None, check=True): + r = subprocess.run(["git"] + args, cwd=cwd, capture_output=True, text=True) + if check and r.returncode != 0: + fail("git %s failed: %s" % (" ".join(args), (r.stderr or r.stdout).strip())) + return r + + +# --------------------------------------------------------------------------- +# nimble checksum reproduction (verified byte-for-byte against nimble v0.22.3) +# --------------------------------------------------------------------------- +def compute_checksum(checkout_dir): + out = git(["-C", checkout_dir, "ls-files"]).stdout + files = out.strip().splitlines() + files.sort() + h = hashlib.sha1() + for rel in files: + path = os.path.join(checkout_dir, rel) + if not os.path.isfile(path): + # Skips directories / gitlinks / broken symlinks, matching nimble's + # `fileExists` guard (regular file or symlink-to-file only). + continue + h.update(rel.encode("utf-8")) + if os.path.islink(path): + h.update(os.readlink(path).encode("utf-8")) + else: + with open(path, "rb") as fh: + while True: + chunk = fh.read(8192) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def get_checkout(url, rev, tmpdir): + """Return (checkout_dir, cleanup_fn). Reuses ~/.nimble/pkgcache when the + exact commit is already cloned; otherwise clones from the URL.""" + # pkgcache dirs are suffixed with the commit sha (commit pins only). + if os.path.isdir(PKGCACHE): + for name in os.listdir(PKGCACHE): + if name.endswith("_" + rev) and os.path.isdir(os.path.join(PKGCACHE, name, ".git")): + cache = os.path.join(PKGCACHE, name) + git(["-C", cache, "checkout", "-q", rev]) + return cache, (lambda: None) + # Fall back to a fresh clone (network). Full clone, then checkout the ref. + dest = os.path.join(tmpdir, "clone") + print(" cloning %s ..." % url) + git(["clone", "--quiet", url, dest]) + r = git(["-C", dest, "checkout", "-q", rev], check=False) + if r.returncode != 0: + # commit may live on a ref not fetched by default; try fetching it + git(["-C", dest, "fetch", "--quiet", "origin", rev], check=False) + git(["-C", dest, "checkout", "-q", rev]) + return dest, (lambda: shutil.rmtree(dest, ignore_errors=True)) + + +def dep_requires_count(checkout_dir): + """Best-effort count of git/registry `requires` in the dep's .nimble file, + for a heads-up if the lock `dependencies` array may be stale.""" + nimbles = [f for f in os.listdir(checkout_dir) if f.endswith(".nimble")] + if not nimbles: + return None + try: + txt = open(os.path.join(checkout_dir, nimbles[0])).read() + except OSError: + return None + n = 0 + for m in re.finditer(r'requires\s+"([^"]+)"', txt): + n += len([p for p in m.group(1).split(",") if p.strip()]) + return n or None + + +# --------------------------------------------------------------------------- +# detect changes +# --------------------------------------------------------------------------- +def parse_changed(base): + r = git(["-C", ROOT, "diff", base, "--", "waku.nimble"], check=False) + if r.returncode != 0: + fail("git diff against %r failed: %s" % (base, r.stderr.strip())) + changed, seen = [], set() + for line in r.stdout.splitlines(): + if not line.startswith("+") or line.startswith("+++"): + continue + m = REQ_RE.search(line[1:]) + if not m: + continue + url, rev = m.group(1), m.group(2) + key = norm_url(url) + if key in seen: + continue + seen.add(key) + if not COMMIT_RE.match(rev) and NEAR_HASH_RE.match(rev): + fail("invalid commit hash for %s: %r is not a valid 40-char hex SHA " + "(stray character / typo?)" % (url, rev)) + changed.append((url, rev)) + return changed + + +# --------------------------------------------------------------------------- +# surgical lock patch (text-level: preserves formatting & all other entries) +# --------------------------------------------------------------------------- +PKG_OPEN_RE = re.compile(r'^\s{4}"[^"]+":\s*\{\s*$') +PKG_CLOSE_RE = re.compile(r'^\s{4}\},?\s*$') + + +def set_value(line, key, val): + return re.sub(r'(^\s*"' + re.escape(key) + r'":\s*")[^"]*(")', + lambda m: m.group(1) + val + m.group(2), line, count=1) + + +def patch_lock_text(text, url, version, vcs_rev, sha1): + lines = text.splitlines(keepends=True) + url_re = re.compile(r'^\s*"url":\s*"' + re.escape(url) + r'"\s*,?\s*$') + ui = next((i for i, l in enumerate(lines) if url_re.match(l)), None) + if ui is None: + return None + # block bounds + start = next(i for i in range(ui, -1, -1) if PKG_OPEN_RE.match(lines[i])) + end = next(i for i in range(ui, len(lines)) if PKG_CLOSE_RE.match(lines[i])) + done = set() + for i in range(start, end + 1): + if "version" not in done and re.match(r'^\s*"version":', lines[i]): + lines[i] = set_value(lines[i], "version", version); done.add("version") + elif "vcsRevision" not in done and re.match(r'^\s*"vcsRevision":', lines[i]): + lines[i] = set_value(lines[i], "vcsRevision", vcs_rev); done.add("vcsRevision") + elif "sha1" not in done and re.match(r'^\s*"sha1":', lines[i]): + lines[i] = set_value(lines[i], "sha1", sha1); done.add("sha1") + missing = {"version", "vcsRevision", "sha1"} - done + if missing: + fail("could not locate field(s) %s in lock block for %s" % (sorted(missing), url)) + return "".join(lines) + + +# --------------------------------------------------------------------------- +def main(): + for p in (NIMBLE_FILE, LOCK_FILE): + if not os.path.isfile(p): + fail("%s not found" % p) + + changed = parse_changed(BASE) + if not changed: + print("No changed git-URL `requires` in waku.nimble vs %s — nothing to sync." % BASE) + return 0 + + lock = json.load(open(LOCK_FILE)) + by_url = {} + for name, e in lock.get("packages", {}).items(): + if e.get("url"): + by_url[norm_url(e["url"])] = (name, e) + + drift = [] # (url, rev, name_or_None, cur_version_or_None) + for url, rev in changed: + hit = by_url.get(norm_url(url)) + want = "#" + rev + if hit is None: + drift.append((url, rev, None, None)) + elif hit[1].get("version") != want: + drift.append((url, rev, hit[0], hit[1].get("version"))) + + if not drift: + print("nimble.lock already in sync with waku.nimble (%d changed pin(s) checked)." % len(changed)) + return 0 + + print("Dependency drift (waku.nimble vs nimble.lock):") + for url, rev, name, cur in drift: + tag = name or "(missing)" + print(" ~ %s [%s]\n waku.nimble: #%s\n nimble.lock: %s" % (url, tag, rev, cur)) + + if not APPLY: + print("\nRun with --apply to update nimble.lock (computes checksum itself; no `nimble lock`).") + return 1 + + print("\nApplying (computing checksums; not running `nimble lock`)...") + text = open(LOCK_FILE).read() + updated = [] + tmproot = tempfile.mkdtemp(prefix="sync-nimble-lock.") + try: + for url, rev, name, _cur in drift: + if name is None: + fail("%s has no entry in nimble.lock; this script updates existing " + "entries only (add new deps with a normal nimble install first)." % url) + sub = os.path.join(tmproot, re.sub(r"\W+", "_", norm_url(url))) + os.makedirs(sub, exist_ok=True) + checkout, cleanup = get_checkout(url, rev, sub) + try: + vcs_rev = git(["-C", checkout, "rev-parse", "HEAD"]).stdout.strip() + sha1 = compute_checksum(checkout) + # dependency-drift heads-up + cnt = dep_requires_count(checkout) + lock_deps = len(by_url[norm_url(url)][1].get("dependencies", [])) + if cnt is not None and lock_deps and cnt != lock_deps: + warn("%s: .nimble has %d `requires` but lock lists %d dependencies; " + "review the `dependencies` array manually." % (name, cnt, lock_deps)) + finally: + cleanup() + new_text = patch_lock_text(text, url, "#" + rev, vcs_rev, sha1) + if new_text is None: + fail("could not find lock block for url %s" % url) + text = new_text + updated.append((name, "#" + rev, vcs_rev, sha1)) + finally: + shutil.rmtree(tmproot, ignore_errors=True) + + with open(LOCK_FILE, "w") as f: + f.write(text) + + print("\nUpdated nimble.lock (only these entries; all others untouched):") + for name, ver, vcs, sha1 in updated: + print(" %-16s version=%s" % (name, ver)) + print(" %-16s vcsRevision=%s" % ("", vcs)) + print(" %-16s sha1=%s" % ("", sha1)) + return 0 + + +sys.exit(main()) +PYEOF