mirror of
https://github.com/logos-messaging/logos-delivery.git
synced 2026-06-03 04:29:33 +00:00
tools: add sync-nimble-lock.sh to cross-check waku.nimble pins into nimble.lock (#3924)
Adds a portable (macOS bash 3.2 / Linux) helper that detects git-URL pinned `requires` in waku.nimble which changed vs a git base ref (default HEAD) and updates ONLY those nimble.lock entries — version, vcsRevision and the sha1 checksum — leaving every other entry byte-for-byte untouched. It does not run `nimble lock` (which rewrites the whole file). The sha1 is computed directly, reproducing nimble's algorithm from src/nimblepkg/checksums.nim (git ls-files -> sort -> SHA1 over path + symlink-target/file-bytes). Resolves tags to commits via git rev-parse and guards against invalid commit hashes (e.g. a stray leading character). Dry-run by default (exit 1 on drift); --apply writes; --base REF to compare against another ref. Requires git + python3; nimble not required. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
NagyZoltanPeter
GitHub
parent
8b0e21fada
commit
b593d16d11
322
tools/sync-nimble-lock.sh
Executable file
322
tools/sync-nimble-lock.sh
Executable file
@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# sync-nimble-lock.sh
|
||||
#
|
||||
# Cross-check git-URL pinned `requires` in waku.nimble against nimble.lock and
|
||||
# sync the lock entry for any pin that CHANGED relative to a git base ref
|
||||
# (default: HEAD) -- and ONLY those entries. No other package is touched.
|
||||
#
|
||||
# It does NOT run `nimble lock` (which rewrites the whole file and churns
|
||||
# unrelated packages). Instead it computes the package sha1 checksum itself,
|
||||
# reproducing nimble's algorithm exactly (src/nimblepkg/checksums.nim):
|
||||
#
|
||||
# files = `git ls-files` in the package's git checkout at the pinned rev
|
||||
# files.sort() # lexicographic
|
||||
# sha1 = SHA1 over, for each existing regular file (in sorted order):
|
||||
# update(relative_path_string)
|
||||
# if symlink: update(symlink_target_string)
|
||||
# else: update(file_bytes) # 8192-byte chunks
|
||||
#
|
||||
# For each changed pin it updates exactly three fields of the matching lock
|
||||
# entry, preserving all formatting and every other entry byte-for-byte:
|
||||
# version = "#" + <rev-as-written-in-waku.nimble> (commit or tag)
|
||||
# vcsRevision = git rev-parse of the ref (resolves tags)
|
||||
# checksums.sha1 = the self-computed checksum
|
||||
#
|
||||
# The `dependencies` array is intentionally left untouched (see NOTE below).
|
||||
#
|
||||
# Usage:
|
||||
# tools/sync-nimble-lock.sh # dry-run; exit 1 if drift
|
||||
# tools/sync-nimble-lock.sh --apply # update nimble.lock
|
||||
# tools/sync-nimble-lock.sh --base origin/master # compare against a ref
|
||||
#
|
||||
# Exit codes: 0 = in sync / applied, 1 = drift (dry-run), 2 = usage/tooling error
|
||||
#
|
||||
# Portable across macOS (bash 3.2, BSD tools) and Linux: all logic is in
|
||||
# python3; bash only parses args and checks tools. Requires: git, python3.
|
||||
#
|
||||
# NOTE on `dependencies`: a version bump can in principle change a package's
|
||||
# direct dependency set. Reproducing nimble's dependency-name normalization
|
||||
# without running nimble is fragile, and the user-requested scope is
|
||||
# version/vcsRevision/sha1. If a bumped dependency added/removed a `requires`,
|
||||
# update its lock `dependencies` array by hand. The script warns when the
|
||||
# bumped package's own .nimble `requires` count differs from the lock entry.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
APPLY=0
|
||||
BASE="HEAD"
|
||||
|
||||
usage() { sed -n '2,55p' "$0" | sed 's/^#\{0,1\} \{0,1\}//'; }
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--apply) APPLY=1 ;;
|
||||
--base) shift; [ $# -gt 0 ] || { echo "error: --base needs a ref" >&2; exit 2; }; BASE="$1" ;;
|
||||
--base=*) BASE="${1#*=}" ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "error: unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
command -v python3 >/dev/null 2>&1 || { echo "error: python3 is required" >&2; exit 2; }
|
||||
command -v git >/dev/null 2>&1 || { echo "error: git is required" >&2; exit 2; }
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" || { echo "error: not in a git repo" >&2; exit 2; }
|
||||
|
||||
export SYNC_ROOT="$ROOT" SYNC_APPLY="$APPLY" SYNC_BASE="$BASE" SYNC_PKGCACHE="${HOME}/.nimble/pkgcache"
|
||||
|
||||
exec python3 - <<'PYEOF'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ROOT = os.environ["SYNC_ROOT"]
|
||||
APPLY = os.environ["SYNC_APPLY"] == "1"
|
||||
BASE = os.environ["SYNC_BASE"]
|
||||
PKGCACHE = os.environ["SYNC_PKGCACHE"]
|
||||
|
||||
NIMBLE_FILE = os.path.join(ROOT, "waku.nimble")
|
||||
LOCK_FILE = os.path.join(ROOT, "nimble.lock")
|
||||
|
||||
REQ_RE = re.compile(r'requires\s+"(https?://[^"#]+)#([^"]+)"')
|
||||
COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
NEAR_HASH_RE = re.compile(r"^[0-9a-fx]{38,42}$") # catches the leading-`x` typo
|
||||
|
||||
|
||||
def fail(msg):
|
||||
sys.stderr.write("error: %s\n" % msg)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def warn(msg):
|
||||
sys.stderr.write("warning: %s\n" % msg)
|
||||
|
||||
|
||||
def norm_url(url):
|
||||
u = url.rstrip("/")
|
||||
return u[:-4] if u.endswith(".git") else u
|
||||
|
||||
|
||||
def git(args, cwd=None, check=True):
|
||||
r = subprocess.run(["git"] + args, cwd=cwd, capture_output=True, text=True)
|
||||
if check and r.returncode != 0:
|
||||
fail("git %s failed: %s" % (" ".join(args), (r.stderr or r.stdout).strip()))
|
||||
return r
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# nimble checksum reproduction (verified byte-for-byte against nimble v0.22.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
def compute_checksum(checkout_dir):
|
||||
out = git(["-C", checkout_dir, "ls-files"]).stdout
|
||||
files = out.strip().splitlines()
|
||||
files.sort()
|
||||
h = hashlib.sha1()
|
||||
for rel in files:
|
||||
path = os.path.join(checkout_dir, rel)
|
||||
if not os.path.isfile(path):
|
||||
# Skips directories / gitlinks / broken symlinks, matching nimble's
|
||||
# `fileExists` guard (regular file or symlink-to-file only).
|
||||
continue
|
||||
h.update(rel.encode("utf-8"))
|
||||
if os.path.islink(path):
|
||||
h.update(os.readlink(path).encode("utf-8"))
|
||||
else:
|
||||
with open(path, "rb") as fh:
|
||||
while True:
|
||||
chunk = fh.read(8192)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def get_checkout(url, rev, tmpdir):
|
||||
"""Return (checkout_dir, cleanup_fn). Reuses ~/.nimble/pkgcache when the
|
||||
exact commit is already cloned; otherwise clones from the URL."""
|
||||
# pkgcache dirs are suffixed with the commit sha (commit pins only).
|
||||
if os.path.isdir(PKGCACHE):
|
||||
for name in os.listdir(PKGCACHE):
|
||||
if name.endswith("_" + rev) and os.path.isdir(os.path.join(PKGCACHE, name, ".git")):
|
||||
cache = os.path.join(PKGCACHE, name)
|
||||
git(["-C", cache, "checkout", "-q", rev])
|
||||
return cache, (lambda: None)
|
||||
# Fall back to a fresh clone (network). Full clone, then checkout the ref.
|
||||
dest = os.path.join(tmpdir, "clone")
|
||||
print(" cloning %s ..." % url)
|
||||
git(["clone", "--quiet", url, dest])
|
||||
r = git(["-C", dest, "checkout", "-q", rev], check=False)
|
||||
if r.returncode != 0:
|
||||
# commit may live on a ref not fetched by default; try fetching it
|
||||
git(["-C", dest, "fetch", "--quiet", "origin", rev], check=False)
|
||||
git(["-C", dest, "checkout", "-q", rev])
|
||||
return dest, (lambda: shutil.rmtree(dest, ignore_errors=True))
|
||||
|
||||
|
||||
def dep_requires_count(checkout_dir):
|
||||
"""Best-effort count of git/registry `requires` in the dep's .nimble file,
|
||||
for a heads-up if the lock `dependencies` array may be stale."""
|
||||
nimbles = [f for f in os.listdir(checkout_dir) if f.endswith(".nimble")]
|
||||
if not nimbles:
|
||||
return None
|
||||
try:
|
||||
txt = open(os.path.join(checkout_dir, nimbles[0])).read()
|
||||
except OSError:
|
||||
return None
|
||||
n = 0
|
||||
for m in re.finditer(r'requires\s+"([^"]+)"', txt):
|
||||
n += len([p for p in m.group(1).split(",") if p.strip()])
|
||||
return n or None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# detect changes
|
||||
# ---------------------------------------------------------------------------
|
||||
def parse_changed(base):
|
||||
r = git(["-C", ROOT, "diff", base, "--", "waku.nimble"], check=False)
|
||||
if r.returncode != 0:
|
||||
fail("git diff against %r failed: %s" % (base, r.stderr.strip()))
|
||||
changed, seen = [], set()
|
||||
for line in r.stdout.splitlines():
|
||||
if not line.startswith("+") or line.startswith("+++"):
|
||||
continue
|
||||
m = REQ_RE.search(line[1:])
|
||||
if not m:
|
||||
continue
|
||||
url, rev = m.group(1), m.group(2)
|
||||
key = norm_url(url)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if not COMMIT_RE.match(rev) and NEAR_HASH_RE.match(rev):
|
||||
fail("invalid commit hash for %s: %r is not a valid 40-char hex SHA "
|
||||
"(stray character / typo?)" % (url, rev))
|
||||
changed.append((url, rev))
|
||||
return changed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# surgical lock patch (text-level: preserves formatting & all other entries)
|
||||
# ---------------------------------------------------------------------------
|
||||
PKG_OPEN_RE = re.compile(r'^\s{4}"[^"]+":\s*\{\s*$')
|
||||
PKG_CLOSE_RE = re.compile(r'^\s{4}\},?\s*$')
|
||||
|
||||
|
||||
def set_value(line, key, val):
|
||||
return re.sub(r'(^\s*"' + re.escape(key) + r'":\s*")[^"]*(")',
|
||||
lambda m: m.group(1) + val + m.group(2), line, count=1)
|
||||
|
||||
|
||||
def patch_lock_text(text, url, version, vcs_rev, sha1):
|
||||
lines = text.splitlines(keepends=True)
|
||||
url_re = re.compile(r'^\s*"url":\s*"' + re.escape(url) + r'"\s*,?\s*$')
|
||||
ui = next((i for i, l in enumerate(lines) if url_re.match(l)), None)
|
||||
if ui is None:
|
||||
return None
|
||||
# block bounds
|
||||
start = next(i for i in range(ui, -1, -1) if PKG_OPEN_RE.match(lines[i]))
|
||||
end = next(i for i in range(ui, len(lines)) if PKG_CLOSE_RE.match(lines[i]))
|
||||
done = set()
|
||||
for i in range(start, end + 1):
|
||||
if "version" not in done and re.match(r'^\s*"version":', lines[i]):
|
||||
lines[i] = set_value(lines[i], "version", version); done.add("version")
|
||||
elif "vcsRevision" not in done and re.match(r'^\s*"vcsRevision":', lines[i]):
|
||||
lines[i] = set_value(lines[i], "vcsRevision", vcs_rev); done.add("vcsRevision")
|
||||
elif "sha1" not in done and re.match(r'^\s*"sha1":', lines[i]):
|
||||
lines[i] = set_value(lines[i], "sha1", sha1); done.add("sha1")
|
||||
missing = {"version", "vcsRevision", "sha1"} - done
|
||||
if missing:
|
||||
fail("could not locate field(s) %s in lock block for %s" % (sorted(missing), url))
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
def main():
|
||||
for p in (NIMBLE_FILE, LOCK_FILE):
|
||||
if not os.path.isfile(p):
|
||||
fail("%s not found" % p)
|
||||
|
||||
changed = parse_changed(BASE)
|
||||
if not changed:
|
||||
print("No changed git-URL `requires` in waku.nimble vs %s — nothing to sync." % BASE)
|
||||
return 0
|
||||
|
||||
lock = json.load(open(LOCK_FILE))
|
||||
by_url = {}
|
||||
for name, e in lock.get("packages", {}).items():
|
||||
if e.get("url"):
|
||||
by_url[norm_url(e["url"])] = (name, e)
|
||||
|
||||
drift = [] # (url, rev, name_or_None, cur_version_or_None)
|
||||
for url, rev in changed:
|
||||
hit = by_url.get(norm_url(url))
|
||||
want = "#" + rev
|
||||
if hit is None:
|
||||
drift.append((url, rev, None, None))
|
||||
elif hit[1].get("version") != want:
|
||||
drift.append((url, rev, hit[0], hit[1].get("version")))
|
||||
|
||||
if not drift:
|
||||
print("nimble.lock already in sync with waku.nimble (%d changed pin(s) checked)." % len(changed))
|
||||
return 0
|
||||
|
||||
print("Dependency drift (waku.nimble vs nimble.lock):")
|
||||
for url, rev, name, cur in drift:
|
||||
tag = name or "(missing)"
|
||||
print(" ~ %s [%s]\n waku.nimble: #%s\n nimble.lock: %s" % (url, tag, rev, cur))
|
||||
|
||||
if not APPLY:
|
||||
print("\nRun with --apply to update nimble.lock (computes checksum itself; no `nimble lock`).")
|
||||
return 1
|
||||
|
||||
print("\nApplying (computing checksums; not running `nimble lock`)...")
|
||||
text = open(LOCK_FILE).read()
|
||||
updated = []
|
||||
tmproot = tempfile.mkdtemp(prefix="sync-nimble-lock.")
|
||||
try:
|
||||
for url, rev, name, _cur in drift:
|
||||
if name is None:
|
||||
fail("%s has no entry in nimble.lock; this script updates existing "
|
||||
"entries only (add new deps with a normal nimble install first)." % url)
|
||||
sub = os.path.join(tmproot, re.sub(r"\W+", "_", norm_url(url)))
|
||||
os.makedirs(sub, exist_ok=True)
|
||||
checkout, cleanup = get_checkout(url, rev, sub)
|
||||
try:
|
||||
vcs_rev = git(["-C", checkout, "rev-parse", "HEAD"]).stdout.strip()
|
||||
sha1 = compute_checksum(checkout)
|
||||
# dependency-drift heads-up
|
||||
cnt = dep_requires_count(checkout)
|
||||
lock_deps = len(by_url[norm_url(url)][1].get("dependencies", []))
|
||||
if cnt is not None and lock_deps and cnt != lock_deps:
|
||||
warn("%s: .nimble has %d `requires` but lock lists %d dependencies; "
|
||||
"review the `dependencies` array manually." % (name, cnt, lock_deps))
|
||||
finally:
|
||||
cleanup()
|
||||
new_text = patch_lock_text(text, url, "#" + rev, vcs_rev, sha1)
|
||||
if new_text is None:
|
||||
fail("could not find lock block for url %s" % url)
|
||||
text = new_text
|
||||
updated.append((name, "#" + rev, vcs_rev, sha1))
|
||||
finally:
|
||||
shutil.rmtree(tmproot, ignore_errors=True)
|
||||
|
||||
with open(LOCK_FILE, "w") as f:
|
||||
f.write(text)
|
||||
|
||||
print("\nUpdated nimble.lock (only these entries; all others untouched):")
|
||||
for name, ver, vcs, sha1 in updated:
|
||||
print(" %-16s version=%s" % (name, ver))
|
||||
print(" %-16s vcsRevision=%s" % ("", vcs))
|
||||
print(" %-16s sha1=%s" % ("", sha1))
|
||||
return 0
|
||||
|
||||
|
||||
sys.exit(main())
|
||||
PYEOF
|
||||
Loading…
x
Reference in New Issue
Block a user