diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml new file mode 100644 index 0000000..563837e --- /dev/null +++ b/.github/workflows/build-and-publish.yml @@ -0,0 +1,232 @@ +name: Build and Publish + +on: + workflow_dispatch: + inputs: + include: + description: 'Comma-separated list of modules to build (leave empty to build all)' + required: false + default: '' + exclude: + description: 'Comma-separated list of modules to exclude from build' + required: false + default: '' + +permissions: + contents: write + +jobs: + discover: + runs-on: ubuntu-latest + outputs: + modules: ${{ steps.modules.outputs.modules }} + steps: + - uses: actions/checkout@v6 + + - name: List modules + id: modules + run: | + # Build module list with flake references from submodule URLs and pinned commits + all_modules=$(python3 <<'PY' + import json, subprocess, re + + lines = subprocess.check_output( + ["git", "config", "--file", ".gitmodules", "--get-regexp", r"submodule\..*\.path"], + text=True + ).strip().split("\n") + + result = [] + for line in lines: + if not line.strip(): + continue + key, path = line.split(None, 1) + name = re.match(r"submodule\.(.*)\.path", key).group(1) + url = subprocess.check_output( + ["git", "config", "--file", ".gitmodules", f"submodule.{name}.url"], + text=True + ).strip() + commit = subprocess.check_output( + ["git", "ls-tree", "HEAD", path], + text=True + ).strip().split()[2] + + # Convert GitHub URL to flake reference + m = re.match(r"https?://github\.com/(.+?)(?:\.git)?$", url) + if not m: + m = re.match(r"git@github\.com:(.+?)(?:\.git)?$", url) + if m: + flake = f"github:{m.group(1)}/{commit}" + else: + flake = f"git+{url}?rev={commit}" + + result.append({"path": path, "flake": flake}) + + print(json.dumps(result)) + PY + ) + + include_input="${{ inputs.include }}" + exclude_input="${{ inputs.exclude }}" + + if [[ -n "$include_input" ]]; then + include_json=$(echo "$include_input" | tr ',' '\n' | sed 's/^ *//;s/ *$//' | jq -R -s -c 'split("\n") | map(select(length > 0))') + modules=$(echo "$all_modules" | jq -c --argjson inc "$include_json" '[.[] | select(.path as $p | $inc | any(. == $p))]') + elif [[ -n "$exclude_input" ]]; then + exclude_json=$(echo "$exclude_input" | tr ',' '\n' | sed 's/^ *//;s/ *$//' | jq -R -s -c 'split("\n") | map(select(length > 0))') + modules=$(echo "$all_modules" | jq -c --argjson exc "$exclude_json" '[.[] | select(.path as $p | $exc | any(. == $p) | not)]') + else + modules="$all_modules" + fi + + echo "Building modules: $modules" + echo "modules=$modules" >> "$GITHUB_OUTPUT" + + build: + needs: discover + strategy: + fail-fast: false + matrix: + variant: + - os: ubuntu-24.04 + name: linux-amd64 + - os: ubuntu-24.04-arm + name: linux-arm64 + - os: macos-15 + name: darwin-arm64 + module: ${{ fromJson(needs.discover.outputs.modules) }} + + runs-on: ${{ matrix.variant.os }} + continue-on-error: true + + steps: + - uses: DeterminateSystems/nix-installer-action@main + + - uses: DeterminateSystems/magic-nix-cache-action@main + + - name: Build ${{ matrix.module.path }} + id: build + run: | + nix bundle --bundler github:logos-co/nix-bundle-lgx#portable -o result '${{ matrix.module.flake }}#lib' + + - name: Collect output + if: steps.build.outcome == 'success' + run: | + mkdir -p "lgx-output/${{ matrix.variant.name }}/${{ matrix.module.path }}" + cp result/*.lgx "lgx-output/${{ matrix.variant.name }}/${{ matrix.module.path }}/" + + - uses: actions/upload-artifact@v6 + if: steps.build.outcome == 'success' + with: + name: lgx-${{ matrix.variant.name }}--${{ matrix.module.path }} + path: lgx-output/ + retention-days: 1 + + package: + needs: [discover, build] + if: always() + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - uses: DeterminateSystems/nix-installer-action@main + + - uses: DeterminateSystems/magic-nix-cache-action@main + + - name: Build lgx tool + run: | + nix build .#lgx + echo "LGX=$(readlink -f result/bin/lgx)" >> "$GITHUB_ENV" + + - uses: actions/download-artifact@v7 + with: + path: artifacts + pattern: lgx-* + merge-multiple: true + + - name: Package modules + run: | + export ARTIFACTS_DIR="artifacts" + bash ci/package-modules.sh + + - name: Prepare release tag + id: tag + run: | + short_sha="${GITHUB_SHA::7}" + date_str="$(date -u +%Y%m%d)" + tag="build-${date_str}-${short_sha}-${GITHUB_RUN_NUMBER}" + name="Build $(date -u +%Y-%m-%d) (${short_sha}) #${GITHUB_RUN_NUMBER}" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "name=$name" >> "$GITHUB_OUTPUT" + + include="${{ inputs.include }}" + exclude="${{ inputs.exclude }}" + if [[ -n "$include" ]]; then + config="Modules: \`${include}\`" + elif [[ -n "$exclude" ]]; then + config="Modules: all (excluding: \`${exclude}\`)" + else + config="Modules: all" + fi + echo "config=$config" >> "$GITHUB_OUTPUT" + + - name: Generate release table + id: table + env: + RELEASE_TAG: ${{ steps.tag.outputs.tag }} + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json, os + + with open("output/list.json") as f: + modules = json.load(f) + + repo = os.environ.get("GITHUB_REPOSITORY", "") + tag = os.environ.get("RELEASE_TAG", "") + + all_variants = ["linux-amd64", "linux-arm64", "darwin-arm64"] + header = "| Module | " + " | ".join(all_variants) + " |" + sep = "|--------|" + "|".join([":---:" for _ in all_variants]) + "|" + + rows = [] + for m in modules: + name = m.get("moduleName", m.get("name", "")) + version = m.get("version", "") + label = f"{name} v{version}" if version else name + package = m.get("package", "") + if repo and tag and package: + url = f"https://github.com/{repo}/releases/download/{tag}/{package}" + label = f"[{label}]({url})" + variants = set(m.get("variants", [])) + cells = " | ".join("\u2705" if v in variants else "\u274c" for v in all_variants) + rows.append(f"| {label} | {cells} |") + + table = "\n".join([header, sep] + rows) + + delim = "EOF_TABLE" + print(f"table<<{delim}") + print(table) + print(delim) + PY + + - name: Create GitHub release + if: github.ref == 'refs/heads/master' + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.tag }} + name: ${{ steps.tag.outputs.name }} + body: | + Automated build from ${{ github.sha }} + + ${{ steps.tag.outputs.config }} + + ## Module Availability + + ${{ steps.table.outputs.table }} + files: | + output/*.lgx + output/list.json + draft: false + prerelease: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-libraries.yml b/.github/workflows/publish-libraries.yml deleted file mode 100644 index 6b083ae..0000000 --- a/.github/workflows/publish-libraries.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Publish Libraries - -on: - push: - branches: [outputs] - -permissions: - contents: write - -jobs: - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Gather library files - id: gather - run: | - mkdir -p upload - cp -a libraries/list.json upload/ 2>/dev/null || true - find "libraries" -maxdepth 1 -name "*.lgx" -type f -exec cp {} upload/ \; - files_count=$(find upload -maxdepth 1 -type f | wc -l) - echo "files=$files_count" >> "$GITHUB_OUTPUT" - if [ "$files_count" -eq 0 ]; then - echo "No library files found to upload" >&2 - exit 1 - fi - - - name: Publish release assets - uses: softprops/action-gh-release@v1 - with: - tag_name: outputs-libraries - name: Libraries - body: "Libraries from branch outputs. Each asset is an individual file." - files: upload/* - draft: false - prerelease: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitmodules b/.gitmodules index 2cf91cb..d6ea3db 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,9 +10,9 @@ [submodule "logos-irc-module"] path = logos-irc-module url = https://github.com/logos-co/logos-irc-module -[submodule "logos-package-manager"] - path = logos-package-manager - url = https://github.com/logos-co/logos-package-manager +[submodule "logos-package-manager-module"] + path = logos-package-manager-module + url = https://github.com/logos-co/logos-package-manager-module [submodule "logos-capability-module"] path = logos-capability-module url = https://github.com/logos-co/logos-capability-module @@ -31,3 +31,21 @@ [submodule "logos-chatsdk-ui"] path = logos-chatsdk-ui url = https://github.com/logos-co/logos-chatsdk-ui +[submodule "logos-blockchain-module"] + path = logos-blockchain-module + url = https://github.com/logos-blockchain/logos-blockchain-module +[submodule "logos-execution-zone-module"] + path = logos-execution-zone-module + url = https://github.com/logos-blockchain/logos-execution-zone-module +[submodule "logos-blockchain-ui"] + path = logos-blockchain-ui + url = https://github.com/logos-blockchain/logos-blockchain-ui +[submodule "logos-storage-module"] + path = logos-storage-module + url = https://github.com/logos-co/logos-storage-module +[submodule "logos-delivery-module"] + path = logos-delivery-module + url = https://github.com/logos-co/logos-delivery-module.git +[submodule "logos-storage-ui"] + path = logos-storage-ui + url = https://github.com/logos-co/logos-storage-ui diff --git a/README.md b/README.md index 1f004be..238d4c0 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,19 @@ to compile all modules in one go | logos-wallet-module | https://github.com/logos-co/logos-wallet-module | | logos-chat-module | https://github.com/logos-co/logos-chat-module | | logos-irc-module | https://github.com/logos-co/logos-irc-module | -| logos-package-manager | https://github.com/logos-co/logos-package-manager | +| logos-package-manager-module | https://github.com/logos-co/logos-package-manager-module | | logos-capability-module | https://github.com/logos-co/logos-capability-module | | logos-accounts-module | https://github.com/logos-co/logos-accounts-module | | logos-wallet-ui | https://github.com/logos-co/logos-wallet-ui | | logos-chat-ui | https://github.com/logos-co/logos-chat-ui | | logos-accounts-ui | https://github.com/logos-co/logos-accounts-ui | | logos-chatsdk-ui | https://github.com/logos-co/logos-chatsdk-ui | +| logos-blockchain | https://github.com/logos-blockchain/logos-blockchain-module | +| logos-execution-zone | https://github.com/logos-blockchain/logos-execution-zone-module | +| logos-blockchain-ui | https://github.com/logos-blockchain/logos-blockchain-ui | +| logos-storage-module | https://github.com/logos-co/logos-storage-module | +| logos-delivery-module | https://github.com/logos-co/logos-delivery-module | +| logos-storage-ui | https://github.com/logos-co/logos-storage-ui | ## Requirements diff --git a/ci/package-modules.sh b/ci/package-modules.sh new file mode 100755 index 0000000..88da527 --- /dev/null +++ b/ci/package-modules.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# package-modules.sh — Merge single-variant .lgx packages into multi-variant ones. +# +# Expected environment: +# LGX — path to the lgx binary +# ARTIFACTS_DIR — path to downloaded artifacts (default: "artifacts") +# +# Expected directory layout under ARTIFACTS_DIR: +# //.lgx +# +# Each input .lgx is a single-variant package produced by nix-bundle-lgx. +# This script verifies that manifests match across variants (ignoring the "main" +# field which differs per platform), then uses the lgx tool to create a fresh +# multi-variant package from the extracted per-platform files. +# +# All metadata is extracted from the single-variant lgx manifests — no submodule +# checkout is required. + +script_dir="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "$script_dir/.." && pwd)" + +: "${LGX:?LGX env var must point to the lgx binary}" +: "${ARTIFACTS_DIR:=artifacts}" + +output_dir="$repo_dir/output" +mkdir -p "$output_dir" + +ALL_VARIANTS=("linux-amd64" "linux-arm64" "darwin-arm64") + +if [[ ! -f "$repo_dir/.gitmodules" ]]; then + echo "No .gitmodules found in $repo_dir" >&2 + exit 1 +fi + +modules=$(git config --file "$repo_dir/.gitmodules" --get-regexp path | awk '{print $2}') + +if [[ -z "$modules" ]]; then + echo "No module paths found in .gitmodules" >&2 + exit 1 +fi + +# Store module metadata for list.json generation +module_entries=() + +for module in $modules; do + echo "=== Processing $module ===" + + # Collect all single-variant lgx files for this module + declare -A variant_lgx_map=() + available_variants=() + + for variant in "${ALL_VARIANTS[@]}"; do + variant_module_dir="$ARTIFACTS_DIR/${variant}/${module}" + if [[ -d "$variant_module_dir" ]]; then + lgx_file=$(find "$variant_module_dir" -maxdepth 1 -name '*.lgx' 2>/dev/null | head -n 1) + if [[ -n "$lgx_file" ]]; then + variant_lgx_map["$variant"]="$lgx_file" + available_variants+=("$variant") + fi + fi + done + + if [[ ${#available_variants[@]} -eq 0 ]]; then + echo "No lgx artifacts found for $module, skipping." + continue + fi + + echo "Available variants for $module: ${available_variants[*]}" + + # Collect lgx file paths for verification + lgx_file_args=() + for variant in "${available_variants[@]}"; do + lgx_file_args+=("${variant_lgx_map[$variant]}") + done + + # --- Verify manifests match across all single-variant lgx files (ignoring "main") --- + python3 - "${lgx_file_args[@]}" <<'PY' +import tarfile, json, sys + +def read_manifest(lgx_path): + with tarfile.open(lgx_path, 'r:gz') as tar: + for member in tar.getmembers(): + if member.name == 'manifest.json': + return json.loads(tar.extractfile(member).read()) + return None + +manifests = [] +for path in sys.argv[1:]: + m = read_manifest(path) + if m is None: + print(f"ERROR: no manifest.json found in {path}", file=sys.stderr) + sys.exit(1) + manifests.append((path, m)) + +# Compare all manifests ignoring the "main" field (differs per platform, e.g. .dylib vs .so) +def manifest_without_main(m): + return {k: v for k, v in m.items() if k != "main"} + +reference_path, reference = manifests[0] +ref_comparable = manifest_without_main(reference) + +for path, m in manifests[1:]: + comparable = manifest_without_main(m) + if comparable != ref_comparable: + print(f"ERROR: manifest mismatch between {reference_path} and {path}", file=sys.stderr) + print(f" Reference: {json.dumps(ref_comparable, sort_keys=True)}", file=sys.stderr) + print(f" Mismatch: {json.dumps(comparable, sort_keys=True)}", file=sys.stderr) + sys.exit(1) + +print(f"Manifests verified: all {len(manifests)} variant(s) match (ignoring main field).") +PY + + # --- Extract metadata from the first single-variant lgx manifest --- + first_lgx="${variant_lgx_map[${available_variants[0]}]}" + + manifest_json=$(python3 - "$first_lgx" <<'PY' +import tarfile, json, sys +with tarfile.open(sys.argv[1], 'r:gz') as tar: + for member in tar.getmembers(): + if member.name == 'manifest.json': + print(tar.extractfile(member).read().decode()) + break +PY +) + + package_name=$(echo "$manifest_json" | python3 -c "import json, sys; print(json.load(sys.stdin).get('name', ''))") + + if [[ -z "$package_name" ]]; then + # Fall back to lgx filename + package_name=$(basename "$first_lgx" .lgx) + fi + + lgx_package_path="$output_dir/${package_name}.lgx" + + # --- Create multi-variant lgx package using the lgx tool --- + + rm -f "${package_name}.lgx" + "$LGX" create "$package_name" + mv "${package_name}.lgx" "$lgx_package_path" + + # Patch manifest with metadata extracted from the single-variant lgx + echo "Updating package manifest with metadata..." + python3 - "$lgx_package_path" "$manifest_json" <<'PY' +import json, sys, tarfile, io + +lgx_path = sys.argv[1] +metadata = json.loads(sys.argv[2]) + +with tarfile.open(lgx_path, 'r:gz') as tar: + members = [] + for member in tar.getmembers(): + if member.isfile(): + members.append((member, tar.extractfile(member).read())) + else: + members.append((member, None)) + +patched = [] +for member, data in members: + if member.name == 'manifest.json': + manifest = json.loads(data) + for key in ('name', 'version', 'description', 'author', 'type', 'category', 'dependencies', 'icon'): + if metadata.get(key): + manifest[key] = metadata[key] + data = json.dumps(manifest, indent=2).encode() + member.size = len(data) + patched.append((member, data)) + +with tarfile.open(lgx_path, 'w:gz', format=tarfile.GNU_FORMAT) as tar: + for member, data in patched: + if data is not None: + tar.addfile(member, io.BytesIO(data)) + else: + tar.addfile(member) +PY + + # Add each variant by extracting files from its single-variant lgx + for variant in "${available_variants[@]}"; do + lgx_file="${variant_lgx_map[$variant]}" + + # Read the per-variant main file from this variant's lgx manifest + variant_main=$(python3 - "$lgx_file" "$variant" <<'PY' +import tarfile, json, sys +with tarfile.open(sys.argv[1], 'r:gz') as tar: + for member in tar.getmembers(): + if member.name == 'manifest.json': + m = json.loads(tar.extractfile(member).read()) + main = m.get('main', {}) + print(main.get(sys.argv[2], '')) + break +PY +) + + # Extract variant files from the single-variant lgx into a temp directory + extract_dir=$(mktemp -d) + python3 - "$lgx_file" "$variant" "$extract_dir" <<'PY' +import tarfile, sys, os + +lgx_path = sys.argv[1] +variant = sys.argv[2] +extract_dir = sys.argv[3] + +prefix = f"variants/{variant}/" + +with tarfile.open(lgx_path, 'r:gz') as tar: + for member in tar.getmembers(): + if member.name.startswith(prefix) and member.isfile(): + rel = member.name[len(prefix):] + target = os.path.join(extract_dir, rel) + os.makedirs(os.path.dirname(target), exist_ok=True) + with tar.extractfile(member) as src: + with open(target, 'wb') as dst: + dst.write(src.read()) +PY + + # Determine main file for lgx add + if [[ -z "$variant_main" || ! -f "$extract_dir/$variant_main" ]]; then + echo "ERROR: main file '${variant_main:-}' not found for variant $variant of $module, skipping variant." >&2 + rm -rf "$extract_dir" + continue + fi + + main_path="$variant_main" + + echo "Adding variant $variant to ${package_name}.lgx (main: $main_path)" + "$LGX" add "$lgx_package_path" \ + --variant "$variant" \ + --files "$extract_dir/." \ + --main "$main_path" \ + -y || { + echo "Failed to add variant $variant to LGX package for $package_name" >&2 + exit 1 + } + + rm -rf "$extract_dir" + echo "Successfully added variant $variant to ${package_name}.lgx" + done + + # Store entry for list.json generation (metadata from the lgx manifest) + variants_csv=$(IFS=,; echo "${available_variants[*]}") + module_entries+=("$module::$manifest_json::${package_name}.lgx::${variants_csv}") +done + +# Generate list.json +list_json_path="$output_dir/list.json" + +python3 - "$list_json_path" "${module_entries[@]}" <<'PY' +import json, os, sys + +list_path = sys.argv[1] +entries = sys.argv[2:] + +result_index = {} + +for raw in entries: + if "::" not in raw: + continue + parts = raw.split("::", 3) + if len(parts) < 3: + continue + + name, metadata_json, package_filename = parts[0], parts[1], parts[2] + variants_csv = parts[3] if len(parts) > 3 else "" + + try: + metadata = json.loads(metadata_json) + except json.JSONDecodeError: + metadata = {} + + item = {"name": name} + item["package"] = package_filename + + if "type" in metadata: + item["type"] = metadata["type"] + if "name" in metadata: + item["moduleName"] = metadata["name"] + if "description" in metadata: + item["description"] = metadata["description"] + if "dependencies" in metadata: + item["dependencies"] = metadata["dependencies"] + if "category" in metadata: + item["category"] = metadata["category"] + if "author" in metadata: + item["author"] = metadata["author"] + if metadata.get("version"): + item["version"] = metadata["version"] + if variants_csv: + item["variants"] = [v for v in variants_csv.split(",") if v] + + result_index[name] = item + +result = [result_index[k] for k in sorted(result_index)] +os.makedirs(os.path.dirname(list_path), exist_ok=True) +with open(list_path, "w") as f: + json.dump(result, f, indent=2) +PY + +echo "" +echo "All modules packaged successfully." +echo "LGX packages created in $output_dir" +echo "Package list written to $list_json_path" diff --git a/compile.sh b/compile.sh index 94a0d8d..c3df862 100755 --- a/compile.sh +++ b/compile.sh @@ -8,38 +8,7 @@ cd "$script_dir" base_libraries_dir="$script_dir/libraries" list_json_path="$base_libraries_dir/list.json" -# Detect platform variant for LGX -os_name="$(uname -s)" -arch_name="$(uname -m)" - -case "$os_name" in - Darwin) - case "$arch_name" in - arm64) lgx_variant="darwin-arm64" ;; - x86_64) lgx_variant="darwin-amd64" ;; - *) - echo "Unsupported Darwin architecture: $arch_name" >&2 - exit 1 - ;; - esac - ;; - Linux) - case "$arch_name" in - x86_64) lgx_variant="linux-amd64" ;; - aarch64) lgx_variant="linux-arm64" ;; - *) - echo "Unsupported Linux architecture: $arch_name" >&2 - exit 1 - ;; - esac - ;; - *) - echo "Unsupported platform: $os_name" >&2 - exit 1 - ;; -esac - -echo "Building for platform variant: $lgx_variant" +BUNDLER="${BUNDLER:-github:logos-co/nix-bundle-lgx#portable}" if [[ ! -f .gitmodules ]]; then echo "No .gitmodules found in $script_dir" >&2 @@ -55,205 +24,52 @@ fi mkdir -p "$base_libraries_dir" -# Build lgx binary first -echo "Building lgx tool..." -if ! nix build --extra-experimental-features 'nix-command flakes' '.#lgx'; then - echo "Failed to build lgx tool" >&2 - exit 1 -fi - -lgx_binary="$script_dir/result/bin/lgx" -if [[ ! -x "$lgx_binary" ]]; then - echo "lgx binary not found at $lgx_binary" >&2 - exit 1 -fi - -echo "lgx binary ready at $lgx_binary" - # Store module metadata for list.json generation module_entries=() for module in $modules; do echo "Building $module..." - if (cd "$module" && nix build --extra-experimental-features 'nix-command flakes' '.#lib'); then + if (cd "$module" && nix bundle --extra-experimental-features 'nix-command flakes' --bundler "$BUNDLER" -o result .#lib); then echo "Built $module" - module_lib_dir="$script_dir/$module/result/lib" - if [[ ! -d "$module_lib_dir" ]]; then - echo "Expected library output directory not found for $module at $module_lib_dir" >&2 + + lgx_file=$(find "$script_dir/$module/result" -maxdepth 1 -name '*.lgx' 2>/dev/null | head -n 1) + if [[ -z "$lgx_file" ]]; then + echo "No lgx file found in $module/result/" >&2 exit 1 fi - # Extract metadata from metadata.json + package_name=$(basename "$lgx_file" .lgx) + cp "$lgx_file" "$base_libraries_dir/" + + echo "Created ${package_name}.lgx" + + # Read metadata for list.json module_metadata_path="$script_dir/$module/metadata.json" module_metadata_json=$(python3 - "$module_metadata_path" <<'PY' -import json -import sys - +import json, sys try: - with open(sys.argv[1], "r") as f: - metadata = json.load(f) - result = { - "type": metadata.get("type", ""), - "name": metadata.get("name", ""), - "description": metadata.get("description", ""), - "dependencies": metadata.get("dependencies", []), - "category": metadata.get("category", ""), - "author": metadata.get("author", ""), - "version": metadata.get("version", "0.0.1"), - "main": metadata.get("main", "") - } - print(json.dumps(result)) -except (FileNotFoundError, json.JSONDecodeError, KeyError): - print(json.dumps({ - "type": "", - "name": "", - "description": "", - "dependencies": [], - "category": "", - "author": "", - "version": "0.0.1", - "main": "" - })) + with open(sys.argv[1]) as f: + print(json.dumps(json.load(f))) +except: + print("{}") PY ) - module_metadata_json=${module_metadata_json//$'\n'/} - - # Parse metadata to get package name - package_name=$(echo "$module_metadata_json" | python3 -c "import json, sys; print(json.load(sys.stdin).get('name', ''))") - - if [[ -z "$package_name" ]]; then - echo "No package name found in metadata.json for $module" >&2 - exit 1 - fi - - lgx_package_path="$base_libraries_dir/${package_name}.lgx" - - # Create or update LGX package - if [[ ! -f "$lgx_package_path" ]]; then - echo "Creating new LGX package: ${package_name}.lgx" - - # Remove any stale lgx file in current directory from previous failed run - rm -f "${package_name}.lgx" - - "$lgx_binary" create "$package_name" || { - echo "Failed to create LGX package for $package_name" >&2 - exit 1 - } - - # Move created package to libraries directory - mv "${package_name}.lgx" "$lgx_package_path" - - # Update manifest.json inside the package with metadata - echo "Updating package manifest with metadata..." - python3 - "$lgx_package_path" "$module_metadata_json" <<'PY' -import json -import sys -import tarfile -import gzip -import tempfile -import os -import shutil -lgx_path = sys.argv[1] -metadata_json = sys.argv[2] -metadata = json.loads(metadata_json) - -# Extract to temp directory -temp_dir = tempfile.mkdtemp() -try: - # Extract existing package - with tarfile.open(lgx_path, 'r:gz') as tar: - tar.extractall(temp_dir) - - # Read and update manifest - manifest_path = os.path.join(temp_dir, 'manifest.json') - with open(manifest_path, 'r') as f: - manifest = json.load(f) - - # Update manifest fields from metadata - manifest['name'] = metadata.get('name', manifest['name']) - manifest['version'] = metadata.get('version', manifest['version']) - manifest['description'] = metadata.get('description', manifest['description']) - manifest['author'] = metadata.get('author', manifest['author']) - manifest['type'] = metadata.get('type', manifest['type']) - manifest['category'] = metadata.get('category', manifest['category']) - manifest['dependencies'] = metadata.get('dependencies', manifest['dependencies']) - - # Write updated manifest - with open(manifest_path, 'w') as f: - json.dump(manifest, f, indent=2) - - # Recreate the package - use lgx CLI for proper deterministic packing - # For now, we'll trust that lgx add will update it properly when we add variants - -finally: - shutil.rmtree(temp_dir, ignore_errors=True) -PY - fi - - # Get main entry from metadata, or fall back to first file - main_entry=$(echo "$module_metadata_json" | python3 -c "import json, sys; print(json.load(sys.stdin).get('main', ''))") - - if [[ -n "$main_entry" ]]; then - # Determine extension based on platform - case "$os_name" in - Darwin) lib_ext=".dylib" ;; - Linux) lib_ext=".so" ;; - esac - main_path="${main_entry}${lib_ext}" - else - # Fallback: use first file in library directory - main_file=$(ls "$module_lib_dir" | head -n 1) - if [[ -z "$main_file" ]]; then - echo "No library files found in $module_lib_dir" >&2 - exit 1 - fi - main_path="$main_file" - fi - - echo "Adding variant $lgx_variant to ${package_name}.lgx" - "$lgx_binary" add "$lgx_package_path" \ - --variant "$lgx_variant" \ - --files "$module_lib_dir/." \ - --main "$main_path" \ - -y || { - echo "Failed to add variant to LGX package for $package_name" >&2 - exit 1 - } - - echo "Successfully added variant $lgx_variant to ${package_name}.lgx" - - # Store entry for list.json generation module_entries+=("$module::$module_metadata_json::${package_name}.lgx") else - echo "Failed building $module (nix build '.#lib')" >&2 + echo "Failed building $module" >&2 exit 1 fi done -# Generate list.json with package references +# Generate list.json python3 - "$list_json_path" "${module_entries[@]}" <<'PY' -import json -import os -import sys +import json, os, sys list_path = sys.argv[1] entries = sys.argv[2:] -def load_existing(path): - try: - with open(path, "r") as f: - return json.load(f) - except FileNotFoundError: - return [] - except Exception: - return [] - -data = load_existing(list_path) -index = {} -for item in data: - if isinstance(item, dict) and "name" in item: - index[item["name"]] = item +result_index = {} for raw in entries: if "::" not in raw: @@ -261,19 +77,14 @@ for raw in entries: parts = raw.split("::", 2) if len(parts) != 3: continue - + name, metadata_json, package_filename = parts try: metadata = json.loads(metadata_json) except json.JSONDecodeError: metadata = {} - - item = index.get(name, {"name": name}) - - # Set package field - item["package"] = package_filename - - # Update metadata fields from metadata.json + + item = {"name": name, "package": package_filename} if "type" in metadata: item["type"] = metadata["type"] if "name" in metadata: @@ -286,10 +97,12 @@ for raw in entries: item["category"] = metadata["category"] if "author" in metadata: item["author"] = metadata["author"] - - index[name] = item + if metadata.get("version"): + item["version"] = metadata["version"] -result = [index[k] for k in sorted(index)] + result_index[name] = item + +result = [result_index[k] for k in sorted(result_index)] os.makedirs(os.path.dirname(list_path), exist_ok=True) with open(list_path, "w") as f: json.dump(result, f, indent=2) diff --git a/flake.lock b/flake.lock index a52da91..e78b27d 100644 --- a/flake.lock +++ b/flake.lock @@ -1,15 +1,195 @@ { "nodes": { - "logos-package": { + "logos-capability-module": { + "inputs": { + "logos-cpp-sdk": "logos-cpp-sdk", + "logos-liblogos": "logos-liblogos_2", + "nixpkgs": [ + "logos-package", + "logos-liblogos", + "logos-capability-module", + "logos-liblogos", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1767809111, + "narHash": "sha256-jehjsB+BpDJlVu3I7x+vFVOdXmy9MDmFTJtRqzFUONo=", + "owner": "logos-co", + "repo": "logos-capability-module", + "rev": "7b35383e0aa4e28a4633ed18a87efb57636939b1", + "type": "github" + }, + "original": { + "owner": "logos-co", + "repo": "logos-capability-module", + "type": "github" + } + }, + "logos-cpp-sdk": { "inputs": { "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1768925546, - "narHash": "sha256-Y4sgYs9wtZ9sHAuKl9LUy//ReeF4/AyK8HlnZsYrSqg=", + "lastModified": 1761230734, + "narHash": "sha256-CMRUwXH7pJZ1OI6bd/TDDDXKqQ1tQZHQEOOwK8TgYHI=", + "owner": "logos-co", + "repo": "logos-cpp-sdk", + "rev": "4b143922c190df00bb3835441c9f0075cb28283b", + "type": "github" + }, + "original": { + "owner": "logos-co", + "repo": "logos-cpp-sdk", + "type": "github" + } + }, + "logos-cpp-sdk_2": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1761230734, + "narHash": "sha256-CMRUwXH7pJZ1OI6bd/TDDDXKqQ1tQZHQEOOwK8TgYHI=", + "owner": "logos-co", + "repo": "logos-cpp-sdk", + "rev": "4b143922c190df00bb3835441c9f0075cb28283b", + "type": "github" + }, + "original": { + "owner": "logos-co", + "repo": "logos-cpp-sdk", + "type": "github" + } + }, + "logos-cpp-sdk_3": { + "inputs": { + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1767724329, + "narHash": "sha256-UPkqxqxbKwU5Dmu00TnjiJVXUmfVylF3p1qziEuYwIE=", + "owner": "logos-co", + "repo": "logos-cpp-sdk", + "rev": "32f1d7080d784ff044d91d076ef2f0c7305d4784", + "type": "github" + }, + "original": { + "owner": "logos-co", + "repo": "logos-cpp-sdk", + "type": "github" + } + }, + "logos-cpp-sdk_4": { + "inputs": { + "nixpkgs": "nixpkgs_4" + }, + "locked": { + "lastModified": 1767724329, + "narHash": "sha256-UPkqxqxbKwU5Dmu00TnjiJVXUmfVylF3p1qziEuYwIE=", + "owner": "logos-co", + "repo": "logos-cpp-sdk", + "rev": "32f1d7080d784ff044d91d076ef2f0c7305d4784", + "type": "github" + }, + "original": { + "owner": "logos-co", + "repo": "logos-cpp-sdk", + "type": "github" + } + }, + "logos-liblogos": { + "inputs": { + "logos-capability-module": "logos-capability-module", + "logos-cpp-sdk": "logos-cpp-sdk_3", + "logos-module": "logos-module", + "nixpkgs": [ + "logos-package", + "logos-liblogos", + "logos-cpp-sdk", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1770154824, + "narHash": "sha256-WGI+3FkPdeytfLJ3ZJYr1O8esUnQjcmMJEYwB/EBZMs=", + "owner": "logos-co", + "repo": "logos-liblogos", + "rev": "901dd86d47216b15b6f1260b7b6bb4ecd88a8f9d", + "type": "github" + }, + "original": { + "owner": "logos-co", + "repo": "logos-liblogos", + "type": "github" + } + }, + "logos-liblogos_2": { + "inputs": { + "logos-cpp-sdk": "logos-cpp-sdk_2", + "nixpkgs": [ + "logos-package", + "logos-liblogos", + "logos-capability-module", + "logos-liblogos", + "logos-cpp-sdk", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1761845775, + "narHash": "sha256-ulK8xq05ejK6qIgZ7WtWb/MJt2rk5BKfDA2z7mM3wq8=", + "owner": "logos-co", + "repo": "logos-liblogos", + "rev": "a92c2c1268bc70764c8f73c7bce07d21024f5af9", + "type": "github" + }, + "original": { + "owner": "logos-co", + "repo": "logos-liblogos", + "type": "github" + } + }, + "logos-module": { + "inputs": { + "logos-cpp-sdk": "logos-cpp-sdk_4", + "nixpkgs": [ + "logos-package", + "logos-liblogos", + "logos-module", + "logos-cpp-sdk", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1770062426, + "narHash": "sha256-zc7ZxDTlqOCYGyEHhrTA/7GS1EWh7+4amdPUKh+gGds=", + "owner": "logos-co", + "repo": "logos-module", + "rev": "f7ee69d9ad9f27c84f04f59896e9194125e951dc", + "type": "github" + }, + "original": { + "owner": "logos-co", + "repo": "logos-module", + "type": "github" + } + }, + "logos-package": { + "inputs": { + "logos-liblogos": "logos-liblogos", + "nixpkgs": [ + "logos-package", + "logos-liblogos", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1771887671, + "narHash": "sha256-eIzmR3N5QnR5lcBT3N58v49ggl9MzUYSRgQznex1ayM=", "owner": "logos-co", "repo": "logos-package", - "rev": "9230ae37c9d289c0c355dcf9fa40fd3be2e99f17", + "rev": "d9a741359b17b928afae59cf3c6ee00057b7d93e", "type": "github" }, "original": { @@ -20,11 +200,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1768127708, - "narHash": "sha256-1Sm77VfZh3mU0F5OqKABNLWxOuDeHIlcFjsXeeiPazs=", + "lastModified": 1759036355, + "narHash": "sha256-0m27AKv6ka+q270dw48KflE0LwQYrO7Fm4/2//KCVWg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ffbc9f8cbaacfb331b6017d5a5abb21a492c9a38", + "rev": "e9f00bd893984bc8ce46c895c3bf7cac95331127", "type": "github" }, "original": { @@ -36,11 +216,59 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1768564909, - "narHash": "sha256-Kell/SpJYVkHWMvnhqJz/8DqQg2b6PguxVWOuadbHCc=", + "lastModified": 1759036355, + "narHash": "sha256-0m27AKv6ka+q270dw48KflE0LwQYrO7Fm4/2//KCVWg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e4bae1bd10c9c57b2cf517953ab70060a828ee6f", + "rev": "e9f00bd893984bc8ce46c895c3bf7cac95331127", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1759036355, + "narHash": "sha256-0m27AKv6ka+q270dw48KflE0LwQYrO7Fm4/2//KCVWg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e9f00bd893984bc8ce46c895c3bf7cac95331127", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1759036355, + "narHash": "sha256-0m27AKv6ka+q270dw48KflE0LwQYrO7Fm4/2//KCVWg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e9f00bd893984bc8ce46c895c3bf7cac95331127", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1771369470, + "narHash": "sha256-0NBlEBKkN3lufyvFegY4TYv5mCNHbi5OmBDrzihbBMQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "0182a361324364ae3f436a63005877674cf45efb", "type": "github" }, "original": { @@ -53,7 +281,7 @@ "root": { "inputs": { "logos-package": "logos-package", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_5" } } }, diff --git a/logos-accounts-module b/logos-accounts-module index 05c9680..d7c8722 160000 --- a/logos-accounts-module +++ b/logos-accounts-module @@ -1 +1 @@ -Subproject commit 05c9680a7a33b4556ed145388b59a9ab769bcd27 +Subproject commit d7c8722e6cbf50701b722db4026946d96cfadaf1 diff --git a/logos-accounts-ui b/logos-accounts-ui index ce5d0af..515e379 160000 --- a/logos-accounts-ui +++ b/logos-accounts-ui @@ -1 +1 @@ -Subproject commit ce5d0af6f58a8f2f8c9bfbaf0cefa1e7496c002e +Subproject commit 515e379d16e2115eed7da21c03ba523ec8e2007c diff --git a/logos-blockchain-module b/logos-blockchain-module new file mode 160000 index 0000000..888a668 --- /dev/null +++ b/logos-blockchain-module @@ -0,0 +1 @@ +Subproject commit 888a66849ffc9babeb33ac070145b29861f0a23d diff --git a/logos-blockchain-ui b/logos-blockchain-ui new file mode 160000 index 0000000..f948acd --- /dev/null +++ b/logos-blockchain-ui @@ -0,0 +1 @@ +Subproject commit f948acd17a3da804ba4eeb70e5222f1c67fbe192 diff --git a/logos-delivery-module b/logos-delivery-module new file mode 160000 index 0000000..af50ea3 --- /dev/null +++ b/logos-delivery-module @@ -0,0 +1 @@ +Subproject commit af50ea3593a18d073e8411ac03bf0b46dbcebd7b diff --git a/logos-execution-zone-module b/logos-execution-zone-module new file mode 160000 index 0000000..15d2e1b --- /dev/null +++ b/logos-execution-zone-module @@ -0,0 +1 @@ +Subproject commit 15d2e1bc9adc1685770e00085584c61c30fc5847 diff --git a/logos-package-manager b/logos-package-manager deleted file mode 160000 index cbdb3ca..0000000 --- a/logos-package-manager +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cbdb3ca8c90705e930dc63500c07bb8c10e85e90 diff --git a/logos-package-manager-module b/logos-package-manager-module new file mode 160000 index 0000000..0cbf250 --- /dev/null +++ b/logos-package-manager-module @@ -0,0 +1 @@ +Subproject commit 0cbf250ad2fec20c79dc5f61729a07c3a2bd50e1 diff --git a/logos-storage-module b/logos-storage-module new file mode 160000 index 0000000..b4ecf7a --- /dev/null +++ b/logos-storage-module @@ -0,0 +1 @@ +Subproject commit b4ecf7a871233608f63b817eeae426f6273695d9 diff --git a/logos-storage-ui b/logos-storage-ui new file mode 160000 index 0000000..006d0b6 --- /dev/null +++ b/logos-storage-ui @@ -0,0 +1 @@ +Subproject commit 006d0b6a68da424a05474344af6070004da0d239 diff --git a/logos-waku-module b/logos-waku-module index 0e01fb8..b4bd68f 160000 --- a/logos-waku-module +++ b/logos-waku-module @@ -1 +1 @@ -Subproject commit 0e01fb8676814dc74f5c2bb399c1c0710e73d449 +Subproject commit b4bd68fa6762c63cf57adbb913475949ee162d64 diff --git a/logos-wallet-module b/logos-wallet-module index ef98e6e..2eba6ad 160000 --- a/logos-wallet-module +++ b/logos-wallet-module @@ -1 +1 @@ -Subproject commit ef98e6ec6db9cffc819dacb45667035294af04b2 +Subproject commit 2eba6ad1933ab7d8651645dfe499c42fc01edf6c diff --git a/logos-wallet-ui b/logos-wallet-ui index 506b207..d260e57 160000 --- a/logos-wallet-ui +++ b/logos-wallet-ui @@ -1 +1 @@ -Subproject commit 506b2071164b202407dfc6785c3e0148d08baea1 +Subproject commit d260e57e849aa15cb6bcf6fccb5584cc1604a562