From 0c6784da7e83b6d7b8ef27a43b9508eaaf0d1d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bur=C4=8Dul?= <39484255+markoburcul@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:07:01 +0100 Subject: [PATCH] nix: make derivation and update shell (#1003) * nix: make derivation and update shell Create a structure for nix files. Add the derivation file which is using system Nim to compile Codex. Referenced issue: https://github.com/codex-storage/nim-codex/issues/940 Signed-off-by: markoburcul * nim-circom-compat: update Include commit which allows building circom-compat-ffi using Nix(doesn't affect current usage of the submodule). Referenced issue: https://github.com/codex-storage/nim-codex/issues/940 Signed-off-by: markoburcul * makefile: fix for detecting linux arch Signed-off-by: markoburcul --------- Signed-off-by: markoburcul --- Makefile | 2 +- flake.lock | 37 +++++++++++++++++- flake.nix | 36 ++++++++++------- nix/README.md | 29 ++++++++++++++ nix/default.nix | 84 ++++++++++++++++++++++++++++++++++++++++ nix/tools.nix | 15 +++++++ vendor/nim-circom-compat | 2 +- 7 files changed, 188 insertions(+), 17 deletions(-) create mode 100644 nix/README.md create mode 100644 nix/default.nix create mode 100644 nix/tools.nix diff --git a/Makefile b/Makefile index 7c8f2b1a..cd6e5a0a 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ ifeq ($(OS),Windows_NT) ARCH = arm64 endif else - UNAME_P := $(shell uname -p) + UNAME_P := $(shell uname -m) ifneq ($(filter $(UNAME_P), i686 i386 x86_64),) ARCH = x86_64 endif diff --git a/flake.lock b/flake.lock index 2ea7b7ee..ac86a75c 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,40 @@ { "nodes": { + "circom-compat": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1732627240, + "narHash": "sha256-GvJTiBWBv799i5ZCCc4gF86bnQY/nZvx0vCPi1+OPD4=", + "owner": "codex-storage", + "repo": "circom-compat-ffi", + "rev": "297c46fdc7d8a8fd53c8076b0be77334e4a54447", + "type": "github" + }, + "original": { + "owner": "codex-storage", + "repo": "circom-compat-ffi", + "type": "github" + } + }, "nixpkgs": { + "locked": { + "lastModified": 1731386116, + "narHash": "sha256-lKA770aUmjPHdTaJWnP3yQ9OI1TigenUqVC3wweqZuI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "689fed12a013f56d4c4d3f612489634267d86529", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { "locked": { "lastModified": 1729449015, "narHash": "sha256-Gf04dXB0n4q0A9G5nTGH3zuMGr6jtJppqdeljxua1fo=", @@ -18,7 +52,8 @@ }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "circom-compat": "circom-compat", + "nixpkgs": "nixpkgs_2" } } }, diff --git a/flake.nix b/flake.nix index 8778ca4a..e0b483c9 100644 --- a/flake.nix +++ b/flake.nix @@ -3,32 +3,40 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + circom-compat.url = "github:codex-storage/circom-compat-ffi"; }; - outputs = { self, nixpkgs }: + outputs = { self, nixpkgs, circom-compat}: let - supportedSystems = [ + stableSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; - forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system); + forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system); pkgsFor = forAllSystems (system: import nixpkgs { inherit system; }); in rec { + packages = forAllSystems (system: let + circomCompatPkg = circom-compat.packages.${system}.default; + buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix { + inherit stableSystems circomCompatPkg; + src = self; + }; + build = targets: buildTarget.override { inherit targets; }; + in rec { + codex = build ["all"]; + default = codex; + }); + devShells = forAllSystems (system: let pkgs = pkgsFor.${system}; - inherit (pkgs) lib stdenv mkShell; in { - default = mkShell.override { stdenv = pkgs.gcc11Stdenv; } { - buildInputs = with pkgs; [ - # General - git pkg-config openssl lsb-release - # Build - rustc cargo nimble gcc11 cmake nim-unwrapped-1 - # Libraries - gmp llvmPackages.openmp - # Tests - nodejs_18 + default = pkgs.mkShell { + inputsFrom = [ + packages.${system}.codex + circom-compat.packages.${system}.default ]; + # Not using buildInputs to override fakeGit and fakeCargo. + nativeBuildInputs = with pkgs; [ git cargo nodejs_18 ]; }; }); }; diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 00000000..ba663622 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,29 @@ +# Usage + +## Shell + +A development shell can be started using: +```sh +nix develop +``` + +## Building + +To build a Codex you can use: +```sh +nix build '.?submodules=1#codex' +``` +The `?submodules=1` part should eventually not be necessary. +For more details see: +https://github.com/NixOS/nix/issues/4423 + +It can be also done without even cloning the repo: +```sh +nix build 'github:codex-storage/nim-codex?submodules=1' +``` + +## Running + +```sh +nix run 'github:codex-storage/nim-codex?submodules=1' +``` diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 00000000..63970106 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,84 @@ +{ + pkgs ? import { }, + src ? ../., + targets ? ["all"], + # Options: 0,1,2 + verbosity ? 0, + # Use system Nim compiler instead of building it with nimbus-build-system + useSystemNim ? true, + commit ? builtins.substring 0 7 (src.rev or "dirty"), + # These are the only platforms tested in CI and considered stable. + stableSystems ? [ + "x86_64-linux" "aarch64-linux" + "x86_64-darwin" "aarch64-darwin" + ], + circomCompatPkg ? ( + builtins.getFlake "github:codex-storage/circom-compat-ffi" + ).packages.${builtins.currentSystem}.default +}: + +let + inherit (pkgs) stdenv lib writeScriptBin callPackage; + + revision = lib.substring 0 8 (src.rev or "dirty"); + + tools = callPackage ./tools.nix {}; +in pkgs.gcc11Stdenv.mkDerivation rec { + + pname = "codex"; + + version = "${tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble}-${revision}"; + + inherit src; + + # Dependencies that should exist in the runtime environment. + buildInputs = with pkgs; [ + openssl + gmp + ]; + + # Dependencies that should only exist in the build environment. + nativeBuildInputs = let + # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'. + fakeGit = writeScriptBin "git" "echo ${version}"; + # Fix for the nim-circom-compat-ffi package that is built with cargo. + fakeCargo = writeScriptBin "cargo" "echo ${version}"; + in + with pkgs; [ + cmake + pkg-config + nimble + which + nim-unwrapped-1 + lsb-release + circomCompatPkg + fakeGit + fakeCargo + ]; + + # Disable CPU optmizations that make binary not portable. + NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}"; + # Avoid Nim cache permission errors. + XDG_CACHE_HOME = "/tmp"; + + makeFlags = targets ++ [ + "V=${toString verbosity}" + "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}" + ]; + + configurePhase = '' + patchShebangs . > /dev/null + ''; + + installPhase = '' + mkdir -p $out/bin + cp build/codex $out/bin/ + ''; + + meta = with pkgs.lib; { + description = "Codex storage system"; + homepage = "https://github.com/codex-storage/nim-codex"; + license = licenses.mit; + platforms = stableSystems; + }; +} diff --git a/nix/tools.nix b/nix/tools.nix new file mode 100644 index 00000000..108d3860 --- /dev/null +++ b/nix/tools.nix @@ -0,0 +1,15 @@ +{ pkgs ? import { } }: + +let + + inherit (pkgs.lib) fileContents last splitString flatten remove; + inherit (builtins) map match; +in { + findKeyValue = regex: sourceFile: + let + linesFrom = file: splitString "\n" (fileContents file); + matching = regex: lines: map (line: match regex line) lines; + extractMatch = matches: last (flatten (remove null matches)); + in + extractMatch (matching regex (linesFrom sourceFile)); +} diff --git a/vendor/nim-circom-compat b/vendor/nim-circom-compat index e710e4c3..d3fb9039 160000 --- a/vendor/nim-circom-compat +++ b/vendor/nim-circom-compat @@ -1 +1 @@ -Subproject commit e710e4c333f367353aaa1ee82a55a47326b63a65 +Subproject commit d3fb903945c3895f28a2e50685745e0a9762ece5