diff --git a/.gitignore b/.gitignore index eb792eca7..283d31082 100644 --- a/.gitignore +++ b/.gitignore @@ -97,4 +97,7 @@ vendor/github.com/waku-org/go-zerokit-rln-arm/ alice.log bob.log test-alice/ -test-bob/ \ No newline at end of file +test-bob/ + +# Nix +/.nix-gcroots/ diff --git a/.golangci.yml b/.golangci.yml index cec8481c6..6724be2e6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,6 +1,6 @@ run: - concurrency: 4 - timeout: 5m + concurrency: 0 + timeout: 10m issues-exit-code: 1 modules-download-mode: vendor tests: true diff --git a/Makefile b/Makefile index 0dd9fa5f4..eb2b6311b 100644 --- a/Makefile +++ b/Makefile @@ -1,65 +1,6 @@ .PHONY: statusgo statusd-prune all test clean help .PHONY: statusgo-android statusgo-ios -RELEASE_TAG := v$(shell cat VERSION) -RELEASE_BRANCH := develop -RELEASE_DIR := /tmp/release-$(RELEASE_TAG) -PRE_RELEASE := "1" -RELEASE_TYPE := $(shell if [ $(PRE_RELEASE) = "0" ] ; then echo release; else echo pre-release ; fi) -GOLANGCI_BINARY=golangci-lint -IPFS_GATEWAY_URL ?= https://ipfs.status.im/ - -ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... - detected_OS := Windows -else - detected_OS := $(strip $(shell uname)) -endif - -ifeq ($(detected_OS),Darwin) - GOBIN_SHARED_LIB_EXT := dylib - GOBIN_SHARED_LIB_CFLAGS := CGO_ENABLED=1 GOOS=darwin -else ifeq ($(detected_OS),Windows) - GOBIN_SHARED_LIB_EXT := dll - GOBIN_SHARED_LIB_CGO_LDFLAGS := CGO_LDFLAGS="" -else - GOBIN_SHARED_LIB_EXT := so - GOBIN_SHARED_LIB_CGO_LDFLAGS := CGO_LDFLAGS="-Wl,-soname,libstatus.so.0" -endif - -help: ##@other Show this help - @perl -e '$(HELP_FUN)' $(MAKEFILE_LIST) - -CGO_CFLAGS = -I/$(JAVA_HOME)/include -I/$(JAVA_HOME)/include/darwin -GOPATH ?= $(HOME)/go - -GIT_ROOT := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) -GIT_COMMIT = $(shell git rev-parse --short HEAD) -GIT_AUTHOR ?= $(shell git config user.email || echo $$USER) - -ENABLE_METRICS ?= true -BUILD_TAGS ?= gowaku_no_rln -BUILD_FLAGS ?= $(shell echo "-ldflags='\ - -X github.com/status-im/status-go/params.Version=$(RELEASE_TAG:v%=%) \ - -X github.com/status-im/status-go/params.GitCommit=$(GIT_COMMIT) \ - -X github.com/status-im/status-go/params.IpfsGatewayURL=$(IPFS_GATEWAY_URL) \ - -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=$(ENABLE_METRICS)'") - -BUILD_FLAGS_MOBILE ?= $(shell echo "-ldflags='\ - -X github.com/status-im/status-go/params.Version=$(RELEASE_TAG:v%=%) \ - -X github.com/status-im/status-go/params.GitCommit=$(GIT_COMMIT) \ - -X github.com/status-im/status-go/params.IpfsGatewayURL=$(IPFS_GATEWAY_URL)'") - -networkid ?= StatusChain - -DOCKER_IMAGE_NAME ?= statusteam/status-go -BOOTNODE_IMAGE_NAME ?= statusteam/bootnode -STATUSD_PRUNE_IMAGE_NAME ?= statusteam/statusd-prune - -DOCKER_IMAGE_CUSTOM_TAG ?= $(RELEASE_TAG) - -DOCKER_TEST_WORKDIR = /go/src/github.com/status-im/status-go/ -DOCKER_TEST_IMAGE = golang:1.13 - # This is a code for automatic help generator. # It supports ANSI colors and categories. # To add new item into help output, simply add comments @@ -81,19 +22,132 @@ HELP_FUN = \ print "\n"; \ } +help: SHELL := /bin/sh +help: ##@other Show this help + @perl -e '$(HELP_FUN)' $(MAKEFILE_LIST) + +RELEASE_TAG := v$(file < VERSION) +RELEASE_DIR := /tmp/release-$(RELEASE_TAG) +GOLANGCI_BINARY=golangci-lint +IPFS_GATEWAY_URL ?= https://ipfs.status.im/ + +ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... + detected_OS := Windows +else + detected_OS := $(strip $(shell uname)) +endif + +ifeq ($(detected_OS),Darwin) + GOBIN_SHARED_LIB_EXT := dylib + GOBIN_SHARED_LIB_CFLAGS := CGO_ENABLED=1 GOOS=darwin +else ifeq ($(detected_OS),Windows) + GOBIN_SHARED_LIB_EXT := dll + GOBIN_SHARED_LIB_CGO_LDFLAGS := CGO_LDFLAGS="" +else + GOBIN_SHARED_LIB_EXT := so + GOBIN_SHARED_LIB_CGO_LDFLAGS := CGO_LDFLAGS="-Wl,-soname,libstatus.so.0" +endif + +CGO_CFLAGS = -I/$(JAVA_HOME)/include -I/$(JAVA_HOME)/include/darwin +export GOPATH ?= $(HOME)/go + +GIT_ROOT := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) +GIT_COMMIT := $(call sh, git rev-parse --short HEAD) +GIT_AUTHOR := $(call sh, git config user.email || echo $$USER) + +ENABLE_METRICS ?= true +BUILD_TAGS ?= gowaku_no_rln +define BUILD_FLAGS ?= + -ldflags="\ + -X github.com/status-im/status-go/params.Version=$(RELEASE_TAG:v%=%) \ + -X github.com/status-im/status-go/params.GitCommit=$(GIT_COMMIT) \ + -X github.com/status-im/status-go/params.IpfsGatewayURL=$(IPFS_GATEWAY_URL) \ + -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=$(ENABLE_METRICS)" +endef + +define BUILD_FLAGS_MOBILE ?= + -ldflags="\ + -X github.com/status-im/status-go/params.Version=$(RELEASE_TAG:v%=%) \ + -X github.com/status-im/status-go/params.GitCommit=$(GIT_COMMIT) \ + -X github.com/status-im/status-go/params.IpfsGatewayURL=$(IPFS_GATEWAY_URL)" +endef + +networkid ?= StatusChain + +DOCKER_IMAGE_NAME ?= statusteam/status-go +BOOTNODE_IMAGE_NAME ?= statusteam/bootnode +STATUSD_PRUNE_IMAGE_NAME ?= statusteam/statusd-prune + +DOCKER_IMAGE_CUSTOM_TAG ?= $(RELEASE_TAG) + +DOCKER_TEST_WORKDIR = /go/src/github.com/status-im/status-go/ +DOCKER_TEST_IMAGE = golang:1.13 + GO_CMD_PATHS := $(filter-out library, $(wildcard cmd/*)) GO_CMD_NAMES := $(notdir $(GO_CMD_PATHS)) GO_CMD_BUILDS := $(addprefix build/bin/, $(GO_CMD_NAMES)) +# Our custom config is located in nix/nix.conf +export NIX_USER_CONF_FILES = $(PWD)/nix/nix.conf +# Location of symlinks to derivations that should not be garbage collected +export _NIX_GCROOTS = ./.nix-gcroots + +#---------------- +# Nix targets +#---------------- + +# Use $(call sh, ) instead of $(shell ) to avoid +# invoking a Nix shell when normal shell will suffice, it's faster. +# This works because it's defined before we set SHELL to Nix one. +define sh +$(shell $(1)) +endef + +SHELL := ./nix/scripts/shell.sh +shell: export TARGET ?= default +shell: ##@prepare Enter into a pre-configured shell +ifndef IN_NIX_SHELL + @ENTER_NIX_SHELL +else + @echo "${YELLOW}Nix shell is already active$(RESET)" +endif + +nix-repl: SHELL := /bin/sh +nix-repl: ##@nix Start an interactive Nix REPL + nix repl shell.nix + +nix-gc-protected: SHELL := /bin/sh +nix-gc-protected: + @echo -e "$(YELLOW)The following paths are protected:$(RESET)" && \ + ls -1 $(_NIX_GCROOTS) | sed 's/^/ - /' + + +nix-upgrade: SHELL := /bin/sh +nix-upgrade: ##@nix Upgrade Nix interpreter to current version. + nix/scripts/upgrade.sh + +nix-gc: nix-gc-protected ##@nix Garbage collect all packages older than 20 days from /nix/store + nix-store --gc + +nix-clean: ##@nix Remove all status-mobile build artifacts from /nix/store + nix/scripts/clean.sh + +nix-purge: SHELL := /bin/sh +nix-purge: ##@nix Completely remove Nix setup, including /nix directory + nix/scripts/purge.sh + +#---------------- +# General targets +#---------------- all: $(GO_CMD_NAMES) .PHONY: $(GO_CMD_NAMES) $(GO_CMD_PATHS) $(GO_CMD_BUILDS) $(GO_CMD_BUILDS): ##@build Build any Go project from cmd folder go build -mod=vendor -v \ -tags '$(BUILD_TAGS)' $(BUILD_FLAGS) \ - -o ./$@ ./cmd/$(notdir $@) - @echo "Compilation done." - @echo "Run \"build/bin/$(notdir $@) -h\" to view available commands." + -o ./$@ ./cmd/$(notdir $@) ;\ + echo "Compilation done." ;\ + echo "Run \"build/bin/$(notdir $@) -h\" to view available commands." bootnode: ##@build Build discovery v5 bootnode using status-go deps bootnode: build/bin/bootnode @@ -114,6 +168,7 @@ spiff-workflow: build/bin/spiff-workflow status-cli: ##@build Build status-cli to send messages status-cli: build/bin/status-cli +statusd-prune-docker-image: SHELL := /bin/sh statusd-prune-docker-image: ##@statusd-prune Build statusd-prune docker image @echo "Building docker image for ststusd-prune..." docker build --file _assets/build/Dockerfile-prune . \ @@ -185,40 +240,48 @@ endif @echo "Shared library built:" @ls -la build/bin/libstatus.* +docker-image: SHELL := /bin/sh docker-image: BUILD_TARGET ?= statusd docker-image: ##@docker Build docker image (use DOCKER_IMAGE_NAME to set the image name) @echo "Building docker image..." docker build --file _assets/build/Dockerfile . \ - --build-arg "build_tags=$(BUILD_TAGS)" \ - --build-arg "build_flags=$(BUILD_FLAGS)" \ - --build-arg "build_target=$(BUILD_TARGET)" \ - --label "commit=$(GIT_COMMIT)" \ - --label "author=$(GIT_AUTHOR)" \ + --build-arg 'build_tags=$(BUILD_TAGS)' \ + --build-arg 'build_flags=$(BUILD_FLAGS)' \ + --build-arg 'build_target=$(BUILD_TARGET)' \ + --label 'commit=$(GIT_COMMIT)' \ + --label 'author=$(GIT_AUTHOR)' \ -t $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_CUSTOM_TAG) \ -t $(DOCKER_IMAGE_NAME):latest +bootnode-image: SHELL := /bin/sh bootnode-image: @echo "Building docker image for bootnode..." docker build --file _assets/build/Dockerfile-bootnode . \ - --build-arg "build_tags=$(BUILD_TAGS)" \ - --build-arg "build_flags=$(BUILD_FLAGS)" \ - --label "commit=$(GIT_COMMIT)" \ - --label "author=$(GIT_AUTHOR)" \ + --build-arg 'build_tags=$(BUILD_TAGS)' \ + --build-arg 'build_flags=$(BUILD_FLAGS)' \ + --label 'commit=$(GIT_COMMIT)' \ + --label 'author=$(GIT_AUTHOR)' \ -t $(BOOTNODE_IMAGE_NAME):$(DOCKER_IMAGE_CUSTOM_TAG) \ -t $(BOOTNODE_IMAGE_NAME):latest +push-docker-images: SHELL := /bin/sh push-docker-images: docker-image bootnode-image docker push $(BOOTNODE_IMAGE_NAME):$(DOCKER_IMAGE_CUSTOM_TAG) docker push $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_CUSTOM_TAG) +clean-docker-images: SHELL := /bin/sh clean-docker-images: - docker rmi -f $(shell docker image ls --filter="reference=$(DOCKER_IMAGE_NAME)" --quiet) + docker rmi -f $$(docker image ls --filter="reference=$(DOCKER_IMAGE_NAME)" --quiet) # See https://www.gnu.org/software/make/manual/html_node/Target_002dspecific.html to understand this magic. +push-docker-images-latest: SHELL := /bin/sh push-docker-images-latest: GIT_BRANCH = $(shell git rev-parse --abbrev-ref HEAD) push-docker-images-latest: GIT_LOCAL = $(shell git rev-parse @) push-docker-images-latest: GIT_REMOTE = $(shell git fetch -q && git rev-parse remotes/origin/develop || echo 'NO_DEVELOP') -push-docker-images-latest: docker-image bootnode-image +push-docker-images-latest: + echo $(GIT_BRANCH) + echo $(GIT_LOCAL) + echo $(GIT_REMOTE) @echo "Pushing latest docker images..." @echo "Checking git branch..." ifneq ("$(GIT_BRANCH)", "develop") @@ -233,49 +296,11 @@ endif docker push $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_CUSTOM_TAG) setup: ##@setup Install all tools -setup: setup-check setup-build setup-dev tidy - -setup-check: ##@setup Check if Go compiler is installed. -ifeq (, $(shell which go)) - $(error "No Go compiler found! Make sure to install 1.19.0 or newer.") -endif +setup: setup-dev setup-dev: ##@setup Install all necessary tools for development -setup-dev: install-lint install-mock install-modvendor install-protobuf tidy install-os-deps - -setup-build: ##@setup Install all necessary build tools -setup-build: install-lint install-release install-gomobile install-junit-report - -install-os-deps: ##@install Operating System Dependencies - _assets/scripts/install_deps.sh - -install-gomobile: install-xtools -install-gomobile: ##@install Go Mobile Build Tools - GO111MODULE=on go install golang.org/x/mobile/cmd/...@5d9a3325 - GO111MODULE=on go mod download golang.org/x/exp@ec7cb31e - GO111MODULE=off go get -d golang.org/x/mobile/cmd/gobind - -install-lint: ##@install Install Linting Tools - GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 - -install-junit-report: ##@install Install Junit Report Tool for Jenkins integration - GO111MODULE=on go install github.com/jstemmer/go-junit-report/v2@latest - -install-mock: ##@install Install Module Mocking Tools - GO111MODULE=on go install github.com/golang/mock/mockgen@v1.4.4 - -install-modvendor: ##@install Install Module Vendoring Tool - GO111MODULE=on go install github.com/goware/modvendor@v0.5.0 - -install-protobuf: ##@install Install Protobuf Generation Tools - GO111MODULE=on go install github.com/kevinburke/go-bindata/go-bindata@v3.13.0 - GO111MODULE=on go install github.com/golang/protobuf/protoc-gen-go@v1.3.4 - -install-release: ##@install Install Github Release Tools - GO111MODULE=on go install github.com/c4milo/github-release@v1.1.0 - -install-xtools: ##@install Install Miscellaneous Go Tools - GO111MODULE=on go install golang.org/x/tools/go/packages/...@v0.1.5 +setup-dev: + echo "Replaced by Nix shell. Use 'make shell' or just any target as-is." generate-handlers: go generate ./_assets/generate_handlers/ @@ -312,7 +337,7 @@ mock: ##@other Regenerate mocks mockgen -package=peer -destination=services/peer/discoverer_mock.go -source=services/peer/service.go docker-test: ##@tests Run tests in a docker container with golang. - docker run --privileged --rm -it -v "$(shell pwd):$(DOCKER_TEST_WORKDIR)" -w "$(DOCKER_TEST_WORKDIR)" $(DOCKER_TEST_IMAGE) go test ${ARGS} + docker run --privileged --rm -it -v "$(PWD):$(DOCKER_TEST_WORKDIR)" -w "$(DOCKER_TEST_WORKDIR)" $(DOCKER_TEST_IMAGE) go test ${ARGS} test: test-unit ##@tests Run basic, short tests during development @@ -321,7 +346,7 @@ test-unit: export UNIT_TEST_COUNT ?= 1 test-unit: export UNIT_TEST_FAILFAST ?= true test-unit: export UNIT_TEST_RERUN_FAILS ?= true test-unit: export UNIT_TEST_USE_DEVELOPMENT_LOGGER ?= true -test-unit: export UNIT_TEST_PACKAGES ?= $(shell go list ./... | \ +test-unit: export UNIT_TEST_PACKAGES ?= $(call sh, go list ./... | \ grep -v /vendor | \ grep -v /t/e2e | \ grep -v /t/benchmarks | \ @@ -353,8 +378,7 @@ canary-test: node-canary #_assets/scripts/canary_test_mailservers.sh ./config/cli/fleet-eth.prod.json lint: - @echo "lint" - @golangci-lint run ./... + golangci-lint run ./... ci: lint canary-test test-unit test-e2e ##@tests Run all linters and tests at once @@ -410,14 +434,14 @@ clean-mailserver-docker: ##@Easy Clean your Docker container running a mailserve migration: DEFAULT_MIGRATION_PATH := appdatabase/migrations/sql migration: - touch $(DEFAULT_MIGRATION_PATH)/$(shell date +%s)_$(D).up.sql + touch $(DEFAULT_MIGRATION_PATH)/$$(date '+%s')_$(D).up.sql migration-check: bash _assets/scripts/migration_check.sh migration-wallet: DEFAULT_WALLET_MIGRATION_PATH := walletdatabase/migrations/sql migration-wallet: - touch $(DEFAULT_WALLET_MIGRATION_PATH)/$(shell date +%s)_$(D).up.sql + touch $(DEFAULT_WALLET_MIGRATION_PATH)/$$(date +%s)_$(D).up.sql install-git-hooks: @ln -sf $(if $(filter $(detected_OS), Linux),-r,) \ @@ -428,7 +452,7 @@ install-git-hooks: migration-protocol: DEFAULT_PROTOCOL_PATH := protocol/migrations/sqlite migration-protocol: - touch $(DEFAULT_PROTOCOL_PATH)/$(shell date +%s)_$(D).up.sql + touch $(DEFAULT_PROTOCOL_PATH)/$$(date +%s)_$(D).up.sql PROXY_WRAPPER_PATH = $(CURDIR)/vendor/github.com/siphiuel/lc-proxy-wrapper -include $(PROXY_WRAPPER_PATH)/Makefile.vars diff --git a/_assets/build/Dockerfile b/_assets/build/Dockerfile index 41a66f4c1..cef35c25d 100644 --- a/_assets/build/Dockerfile +++ b/_assets/build/Dockerfile @@ -10,7 +10,7 @@ ARG build_target=statusgo RUN mkdir -p /go/src/github.com/status-im/status-go WORKDIR /go/src/github.com/status-im/status-go ADD . . -RUN make $build_target BUILD_TAGS="$build_tags" BUILD_FLAGS="$build_flags" +RUN make $build_target BUILD_TAGS="$build_tags" BUILD_FLAGS="$build_flags" SHELL="/bin/sh" # Copy the binary to the second image FROM alpine:latest diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index c303f3d83..59e87b5c6 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -1,5 +1,5 @@ #!/usr/bin/env groovy -library 'status-jenkins-lib@v1.7.0' +library 'status-jenkins-lib@v1.8.10' pipeline { agent { label 'linux' } diff --git a/_assets/ci/Jenkinsfile.android b/_assets/ci/Jenkinsfile.android index 159c21016..271c59cd6 100644 --- a/_assets/ci/Jenkinsfile.android +++ b/_assets/ci/Jenkinsfile.android @@ -1,5 +1,5 @@ #!/usr/bin/env groovy -library 'status-jenkins-lib@v1.7.0' +library 'status-jenkins-lib@v1.8.10' pipeline { agent { label 'linux && x86_64 && nix-2.14' } @@ -31,13 +31,13 @@ pipeline { } environment { - TARGET = 'android' - TMPDIR = "${WORKSPACE_TMP}" - GOPATH = "${WORKSPACE_TMP}/go" - GOCACHE = "${WORKSPACE_TMP}/gocache" - PATH = "${PATH}:${GOPATH}/bin" - REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go" - ARTIFACT = utils.pkgFilename(name: "status-go", type: "android", ext: "aar", version: null) + PLATFORM = 'android' + TMPDIR = "${WORKSPACE_TMP}" + GOPATH = "${WORKSPACE_TMP}/go" + GOCACHE = "${WORKSPACE_TMP}/gocache" + PATH = "${PATH}:${GOPATH}/bin" + REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go" + ARTIFACT = utils.pkgFilename(name: "status-go", type: "android", ext: "aar", version: null) } stages { diff --git a/_assets/ci/Jenkinsfile.docker b/_assets/ci/Jenkinsfile.docker index dc7cfaec2..f216eb303 100644 --- a/_assets/ci/Jenkinsfile.docker +++ b/_assets/ci/Jenkinsfile.docker @@ -1,5 +1,5 @@ #!/usr/bin/env groovy -library 'status-jenkins-lib@v1.7.0' +library 'status-jenkins-lib@v1.8.10' pipeline { agent { label 'linux' } @@ -38,11 +38,11 @@ pipeline { } environment { - TARGET = "docker" - REPO = "${env.WORKSPACE}/src/github.com/status-im/status-go" - GOPATH = "${env.WORKSPACE}" - GOCACHE = "${WORKSPACE_TMP}/gocache" - PATH = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin" + PLATFORM = "docker" + REPO = "${env.WORKSPACE}/src/github.com/status-im/status-go" + GOPATH = "${env.WORKSPACE}" + GOCACHE = "${WORKSPACE_TMP}/gocache" + PATH = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin" /* Makefile parameters */ DOCKER_IMAGE_NAME = 'statusteam/status-go' DOCKER_IMAGE_CUSTOM_TAG = "ci-build-${utils.gitCommit()}" diff --git a/_assets/ci/Jenkinsfile.ios b/_assets/ci/Jenkinsfile.ios index efd62fda9..33bf06a21 100644 --- a/_assets/ci/Jenkinsfile.ios +++ b/_assets/ci/Jenkinsfile.ios @@ -1,5 +1,5 @@ #!/usr/bin/env groovy -library 'status-jenkins-lib@v1.7.0' +library 'status-jenkins-lib@v1.8.10' pipeline { agent { label 'macos && aarch64 && xcode-15.1 && nix-2.19' } @@ -31,7 +31,7 @@ pipeline { } environment { - TARGET = 'ios' + PLATFORM = 'ios' TMPDIR = "${WORKSPACE_TMP}" GOPATH = "${WORKSPACE_TMP}/go" GOCACHE = "${WORKSPACE_TMP}/gocache" diff --git a/_assets/ci/Jenkinsfile.linux b/_assets/ci/Jenkinsfile.linux index 25f6d17b6..8bce253cf 100644 --- a/_assets/ci/Jenkinsfile.linux +++ b/_assets/ci/Jenkinsfile.linux @@ -1,5 +1,5 @@ #!/usr/bin/env groovy -library 'status-jenkins-lib@v1.7.0' +library 'status-jenkins-lib@v1.8.10' pipeline { agent { label 'linux && x86_64 && nix-2.14' } @@ -31,7 +31,7 @@ pipeline { } environment { - TARGET = 'linux' + PLATFORM = 'linux' TMPDIR = "${WORKSPACE_TMP}" GOPATH = "${WORKSPACE_TMP}/go" GOCACHE = "${WORKSPACE_TMP}/gocache" diff --git a/_assets/ci/Jenkinsfile.tests b/_assets/ci/Jenkinsfile.tests index 28e750d49..0f6a434d2 100644 --- a/_assets/ci/Jenkinsfile.tests +++ b/_assets/ci/Jenkinsfile.tests @@ -1,5 +1,5 @@ #!/usr/bin/env groovy -library 'status-jenkins-lib@v1.7.0' +library 'status-jenkins-lib@v1.8.10' pipeline { agent { label 'linux && x86_64 && nix-2.14' } @@ -46,7 +46,7 @@ pipeline { } environment { - TARGET = 'tests' + PLATFORM = 'tests' DB_CONT = "status-go-test-db-${env.EXECUTOR_NUMBER.toInteger() + 1}" DB_PORT = "${5432 + env.EXECUTOR_NUMBER.toInteger()}" TMPDIR = "${WORKSPACE_TMP}" @@ -66,7 +66,6 @@ pipeline { stage('Vendor Check') { steps { script { - nix.shell('make install-modvendor', pure: false) nix.shell('make vendor', pure: false) /* fail build if vendoring hasn't been done */ nix.shell('git diff --exit-code --no-color --stat vendor/') diff --git a/_assets/scripts/install_deps.sh b/_assets/scripts/install_deps.sh deleted file mode 100755 index 936056516..000000000 --- a/_assets/scripts/install_deps.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -if [[ -x $(command -v apt-get) ]]; then - apt-get install -y protobuf-compiler jq -elif [[ -x $(command -v pacman) ]]; then - pacman -Sy protobuf jq --noconfirm -elif [[ -x $(command -v brew) ]]; then - brew install protobuf jq -elif [[ -x $(command -v nix-env) ]]; then - nix-env -iA nixos.protobuf3_17 -else - echo "ERROR: No known package manager found!" >&2 - exit 1 -fi diff --git a/_docs/how-to-build.md b/_docs/how-to-build.md index 595fd3599..69c72c49c 100644 --- a/_docs/how-to-build.md +++ b/_docs/how-to-build.md @@ -8,9 +8,11 @@ status-go is an underlying part of Status. It heavily depends on [go-ethereum](h ### 1. Requirements -* Go version >=1.18 (but check go.mod anyway). +* Nix (Installed automatically) * Docker (only if cross-compiling). +> go is provided by Nix + ### 2. Clone the repository ```shell @@ -20,20 +22,7 @@ cd status-go ### 3. Set up build environment -status-go uses Makefile to perform the most common actions. See `make help` output for available commands. -The first thing to do to get started is to run - -```shell -make setup -``` - -That’ll ensure that all tools required to do a first build are installed and set up. This script prepares and installs the following: -* golangci-lint -* mockgen -* go-bindata -* protobuf compiler and protoc-gen-go -* jq -* modvendor +status-go uses nix in the Makefile to provide every tools required. ### 4. Build the statusd CLI @@ -49,20 +38,6 @@ Once that is completed, you can run it straight away with a default configuratio build/bin/statusd ``` -*Known issues:* - -- You need to downgrade golang version to v1.20 to fix the quic compile issues. -- Add `$HOME/go/bin` in `.zshrc` or `.bashrc` of home folder to fix `modvendor: not found...` -- Fix `undefined: secp256k1.VerifySignature` on Ubuntu, - ```shell - go env -w CGO_ENABLED=1 - - # optional - apt-get update - # optional - apt-get install build-essential - ``` - ### 5. Build a library for Android and iOS ```shell diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..4203ea986 --- /dev/null +++ b/default.nix @@ -0,0 +1 @@ +import ./nix diff --git a/nix/KNOWN_ISSUES.md b/nix/KNOWN_ISSUES.md new file mode 100644 index 000000000..893beba96 --- /dev/null +++ b/nix/KNOWN_ISSUES.md @@ -0,0 +1,10 @@ +# Known Issues + +## Golang version mismatch + +If the go compilation run in error with a version mismatch, unset the variable `GOROOT` + +``` +compile: version "go1.20.13" does not match go tool version "go1.19.9" +# golang.org/x/text/internal/utf8internal +``` diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 000000000..68f4e20d2 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,29 @@ +# Description + +This folder contains configuration for [Nix](https://nixos.org/), a purely functional package manager used by the Status Go for its build process. + +## Configuration + +The main config file is [`nix/nix.conf`](/nix/nix.conf) and its main purpose is defining the [binary caches](https://nixos.org/nix/manual/#ch-basic-package-mgmt) which allow download of packages to avoid having to compile them yourself locally. + + +## Shell + +In order to access an interactive Nix shell a user should run `make shell`. + +The Nix shell is started in this repo via the [`nix/scripts/shell.sh`](/nix/scripts/shell.sh) script, which is a wrapper around the `nix-shell` command and is intended for use with our main [`Makefile`](/Makefile). This allows for an implicit use of `nix-shell` as the default shell in the `Makefile`. + +:warning: __WARNING__: To have Nix pick up all changes a new `nix-shell` needs to be spawned. + +## Resources + +You can learn more about Nix by watching these presentations: + +* [Nix Fundamentals](https://www.youtube.com/watch?v=m4sv2M9jRLg) ([PDF](https://drive.google.com/file/d/1Tt5R7QOubudGiSuZIGxuFWB1OYgcThcL/view?usp=sharing), [src](https://github.com/status-im/infra-docs/tree/master/presentations/nix_basics)) +* [Nix in Status](https://www.youtube.com/watch?v=rEQ1EvRG8Wc) ([PDF](https://drive.google.com/file/d/1Ti0wppMoj40icCPdHy7mJcQj__DeaYBE/view?usp=sharing), [src](https://github.com/status-im/infra-docs/tree/master/presentations/nix_in_status)) + +And you can read [`nix/DETAILS.md`](./DETAILS.md) for more information. + +## Known Issues + +See [`KNOWN_ISSUES.md`](./KNOWN_ISSUES.md). diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 000000000..ecfe8bfc6 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,10 @@ +{ + config ? {}, + pkgs ? import ./pkgs.nix { inherit config; } +}: + let + # put all main targets and shells together for easy import + shells = pkgs.callPackage ./shells.nix { }; +in { + inherit pkgs shells; +} diff --git a/nix.conf b/nix/nix.conf similarity index 100% rename from nix.conf rename to nix/nix.conf diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 000000000..97cdaf52c --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,67 @@ +# Override some packages and utilities in 'pkgs' +# and make them available globally via callPackage. +# +# For more details see: +# - https://nixos.wiki/wiki/Overlays +# - https://nixos.org/nixos/nix-pills/callpackage-design-pattern.html +final: prev: +let + inherit (prev) callPackage; +in rec { + androidPkgs = prev.androidenv.composeAndroidPackages { + toolsVersion = "26.1.1"; + platformToolsVersion = "33.0.3"; + buildToolsVersions = [ "31.0.0" ]; + platformVersions = [ "31" ]; + cmakeVersions = [ "3.18.1" ]; + ndkVersion = "22.1.7171670"; + includeNDK = true; + includeExtras = [ + "extras;android;m2repository" + "extras;google;m2repository" + ]; + }; + + go = prev.go_1_19; + buildGoModule = prev.buildGo119Module; + buildGoPackage = prev.buildGo119Package; + + golangci-lint = prev.golangci-lint.override { + buildGoModule = args: prev.buildGo119Module ( args // rec { + version = "1.52.2"; + src = prev.fetchFromGitHub { + owner = "golangci"; + repo = "golangci-lint"; + rev = "v${version}"; + hash = "sha256-FmNXjOMDDdGxMQvy5f1NoaqrKFpmlPWclXooMxXP8zg="; + }; + vendorHash = "sha256-BhD3a0LNc3hpiH4QC8FpmNn3swx3to8+6gfcgZT8TLg="; + }); + }; + + go-junit-report = prev.go-junit-report.overrideAttrs ( attrs : rec { + version = "2.1.0"; + src = prev.fetchFromGitHub { + owner = "jstemmer"; + repo = "go-junit-report"; + rev = "v${version}"; + sha256 = "sha256-s4XVjACmpd10C5k+P3vtcS/aWxI6UkSUPyxzLhD2vRI="; + }; + }); + + # Custom packages + go-modvendor = callPackage ./pkgs/go-modvendor { }; + + gomobile = (prev.gomobile.overrideAttrs (old: { + patches = [ + (final.fetchurl { # https://github.com/golang/mobile/pull/84 + url = "https://github.com/golang/mobile/commit/f20e966e05b8f7e06bed500fa0da81cf6ebca307.patch"; + sha256 = "sha256-TZ/Yhe8gMRQUZFAs9G5/cf2b9QGtTHRSObBFD5Pbh7Y="; + }) + (final.fetchurl { # https://github.com/golang/go/issues/58426 + url = "https://github.com/golang/mobile/commit/406ed3a7b8e44dc32844953647b49696d8847d51.patch"; + sha256 = "sha256-dqbYukHkQEw8npOkKykOAzMC3ot/Y4DEuh7fE+ptlr8="; + }) + ]; + })); +} diff --git a/nix/pkgs.nix b/nix/pkgs.nix new file mode 100644 index 000000000..8ef6ae8fb --- /dev/null +++ b/nix/pkgs.nix @@ -0,0 +1,28 @@ +# This file controls the pinned version of nixpkgs we use for our Nix environment +# as well as which versions of package we use, including their overrides. +{ config ? { } }: + +let + # For testing local version of nixpkgs + #nixpkgsSrc = (import { }).lib.cleanSource "/home/jakubgs/work/nixpkgs"; + + # We follow the master branch of official nixpkgs. + nixpkgsSrc = builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/224fd9a362487ab2894dac0df161c84ab1d8880b.tar.gz"; + sha256 = "sha256:1syvl39pi1h8lf5gkd9h7ksn5hp34cj7pa3abr59217kv0bdklhy"; + }; + + # Status specific configuration defaults + defaultConfig = { + allowUnfree = true; + android_sdk.accept_license = true; + }; + # Override some packages and utilities + pkgsOverlay = import ./overlay.nix; + +in + # import nixpkgs with a config override + (import nixpkgsSrc) { + config = defaultConfig // config; + overlays = [pkgsOverlay]; + } diff --git a/nix/pkgs/go-modvendor/default.nix b/nix/pkgs/go-modvendor/default.nix new file mode 100644 index 000000000..8c546c66b --- /dev/null +++ b/nix/pkgs/go-modvendor/default.nix @@ -0,0 +1,14 @@ +{ buildGoModule, fetchFromGitHub }: + +buildGoModule rec { + pname = "go-modvendor"; + version = "0.5.0"; + vendorHash = null; + + src = fetchFromGitHub rec { + owner = "goware"; + repo = "modvendor"; + rev = "v${version}"; + hash = "sha256-6Zht3XukH6rZaiz9aNQI+SXuonqw7k2LiElLPH2Zkwo="; + }; +} diff --git a/nix/scripts/build.sh b/nix/scripts/build.sh new file mode 100755 index 000000000..43f43578c --- /dev/null +++ b/nix/scripts/build.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# This script is a wrapper around nix-build with some niceties. +set -e + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +resultPath="${GIT_ROOT}/result/" +source "${GIT_ROOT}/scripts/colors.sh" +source "${GIT_ROOT}/nix/scripts/source.sh" + +# cleanup for artifacts created during builds +cleanup() { + # clear trapped signals + trap - EXIT ERR INT QUIT + # do the actual cleanup, ignore failure + if ${GIT_ROOT}/nix/scripts/clean.sh "${nixResultPath}"; then + echo -e "${GRN}Successful cleanup!${RST}" + elif [[ -n "${JENKINS_URL}" ]]; then + # in CI removing some paths can fail due to parallel builds + echo -e "${YLW}Ignoring cleanup failure in CI.${RST}" + else + echo -e "${RED}Failed cleanup!${RST}" + exit 1 + fi +} + +# If you want to clean after every build set _NIX_CLEAN=true +if [[ -n "${_NIX_CLEAN}" ]]; then + trap cleanup EXIT ERR INT QUIT +fi + +# build output will end up under /nix, we have to extract it +extractResults() { + local nixResultPath="$1" + mkdir -p "${resultPath}" + cp -vfr ${nixResultPath}/* "${resultPath}" | sed 's#'${PWD}'#.#' + chmod -R u+w "${resultPath}" +} + +TARGET="${1}" +shift + +if [[ -z "${TARGET}" ]]; then + echo -e "${RED}First argument is mandatory and has to specify the Nix attribute!${RST}" + exit 1 +fi + +# Hack fix for missing Android SDK for aarch64 on Darwin. See systemOverride in `nix/pkgs.nix`. +if [[ "${TARGET}" =~ ^(targets.status-go.mobile.android|targets.mobile.android.release)$ ]]; then + os=$(uname -s | tr '[:upper:]' '[:lower:]') + export NIXPKGS_SYSTEM_OVERRIDE="x86_64-${os}" +fi + +# Some defaults flags, --pure could be optional in the future. +# NOTE: The --keep-failed flag can be used for debugging issues. +nixOpts=( + "--pure" + "--fallback" + "--no-out-link" + "--show-trace" + "--attr" "${TARGET}" +) + +# Save derivation from being garbage collected +"${GIT_ROOT}/nix/scripts/gcroots.sh" "${TARGET}" "${@}" + +# Run the actual build +echo -e "${GRN}Running:${RST} ${BLD}nix-build "${nixOpts[@]}" ${@}${RST}" +nixResultPath=$(nix-build "${nixOpts[@]}" "${@}" shell.nix) + +echo -e "\n${YLW}Extracting result${RST}: ${BLD}${nixResultPath}${RST}" + +extractResults "${nixResultPath}" + +echo -e "\n${GRN}SUCCESS${RST}" diff --git a/nix/scripts/clean.sh b/nix/scripts/clean.sh new file mode 100755 index 000000000..d40fa7d17 --- /dev/null +++ b/nix/scripts/clean.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -e + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/nix/scripts/source.sh" + +log() { echo "$@" 1>&2; } + +# helpers for getting related paths in Nix store +getSources() { nix-store --query --binding src "${1}"; } +getOutputs() { nix-store --query --outputs "${1}"; } +getDrvFiles() { nix-store --query --deriver "${1}"; } +getReferrers() { nix-store --query --referrers "${1}"; } +getRoots() { nix-store --query --roots "${1}"; } + +findRelated() { + path="${1}" + found+=("${path}") + if [[ "${path}" =~ .*.chroot ]]; then + log " ! Chroot: ${path}" + return + elif [[ "${path}" =~ .*.lock ]]; then + log " ! Lock: ${path}" + return + elif [[ "${path}" =~ .*status-mobile-shell.drv ]]; then + echo -n "${path}" + return + fi + log " ? Checking: ${path}" + drv=$(getDrvFiles "${path}") + # if drv is unknown-deriver then path is a source + if [[ "${drv}" == "unknown-deriver" ]]; then + drv=$(getReferrers "${path}" | head -n1) + src="${path}" + elif [[ -f "${drv}" ]]; then + src=$(getSources "${drv}") + fi + # empty paths means this is a source + if [[ -z "${drv}" ]]; then + echo -n "${src}" + return + fi + if [[ $(getRoots "${drv}" | wc -l) -eq 0 ]]; then + log " - Derivation: ${drv}" + log " - Source: ${src}" + found+=("${drv}" "${src}") + fi + + printf '%s\n' "${found[@]}" +} + +# used to find things to delete based on a regex +findByRegex() { + regex="${1}" + + log "Searching by regex: '${regex}'" + # search for matching entries in the store + drvPaths=$( + nix-store --gc --print-dead 2> /dev/null | grep -E "${regex}" + ) + + # list of store entries to delete + declare -a found + + # for each entry find the source and derivation + for mainPath in ${drvPaths}; do + findRelated "${mainPath}" + done +} + +# used to find things to delete based on a given path +findByResult() { + mainPath="${1}" + log "Searching by result: '${mainPath}'" + + # list of store entries to delete + declare -a found + + findRelated "${mainPath}" +} + +log "Cleanup of /nix/store..." + +# This is an optional CLI argument +nixResultPath="${1}" +if [[ -n "${nixResultPath}" ]]; then + # if provided we can narrow down what to clean based on result path + toDelete=$(findByResult "${nixResultPath}") +else + # use regular expression that should match all status-mobile build artifacts + toDelete=$(findByRegex '.*-status-(react|go)-(shell|source|build|patched-npm-gradle-modules).*') +fi + +# remove duplicates and return +toDelete=$(printf '%s\n' "${toDelete[@]}" | sort | uniq) + +log "Deleting..." +nix-store --delete ${toDelete[@]} diff --git a/nix/scripts/gcroots.sh b/nix/scripts/gcroots.sh new file mode 100755 index 000000000..aae6cea2e --- /dev/null +++ b/nix/scripts/gcroots.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -Ee + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/nix/scripts/source.sh" +source "${GIT_ROOT}/scripts/colors.sh" + +_NIX_GCROOTS="${_NIX_GCROOTS:-${GIT_ROOT}/.nix-gcroots}" + +TARGET="${1}" +shift +if [[ -z "${TARGET}" ]]; then + echo -e "${RED}No target specified for gcroots.sh!${RST}" >&2 + exit 1 +fi + +# Creates a symlink to derivation in _NIX_GCROOTS directory. +# This prevents it from being removed by 'gc-collect-garbage'. +nix-instantiate --add-root "${_NIX_GCROOTS}/${TARGET}" \ + "${@}" "${GIT_ROOT}/shell.nix" >/dev/null diff --git a/nix/scripts/lib.sh b/nix/scripts/lib.sh new file mode 100755 index 000000000..74a550f3f --- /dev/null +++ b/nix/scripts/lib.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -eo pipefail +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) + +# Checking group ownership to identify installation type. +file_group() { + UNAME=$(uname -s) + if [[ "${UNAME}" == "Linux" ]]; then + stat -Lc "%G" "${1}" 2>/dev/null + elif [[ "${UNAME}" == "Darwin" ]]; then + # Avoid using Nix GNU stat when in Nix shell. + /usr/bin/stat -Lf "%Sg" "${1}" 2>/dev/null + fi +} + +os_name() { + source /etc/os-release 2>/dev/null + echo "${NAME}" +} + +is_arch_linux() { + [[ -f /etc/arch-release ]] +} + +nix_install_type() { + NIX_STORE_DIR_GROUP=$(file_group /nix/store) + if [[ "$(os_name)" =~ NixOS ]]; then + echo "nixos" + else + case "${NIX_STORE_DIR_GROUP}" in + "nixbld") echo "multi";; + "30000") echo "multi";; + "(30000)") echo "multi";; + "wheel") echo "single";; + "users") echo "single";; + "${USER}") echo "single";; + "${UID}") echo "single";; + "(${UID})") echo "single";; + "") echo "none"; + echo "No Nix installtion detected!" >&2;; + *) echo "Unknown Nix installtion type!" >&2; exit 1;; + esac + fi +} + +nix_root() { + NIX_ROOT="/nix" + if [[ $(uname -s) == "Darwin" ]]; then + # Special case due to read-only root on MacOS Catalina + NIX_ROOT="/opt/nix" + fi + echo "${NIX_ROOT}" +} + +nix_current_version() { + nix-env --version | awk '{print $3}' +} + +nix_get_local_setting() { + local NIX_LOCAL_CONFIG="${GIT_ROOT}/nix/nix.conf" + local KEY="${1}" + awk -F' = ' "/^${KEY} *=/{print \$2}" nix/nix.conf +} + +nix_set_global_setting() { + local NIX_GLOBAL_CONFIG="/etc/nix/nix.conf" + local KEY="${1}" + local VAL="${2}" + if grep "${KEY}" "${NIX_GLOBAL_CONFIG}" 2>/dev/null; then + sed -i "s/${KEY} = \(.*\)$/${KEY} = ${VAL}/" "${NIX_GLOBAL_CONFIG}" + else + echo "${KEY} = ${VAL}" | sudo tee -a "${NIX_GLOBAL_CONFIG}" >/dev/null + fi +} + +nix_daemon_restart() { + # Restarting Nix Daemon makes sense only on a multi-user install. + [[ $(nix_install_type) != "multi" ]] && return + if [[ "$(uname -s)" == "Darwin" ]]; then + echo "Restarting Nix daemon Launchd service..." >&2 + sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist + elif [[ "$(uname -s)" == "Linux" ]] && [[ "$(nix_install_type)" == "multi" ]]; then + echo "Restarting Nix daemon Systemd service..." >&2 + sudo systemctl daemon-reload + sudo systemctl restart nix-daemon + else + echo "Unknown platform! Unable to restart daemon!" >&2 + exit 1 + fi +} diff --git a/nix/scripts/purge.sh b/nix/scripts/purge.sh new file mode 100755 index 000000000..ae24fa5a4 --- /dev/null +++ b/nix/scripts/purge.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# This script removes all Nix files. + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/nix/scripts/lib.sh" +source "${GIT_ROOT}/scripts/colors.sh" + +nix_purge_linux_multi_user_service() { + NIX_SERVICES=(nix-daemon.service nix-daemon.socket) + for NIX_SERVICE in "${NIX_SERVICES[@]}"; do + sudo systemctl stop "${NIX_SERVICE}" + sudo systemctl disable "${NIX_SERVICE}" + done + sudo systemctl daemon-reload +} + +nix_purge_linux_multi_user_users() { + for NIX_USER in $(awk -F: '/nixbld/{print $1}' /etc/passwd); do + sudo userdel "${NIX_USER}" + done + sudo groupdel nixbld +} + +nix_purge_darwin_multi_user_service() { + cd /Library/LaunchDaemons + NIX_SERVICES=(org.nixos.darwin-store.plist org.nixos.nix-daemon.plist) + for NIX_SERVICE in "${NIX_SERVICES[@]}"; do + sudo launchctl unload "${NIX_SERVICE}" + sudo launchctl remove "${NIX_SERVICE}" + done +} + +nix_purge_darwin_multi_user_users() { + for NIX_USER in $(dscl . list /Users | grep nixbld); do + sudo dscl . -delete "/Users/${NIX_USER}" + done + sudo dscl . -delete /Groups/nixbld +} + +# This still leaves an empty /nix, which will disappear after reboot. +nix_purge_darwin_multi_user_volumes() { + sudo sed -i.bkp '/nix/d' /etc/synthetic.conf + sudo sed -i.bkp '/nix/d' /etc/fstab + sudo diskutil apfs deleteVolume /nix + echo -e "${YLW}You will need to reboot your system!${RST}" >&2 +} + +nix_purge_multi_user() { + if [[ $(uname -s) == "Darwin" ]]; then + nix_purge_darwin_multi_user_service + nix_purge_darwin_multi_user_users + nix_purge_darwin_multi_user_volumes + else + nix_purge_linux_multi_user_service + nix_purge_linux_multi_user_users + fi + + sudo rm -fr /etc/nix + sudo rm -f /etc/profile.d/nix.sh* + + # Restore old shell profiles + NIX_PROFILE_FILES=( + /etc/bash.bashrc /etc/bashrc /etc/bash/bashrc + /etc/zsh.zshhrc /etc/zshrc /etc/zsh/zshrc + ) + for NIX_FILE in "${NIX_PROFILE_FILES[@]}"; do + if [[ -f "${NIX_FILE}.backup-before-nix" ]]; then + sudo mv -f "${NIX_FILE}.backup-before-nix" "${NIX_FILE}" + fi + done +} + +nix_purge_user_profile() { + sudo rm -rf \ + ~/.nix-* \ + ~/.cache/nix \ + ~/.config/nixpkgs \ + "${GIT_ROOT}/.nix-gcroots" +} + +nix_purge_root() { + NIX_ROOT=$(nix_root) + if [[ -z "${NIX_ROOT}" ]]; then + echo -e "${RED}Unable to identify Nix root!${RST}" >&2 + exit 1 + fi + sudo rm -fr "${NIX_ROOT}" +} + +# Don't run anything if script is just sourced. +if (return 0 2>/dev/null); then + echo -e "${YLW}Script sourced, not running purge.${RST}" + return +fi + +# Confirm user decission, unless --force is used. +if [[ "${1}" != "--force" ]]; then + echo -e "${YLW}Are you sure you want to purge Nix?${RST}" >&2 + read -p "[y/n]: " -n 1 -r + echo + if [[ $REPLY =~ ^[^Yy]$ ]]; then + echo -e "${GRN}Aborting Nix purge!${RST}" >&2 + exit 0 + fi +fi + +NIX_INSTALL_TYPE=$(nix_install_type) +# Purging /nix on NixOS would be disasterous. +if [[ "${NIX_INSTALL_TYPE}" == "nixos" ]]; then + echo -e "${RED}You should not purge Nix files on NixOS!${RST}" >&2 + exit +elif [[ "${NIX_INSTALL_TYPE}" == "none" ]] && [[ "${1}" != "--force" ]]; then + echo -e "${YLW}Nothing to remove, Nix not installed.${RST}" >&2 + exit +elif [[ "${NIX_INSTALL_TYPE}" == "multi" ]] || [[ "${1}" == "--force" ]]; then + echo -e "${YLW}Detected multi-user Nix installation.${RST}" >&2 + nix_purge_multi_user +elif [[ "${NIX_INSTALL_TYPE}" == "single" ]] || [[ "${1}" == "--force" ]]; then + echo -e "${YLW}Detected single-user Nix installation.${RST}" >&2 + nix_purge_user_profile +fi +nix_purge_root + +echo -e "${GRN}Purged all Nix files from your system.${RST}" >&2 diff --git a/nix/scripts/setup.sh b/nix/scripts/setup.sh new file mode 100755 index 000000000..00843d6b0 --- /dev/null +++ b/nix/scripts/setup.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# This script installs a specific version of Nix. +set -eo pipefail + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/scripts/colors.sh" +source "${GIT_ROOT}/nix/scripts/lib.sh" +source "${GIT_ROOT}/nix/scripts/version.sh" + +nix_install() { + # Download installer and verify SHA256> + curl -sSf "${NIX_INSTALL_URL}" -o "${NIX_INSTALL_PATH}" + echo "${NIX_INSTALL_SHA256} ${NIX_INSTALL_PATH}" | sha256sum -c + chmod +x "${NIX_INSTALL_PATH}" + + # Identify installation type. + if [[ -z "${NIX_INSTALL_OPTS}" ]]; then + if [[ "$(uname -r)" =~ microsoft ]]; then + # Systemd is not started by default on WSL. + NIX_INSTALL_OPTS="--no-daemon" + elif [[ "$(uname -s)" == "Darwin" ]]; then + # Single-user not supported on Darwin. + NIX_INSTALL_OPTS="--daemon" + elif [[ "$(uname -s)" == "Linux" ]]; then + # Open file limit issues on Linux. + # https://github.com/NixOS/nix/issues/6007 + # Alson known issues with nix-daemon.socket on Arch. + NIX_INSTALL_OPTS="--no-daemon" + fi + fi + + # Run the installer + "${NIX_INSTALL_PATH}" "${NIX_INSTALL_OPTS}" + if [[ $? -eq 0 ]]; then + echo -e "${GRN}The Nix package manager was successfully installed.${RST}" + else + echo -e "${RED}Failed to install Nix package manager!${RST}" >&2 + echo "Please see: https://nixos.org/nix/manual/#chap-installation" >&2 + exit 1 + fi + + # Additional fixes + nix_add_extra_cache + nix_daemon_restart +} + +# Adding directly to global config to avoid warnings like this: +# "ignoring untrusted substituter 'https://nix-cache.status.im/', you are not a trusted user." +nix_add_extra_cache() { + # Single-user installations do not have this issue. + [[ ! -f /etc/nix/nix.conf ]] && return + echo -e 'Adding our cache to Nix daemon config...' >&2 + local NIX_SETTINGS=('substituters' 'trusted-substituters' 'trusted-public-keys') + for NIX_SETTING in "${NIX_SETTINGS[@]}"; do + nix_set_global_setting "${NIX_SETTING}" "$(nix_get_local_setting "${NIX_SETTING}")" + done +} + +if [[ ! -x "$(command -v sha256sum)" ]]; then + echo -e "${RED}The 'sha256sum' utility is required for Nix installation.${RST}" >&2 + echo -e "${YLW}Install 'coreutils' package on your system.${RST}" >&2 + exit 1 +fi + +if [[ ! -x "$(command -v curl)" ]]; then + echo -e "${RED}The 'curl' utility is required for Nix installation.${RST}" >&2 + exit 1 +fi + +if [[ "$(source /etc/os-release 2>/dev/null && echo "${NAME}")" == *NixOS* ]]; then + echo -e "${GRN}Already running NixOS.${RST}" + exit +fi + +if [[ -x "$(command -v nix)" ]]; then + echo -e "${GRN}Nix package manager already installed.${RST}" + exit +fi + +if [[ "${IN_NIX_SHELL}" == 'pure' ]]; then + echo -e "${GRN}Already in a pure Nix shell.${RST}" + exit +fi + +# If none of the checks before succeeded we need to install Nix +echo -e "${GRN}Setting up Nix package manager...${RST}" +nix_install +echo -e "${YLW}See STARTING_GUIDE.md if you're new here.${RST}" diff --git a/nix/scripts/shell.sh b/nix/scripts/shell.sh new file mode 100755 index 000000000..32e3384ca --- /dev/null +++ b/nix/scripts/shell.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# This script is used by the Makefile to have an implicit nix-shell. +# The following environment variables modify the script behavior: +# - TARGET: This attribute is passed via --attr to Nix, defining the scope. +# - _NIX_PURE: This variable allows for making the shell pure with the use of --pure. +# Take note that this makes Nix tools like `nix-build` unavailable in the shell. +# - _NIX_KEEP: This variable allows specifying which env vars to keep for Nix pure shell. + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/scripts/colors.sh" +source "${GIT_ROOT}/nix/scripts/source.sh" + +export TERM=xterm # fix for colors +shift # we remove the first -c from arguments + +if [[ -z "${TARGET}" ]]; then + export TARGET="default" + echo -e "${YLW}Missing TARGET, assuming default target.${RST} See nix/README.md for more details." 1>&2 +fi + +# Minimal shell with just Nix sourced, useful for `make nix-gc`. +if [[ "${TARGET}" == "nix" ]]; then + eval $@ + exit 0 +fi +if [[ -n "${IN_NIX_SHELL}" ]] && [[ -n "${NIX_SHELL_TARGET}" ]]; then + if [[ "${NIX_SHELL_TARGET}" == "${TARGET}" ]]; then + echo -e "${YLW}Nix shell for TARGET=${TARGET} is already active.${RST}" >&2 + exit 0 + else + # Nesting nix shells does not work due to how we detect already present shell. + echo -e "${RED}Cannot nest Nix shells with different targets!${RST}" >&2 + exit 1 + fi +fi + +entryPoint="default.nix" +nixArgs=( + "--show-trace" + "--attr shells.${TARGET}" +) + +# This variable allows specifying which env vars to keep for Nix pure shell +# The separator is a colon +if [[ -n "${_NIX_KEEP}" ]]; then + nixArgs+=("--keep ${_NIX_KEEP//,/ --keep }") +fi + +# Not all builds are ready to be run in a pure environment +if [[ -n "${_NIX_PURE}" ]]; then + nixArgs+=("--pure") + pureDesc='pure ' +fi + +# Hack fix for missing Android SDK for aarch64 on Darwin. See systemOverride in `nix/pkgs.nix`. +if [[ "${TARGET}" =~ ^(android-sdk|android|gradle|keytool|status-go)$ ]]; then + os=$(uname -s | tr '[:upper:]' '[:lower:]') + export NIXPKGS_SYSTEM_OVERRIDE="x86_64-${os}" +fi + +echo -e "${GRN}Configuring ${pureDesc}Nix shell for target '${TARGET}'...${RST}" 1>&2 + +# Save derivation from being garbage collected +"${GIT_ROOT}/nix/scripts/gcroots.sh" "shells.${TARGET}" + +# ENTER_NIX_SHELL is the fake command used when `make shell` is run. +# It is just a special string, not a variable, and a marker to not use `--run`. +if [[ "${@}" == "ENTER_NIX_SHELL" ]]; then + export NIX_SHELL_TARGET="${TARGET}" + exec nix-shell ${nixArgs[@]} --keep NIX_SHELL_TARGET ${entryPoint} +else + exec nix-shell ${nixArgs[@]} --run "$@" ${entryPoint} +fi diff --git a/nix/scripts/source.sh b/nix/scripts/source.sh new file mode 100755 index 000000000..5272bebd9 --- /dev/null +++ b/nix/scripts/source.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# This script makes sure we have Nix tools available +set -e + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/nix/scripts/lib.sh" +source "${GIT_ROOT}/scripts/colors.sh" + +source_nix_profile() { + NIX_INSTALL_TYPE=$(nix_install_type) + if [[ "${NIX_INSTALL_TYPE}" == "multi" ]]; then + source "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" + elif [[ "${NIX_INSTALL_TYPE}" == "single" ]]; then + source "${HOME}/.nix-profile/etc/profile.d/nix.sh" + elif [[ "${NIX_INSTALL_TYPE}" == "nixos" ]]; then + echo "Sourcing profile not necessary on NixOS!" >&2 + fi +} + +main() { + # Just stop if Nix is already available + if [[ -x $(command -v nix) ]]; then + return + fi + + # Setup Nix if not available + if [[ ! -d /nix ]]; then + "${GIT_ROOT}/nix/scripts/setup.sh" + fi + + # Load Nix profile + source_nix_profile + + # Verify Nix is available + if [[ ! -x $(command -v nix) ]]; then + echo -e "${RED}Nix not available, sourcing profile failed!${RST}" >&2 + exit 1 + fi +} + +main diff --git a/nix/scripts/upgrade.sh b/nix/scripts/upgrade.sh new file mode 100755 index 000000000..4597e81f5 --- /dev/null +++ b/nix/scripts/upgrade.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# This script upgrades Nix to specific version. +# https://nixos.org/manual/nix/stable/installation/upgrading.html +set -eo pipefail + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/scripts/colors.sh" +source "${GIT_ROOT}/nix/scripts/lib.sh" +source "${GIT_ROOT}/nix/scripts/source.sh" +source "${GIT_ROOT}/nix/scripts/version.sh" + +nix_upgrade() { + echo -e "Upgrading Nix interpreter to: ${GRN}${NIX_VERSION}${RST}" >&2 + nix-channel --update + nix-env --install --attr "nixpkgs.${NIX_PACKAGE}" "nixpkgs.cacert" + nix_daemon_restart +} + +# Allow for sourcing the script +if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + return +fi + +if [[ "$(nix_current_version)" == "${NIX_VERSION}" ]]; then + echo -e "Nix interpreter already on version: ${GRN}${NIX_VERSION}${RST}" + exit 0 +fi + +NIX_INSTALL_TYPE=$(nix_install_type) +if [[ "${NIX_INSTALL_TYPE}" == "nixos" ]]; then + echo -e "${YLW}WARNING:${RST} Upgrade Nix in your NixOS configuration!" >&2 + exit 0 +elif [[ "${NIX_INSTALL_TYPE}" == "single" ]]; then + nix_upgrade +elif [[ "${NIX_INSTALL_TYPE}" == "multi" ]]; then + sudo -i bash -c "source ${PWD}/${0}; nix_upgrade" +fi diff --git a/nix/scripts/version.sh b/nix/scripts/version.sh new file mode 100755 index 000000000..a58158a0c --- /dev/null +++ b/nix/scripts/version.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +export NIX_VERSION="2.19.3" +export NIX_PACKAGE="nixVersions.nix_2_19" +export NIX_INSTALL_URL="https://nixos.org/releases/nix/nix-${NIX_VERSION}/install" +export NIX_INSTALL_SHA256="73d47b0ab783fddca1b2d44a03d52a74c97c28a1fc8ff5a29419079302ef9c3d" +export NIX_INSTALL_PATH="/tmp/nix-install-${NIX_VERSION}" diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 000000000..b8f8e2697 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1,37 @@ +{ config ? {} +, pkgs ? import ./pkgs.nix { inherit config; } }: + +let + inherit (pkgs) lib stdenv; + /* No Android SDK for Darwin aarch64. */ + isMacM1 = stdenv.isDarwin && stdenv.isAarch64; + /* Lock requires Xcode verison. */ + xcodeWrapper = pkgs.xcodeenv.composeXcodeWrapper { + version = "14.3"; + allowHigher = true; + }; + /* Gomobile also needs the Xcode wrapper. */ + gomobileMod = pkgs.gomobile.override { + inherit xcodeWrapper; + withAndroidPkgs = !isMacM1; + }; +in pkgs.mkShell { + name = "status-go-shell"; + + buildInputs = with pkgs; [ + git jq which + go golangci-lint go-junit-report gopls go-bindata gomobileMod + mockgen protobuf3_20 protoc-gen-go gotestsum go-modvendor openjdk + ] ++ lib.optionals (stdenv.isDarwin) [ xcodeWrapper ]; + + shellHook = lib.optionalString (!isMacM1) '' + ANDROID_HOME=${pkgs.androidPkgs.androidsdk}/libexec/android-sdk + ANDROID_NDK=$ANDROID_HOME/ndk-bundle + ANDROID_SDK_ROOT=$ANDROID_HOME + ANDROID_NDK_HOME=$ANDROID_NDK + ''; + # Sandbox causes Xcode issues on MacOS. Requires sandbox=relaxed. + # https://github.com/status-im/status-mobile/pull/13912 + __noChroot = stdenv.isDarwin; +} + diff --git a/nix/shells.nix b/nix/shells.nix new file mode 100644 index 000000000..2d63a08eb --- /dev/null +++ b/nix/shells.nix @@ -0,0 +1,14 @@ +# This file defines custom shells as well as shortcuts +# for accessing more nested shells. +{ config ? {} +, pkgs ? import ./pkgs.nix { inherit config; } }: + +let + inherit (pkgs) lib mkShell callPackage; + default = callPackage ./shell.nix { }; + + shells = { + inherit default; + }; +in + shells diff --git a/scripts/colors.sh b/scripts/colors.sh new file mode 100644 index 000000000..6b19a153f --- /dev/null +++ b/scripts/colors.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Colors +export YLW='\033[1;33m' +export RED='\033[0;31m' +export GRN='\033[0;32m' +export BLU='\033[0;34m' +export BLD='\033[1m' +export RST='\033[0m' + +# Clear line +export CLR='\033[2K' diff --git a/shell.nix b/shell.nix index 2ef2a1ac0..3e3d61e8f 100644 --- a/shell.nix +++ b/shell.nix @@ -1,77 +1,9 @@ -{ - /* This should match Nixpkgs commit in status-mobile. */ - source ? builtins.fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/e7603eba51f2c7820c0a182c6bbb351181caa8e7.tar.gz"; - sha256 = "sha256:0mwck8jyr74wh1b7g6nac1mxy6a0rkppz8n12andsffybsipz5jw"; - }, - pkgs ? import (source){ - config = { - allowUnfree = true; - android_sdk.accept_license = true; - }; - overlays = [ - (final: prev: { - androidPkgs = pkgs.androidenv.composeAndroidPackages { - toolsVersion = "26.1.1"; - platformToolsVersion = "33.0.3"; - buildToolsVersions = [ "31.0.0" ]; - platformVersions = [ "31" ]; - cmakeVersions = [ "3.18.1" ]; - ndkVersion = "22.1.7171670"; - includeNDK = true; - includeExtras = [ - "extras;android;m2repository" - "extras;google;m2repository" - ]; - }; - - go-junit-report = prev.go-junit-report.overrideAttrs ( attrs : rec { - version = "2.1.0"; - - src = prev.fetchFromGitHub { - owner = "jstemmer"; - repo = "go-junit-report"; - rev = "v${version}"; - sha256 = "sha256-s4XVjACmpd10C5k+P3vtcS/aWxI6UkSUPyxzLhD2vRI="; - }; - }); - }) - ]; - } -}: +# for passing build optionsm see nix/README +# TODO complet nix/README +{ config ? { } }: let - inherit (pkgs) lib stdenv; - - /* No Android SDK for Darwin aarch64. */ - isMacM1 = stdenv.isDarwin && stdenv.isAarch64; - /* Lock requires Xcode verison. */ - xcodeWrapper = pkgs.xcodeenv.composeXcodeWrapper { - version = "14.3"; - allowHigher = true; - }; - /* Gomobile also needs the Xcode wrapper. */ - gomobileMod = pkgs.gomobile.override { - inherit xcodeWrapper; - withAndroidPkgs = !isMacM1; - }; -in pkgs.mkShell { - name = "status-go-shell"; - - buildInputs = with pkgs; [ - git jq which - go_1_19 golangci-lint go-junit-report gopls go-bindata gomobileMod - mockgen protobuf3_20 protoc-gen-go gotestsum - ] ++ lib.optional stdenv.isDarwin xcodeWrapper; - - shellHook = lib.optionalString (!isMacM1) '' - ANDROID_HOME=${pkgs.androidPkgs.androidsdk}/libexec/android-sdk - ANDROID_NDK=$ANDROID_HOME/ndk-bundle - ANDROID_SDK_ROOT=$ANDROID_HOME - ANDROID_NDK_HOME=$ANDROID_NDK - ''; - - # Sandbox causes Xcode issues on MacOS. Requires sandbox=relaxed. - # https://github.com/status-im/status-mobile/pull/13912 - __noChroot = stdenv.isDarwin; -} + main = import ./nix { inherit config; }; +in + # use the default shell when calling nix-shell without arguments + main.shells.default