feat: refactor to support building with Nimble (#52)

Changes include:

- Removing all submodules from vendor folder.
- Updating sds.nimble with required depndencies.
- Generating a nimble.lock file using Nimble.
- Updated Nim code to reference depndencies correctly.
- Added nix/deps.nix fixed output derivation that calls Nimble.
- Updated nixpkgs to use 25.11 commit which provides Nimbe 0.20.1.
- Disabled Nix Android builds on MacOS due to Nimble segfault.

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Ivan FB 2026-02-11 16:32:02 +01:00 committed by GitHub
parent 47757bacea
commit 0dea35d364
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
50 changed files with 589 additions and 289 deletions

View File

@ -29,6 +29,17 @@ jobs:
- system: x86_64-linux
runs_on: [self-hosted, Linux, X64]
# Nimble segfaults on MacOS hosts.
exclude:
- system: aarch64-darwin
nixpkg: libsds-android-arm64
- system: aarch64-darwin
nixpkg: libsds-android-amd64
- system: aarch64-darwin
nixpkg: libsds-android-x86
- system: aarch64-darwin
nixpkg: libsds-android-arm
name: '${{ matrix.system }} / ${{ matrix.nixpkg }}'
runs-on: ${{ matrix.runs_on }}
steps:
@ -39,7 +50,7 @@ jobs:
- name: 'Run Nix build for ${{ matrix.nixpkg }}'
shell: bash
run: |
nix build -L '.?submodules=1#${{ matrix.nixpkg }}' \
nix build -L '.#${{ matrix.nixpkg }}' \
--print-out-paths --accept-flake-config
- name: 'Show result contents'

6
.gitignore vendored
View File

@ -10,8 +10,8 @@ result
sds.nims
/.update.timestamp
# Nimbus Build System
nimbus-build-system.paths
# Nimble packages
/vendor/.nimble
nimble.develop
nimble.paths
nimbledeps

55
.gitmodules vendored
View File

@ -1,55 +0,0 @@
[submodule "vendor/nimbus-build-system"]
path = vendor/nimbus-build-system
url = https://github.com/status-im/nimbus-build-system.git
ignore = untracked
branch = master
[submodule "vendor/nim-chronos"]
path = vendor/nim-chronos
url = https://github.com/status-im/nim-chronos.git
ignore = untracked
branch = master
[submodule "vendor/nim-results"]
path = vendor/nim-results
url = https://github.com/arnetheduck/nim-results.git
ignore = untracked
branch = master
[submodule "vendor/nim-stew"]
path = vendor/nim-stew
url = https://github.com/status-im/nim-stew.git
ignore = untracked
branch = master
[submodule "vendor/nim-chronicles"]
path = vendor/nim-chronicles
url = https://github.com/status-im/nim-chronicles.git
ignore = untracked
branch = master
[submodule "vendor/nim-faststreams"]
path = vendor/nim-faststreams
url = https://github.com/status-im/nim-faststreams.git
ignore = untracked
branch = master
[submodule "vendor/nim-json-serialization"]
path = vendor/nim-json-serialization
url = https://github.com/status-im/nim-json-serialization.git
ignore = untracked
branch = master
[submodule "vendor/nim-serialization"]
path = vendor/nim-serialization
url = https://github.com/status-im/nim-serialization.git
ignore = untracked
branch = master
[submodule "vendor/nim-taskpools"]
path = vendor/nim-taskpools
url = https://github.com/status-im/nim-taskpools.git
ignore = untracked
branch = master
[submodule "vendor/nim-confutils"]
path = vendor/nim-confutils
url = https://github.com/status-im/nim-confutils.git
ignore = untracked
branch = master
[submodule "vendor/nim-libp2p"]
path = vendor/nim-libp2p
url = https://github.com/vacp2p/nim-libp2p.git
ignore = untracked
branch = master

View File

@ -1,49 +1,13 @@
.PHONY: libsds deps
export BUILD_SYSTEM_DIR := vendor/nimbus-build-system
LINK_PCRE := 0
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
nimbus-build-system-nimble-dir:
NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \
PWD_CMD="$(PWD)" \
$(CURDIR)/scripts/generate_nimble_links.sh
ifeq ($(NIM_PARAMS),)
# "variables.mk" was not included, so we update the submodules.
GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
.DEFAULT:
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
$(GIT_SUBMODULE_UPDATE); \
echo
# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
#
# After restarting, it will execute its original goal, so we don't have to start a child Make here
# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
else # "variables.mk" was included. Business as usual until the end of this file.
# default target, because it's the first one that doesn't start with '.'
all: | libsds
sds.nims:
ln -s sds.nimble $@
update: | update-common
rm -rf sds.nims && \
$(MAKE) sds.nims $(HANDLE_OUTPUT)
clean:
rm -rf build
deps: | deps-common sds.nims
# must be included after the default target
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
## Git version
GIT_VERSION ?= $(shell git describe --abbrev=6 --always --tags)
## Compilation parameters. If defined in the CLI the assignments won't be executed
@ -78,8 +42,8 @@ else ifeq ($(detected_OS),Linux)
BUILD_COMMAND := $(BUILD_COMMAND)Linux
endif
libsds: | deps
$(ENV_SCRIPT) nim $(BUILD_COMMAND) $(NIM_PARAMS) sds.nims
libsds:
nimble --verbose $(BUILD_COMMAND) $(NIM_PARAMS) sds.nimble
#####################
## Mobile Bindings ##
@ -119,14 +83,13 @@ build-libsds-for-android-arch:
ARCH_DIRNAME=$(ARCH_DIRNAME) \
ANDROID_ARCH=$(ANDROID_ARCH) \
ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) \
$(ENV_SCRIPT) \
nim libsdsAndroid $(NIM_PARAMS) sds.nims
nimble libsdsAndroid $(NIM_PARAMS) sds.nimble
libsds-android-arm64: ANDROID_ARCH=aarch64-linux-android
libsds-android-arm64: ARCH=arm64
libsds-android-arm64: ABIDIR=arm64-v8a
libsds-android-arm64: ARCH_DIRNAME=aarch64-linux-android
libsds-android-arm64: | libsds-android-precheck build deps
libsds-android-arm64: | libsds-android-precheck
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
@ -134,7 +97,7 @@ libsds-android-amd64: ANDROID_ARCH=x86_64-linux-android
libsds-android-amd64: ARCH=amd64
libsds-android-amd64: ABIDIR=x86_64
libsds-android-amd64: ARCH_DIRNAME=x86_64-linux-android
libsds-android-amd64: | libsds-android-precheck build deps
libsds-android-amd64: | libsds-android-precheck
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
@ -142,7 +105,7 @@ libsds-android-x86: ANDROID_ARCH=i686-linux-android
libsds-android-x86: ARCH=i386
libsds-android-x86: ABIDIR=x86
libsds-android-x86: ARCH_DIRNAME=i686-linux-android
libsds-android-x86: | libsds-android-precheck build deps
libsds-android-x86: | libsds-android-precheck
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
@ -150,7 +113,7 @@ libsds-android-arm: ANDROID_ARCH=armv7a-linux-androideabi
libsds-android-arm: ARCH=arm
libsds-android-arm: ABIDIR=armeabi-v7a
libsds-android-arm: ARCH_DIRNAME=arm-linux-androideabi
libsds-android-arm: | libsds-android-precheck build deps
libsds-android-arm: | libsds-android-precheck
# cross-rs target architecture name does not match the one used in android
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) \
@ -171,10 +134,8 @@ else
$(error Unsupported ARCH '$(ARCH)'. Please set ARCH to one of: arm64, arm, amd64, x86)
endif
endif
# Target iOS
libsds-ios: | deps
$(ENV_SCRIPT) nim libsdsIOS $(NIM_PARAMS) sds.nims
libsds-ios: |
nimble libsdsIOS $(NIM_PARAMS) sds.nimble

125
README.md
View File

@ -1,49 +1,128 @@
# nim-e2e-reliability
Nim implementation of the e2e reliability protocol
# nim-sds
Nim implementation of the e2e reliability protocol.
## Prerequisites
- [Nix](https://nixos.org/download/) package manager
## Quick start
```bash
git clone https://github.com/logos-messaging/nim-sds.git
cd nim-sds
# Build the shared library
nix build '.#libsds'
# Run tests
nix develop --command nimble test
```
## Building
### Nix
### Desktop
```bash
nix build --print-out-paths '.?submodules=1#libsds'
nix build --print-out-paths '.?submodules=1#libsds-android-arm64'
```
### Windows, Linux or MacOS
```code
make libsds
nix build --print-out-paths '.#libsds'
```
### Android
Download the latest Android NDK. For example, on Ubuntu with Intel:
```bash
nix build --print-out-paths '.#libsds-android-arm64'
nix build --print-out-paths '.#libsds-android-amd64'
nix build --print-out-paths '.#libsds-android-x86'
nix build --print-out-paths '.#libsds-android-arm'
```
```code
### iOS
```bash
nix build --print-out-paths '.#libsds-ios'
```
<details>
<summary>Development shell</summary>
Enter the dev shell:
```bash
nix develop
```
Build using nimble tasks:
```bash
# Dynamic library (auto-detects OS)
nimble libsdsDynamicMac # macOS
nimble libsdsDynamicLinux # Linux
nimble libsdsDynamicWindows # Windows
# Static library
nimble libsdsStaticMac # macOS
nimble libsdsStaticLinux # Linux
nimble libsdsStaticWindows # Windows
```
Run tests:
```bash
nimble test
```
The built library is output to `build/`.
</details>
<details>
<summary>Android (without Nix)</summary>
Download the latest Android NDK:
```bash
cd ~
wget https://dl.google.com/android/repository/android-ndk-r27c-linux.zip
```
```code
unzip android-ndk-r27c-linux.zip
```
Then, add the following to your ~/.bashrc file:
```code
Add to `~/.bashrc`:
```bash
export ANDROID_NDK_ROOT=$HOME/android-ndk-r27c
export PATH=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH
```
Then, use one of the following commands, according to the current architecture:
Then build:
```bash
ARCH=arm64 nimble libsdsAndroid
```
| Architecture | command |
| Architecture | Command |
| ------------ | ------- |
| arm64 | `make libsds-android ARCH=arm64` |
| amd64 | `make libsds-android ARCH=amd64` |
| x86 | `make libsds-android ARCH=x86` |
| arm64 | `ARCH=arm64 nimble libsdsAndroid` |
| amd64 | `ARCH=amd64 nimble libsdsAndroid` |
| x86 | `ARCH=x86 nimble libsdsAndroid` |
At the end of the process, the library will be created in build/libsds.so
The library is output to `build/libsds.so`.
</details>
<details>
<summary>Dependency management</summary>
Dependencies are managed by [Nimble](https://github.com/nim-lang/nimble) and pinned via `nimble.lock`.
To set up dependencies locally:
```bash
nimble setup -l
```
To update dependencies:
```bash
nimble lock
```
After updating `nimble.lock`, the Nix `outputHash` in `nix/deps.nix` must be recalculated
by running `nix build` and updating the hash from the error output.
</details>
## License
Licensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or [MIT license](LICENSE-MIT) at your option.

5
config.nims Normal file
View File

@ -0,0 +1,5 @@
# begin Nimble config (version 2)
--noNimblePath
when withDir(thisDir(), system.fileExists("nimble.paths")):
include "nimble.paths"
# end Nimble config

7
env.sh
View File

@ -1,7 +0,0 @@
#!/usr/bin/env bash
# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
# and we fall back to a Zsh-specific special var to also support Zsh.
REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
ABS_PATH="$(cd ${REL_PATH}; pwd)"
source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh

31
flake.lock generated
View File

@ -1,46 +1,23 @@
{
"nodes": {
"nimbusBuildSystem": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1769685810,
"narHash": "sha256-mANqcQcRXb08ZR8WF6SoyVrL/nXUePtMZfpNx25BlLI=",
"ref": "refs/heads/master",
"rev": "0e7a764edae92b224326b3700f062702489ce2b4",
"revCount": 238,
"submodules": true,
"type": "git",
"url": "file:./vendor/nimbus-build-system"
},
"original": {
"submodules": true,
"type": "git",
"url": "file:./vendor/nimbus-build-system"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1757590060,
"narHash": "sha256-EWwwdKLMZALkgHFyKW7rmyhxECO74+N+ZO5xTDnY/5c=",
"lastModified": 1770464364,
"narHash": "sha256-z5NJPSBwsLf/OfD8WTmh79tlSU8XgIbwmk6qB1/TFzY=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "0ef228213045d2cdb5a169a95d63ded38670b293",
"rev": "23d72dabcb3b12469f57b37170fcbc1789bd7457",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "0ef228213045d2cdb5a169a95d63ded38670b293",
"rev": "23d72dabcb3b12469f57b37170fcbc1789bd7457",
"type": "github"
}
},
"root": {
"inputs": {
"nimbusBuildSystem": "nimbusBuildSystem",
"nixpkgs": "nixpkgs"
}
}

View File

@ -8,16 +8,11 @@
inputs = {
# We are pinning the commit because ultimately we want to use same commit across different projects.
# A commit from nixpkgs 24.11 release : https://github.com/NixOS/nixpkgs/tree/release-24.11
nixpkgs.url = "github:NixOS/nixpkgs?rev=0ef228213045d2cdb5a169a95d63ded38670b293";
# WARNING: Remember to update commit and use 'nix flake update' to update flake.lock.
nimbusBuildSystem = {
url = "git+file:./vendor/nimbus-build-system?submodules=1";
inputs.nixpkgs.follows = "nixpkgs";
};
# A commit from nixpkgs 25.11 release : https://github.com/NixOS/nixpkgs/tree/release-25.11
nixpkgs.url = "github:NixOS/nixpkgs?rev=23d72dabcb3b12469f57b37170fcbc1789bd7457";
};
outputs = { self, nixpkgs, nimbusBuildSystem }:
outputs = { self, nixpkgs }:
let
stableSystems = [
"x86_64-linux" "aarch64-linux"
@ -47,35 +42,34 @@
in rec {
packages = forAllSystems (system: let
pkgs = pkgsFor.${system};
nim = nimbusBuildSystem.packages.${system}.nim;
buildTargets = pkgs.callPackage ./nix/default.nix {
inherit stableSystems nim;
src = self;
};
skipAndroidOnDarwin = t: !(pkgs.stdenv.isDarwin);
targets = [
# All potential targets
allTargets = [
"libsds"
"libsds-android-arm64"
"libsds-android-amd64"
"libsds-android-x86"
"libsds-android-arm"
"libsds-ios"
];
in rec {
# non-Android package
libsds = buildTargets.override { targets = [ "libsds" ]; };
default = libsds;
}
# Generate a package for each target dynamically
// builtins.listToAttrs (map (name: {
inherit name;
value = buildTargets.override { targets = [ name ]; };
}) targets));
# Create a package for each target
allPackages = builtins.listToAttrs (map (t: {
name = t;
value = buildTargets.override { targets = [ t ]; };
}) allTargets);
in
allPackages // {
default = allPackages.libsds;
}
);
devShells = forAllSystems (system: {
default = pkgsFor.${system}.callPackage ./nix/shell.nix {
inherit (nimbusBuildSystem.packages.${system}) nim;
};
});
};

3
go.mod
View File

@ -1,3 +0,0 @@
module sds-bindings
go 1.22.5

View File

@ -1,5 +1,5 @@
import std/json
import ./json_base_event, ../../src/[message]
import ./json_base_event, sds/[message]
type JsonMessageReadyEvent* = ref object of JsonEvent
messageId*: SdsMessageID

View File

@ -1,5 +1,5 @@
import std/json
import ./json_base_event, ../../src/[message]
import ./json_base_event, sds/[message]
type JsonMessageSentEvent* = ref object of JsonEvent
messageId*: SdsMessageID

View File

@ -1,5 +1,5 @@
import std/json
import ./json_base_event, ../../src/[message], std/base64
import ./json_base_event, sds/[message], std/base64
type JsonMissingDependenciesEvent* = ref object of JsonEvent
messageId*: SdsMessageID

View File

@ -13,7 +13,7 @@ import
./sds_thread/inter_thread_communication/sds_thread_request,
./sds_thread/inter_thread_communication/requests/
[sds_lifecycle_request, sds_message_request, sds_dependencies_request],
../src/[reliability_utils, message],
sds,
./events/[
json_message_ready_event, json_message_sent_event, json_missing_dependencies_event,
json_periodic_sync_event,

1
library/nim.cfg Normal file
View File

@ -0,0 +1 @@
path = "../"

View File

@ -1,8 +1,8 @@
import std/[json, strutils, net, sequtils]
import chronos, chronicles, results
import ../../../alloc
import ../../../../src/[reliability_utils, reliability]
import library/alloc
import sds
type SdsDependenciesMsgType* = enum
MARK_DEPENDENCIES_MET

View File

@ -1,8 +1,8 @@
import std/json
import chronos, chronicles, results
import ../../../alloc
import ../../../../src/[reliability_utils, reliability]
import library/alloc
import sds
type SdsLifecycleMsgType* = enum
CREATE_RELIABILITY_MANAGER

View File

@ -1,8 +1,8 @@
import std/[json, strutils, net, sequtils, base64]
import chronos, chronicles, results
import ../../../alloc
import ../../../../src/[reliability_utils, reliability, message]
import library/alloc
import sds
type SdsMessageMsgType* = enum
WRAP_MESSAGE

View File

@ -7,7 +7,7 @@ import chronos, chronos/threadsync
import
../../ffi_types,
./requests/[sds_lifecycle_request, sds_message_request, sds_dependencies_request],
../../../src/[reliability_utils]
sds/sds_utils
type RequestType* {.pure.} = enum
LIFECYCLE

View File

@ -7,7 +7,7 @@ import chronicles, chronos, chronos/threadsync, taskpools/channels_spsc_single,
import
../ffi_types,
./inter_thread_communication/sds_thread_request,
../../src/[reliability_utils]
sds/sds_utils
type SdsContext* = object
thread: Thread[(ptr SdsContext)]

1
nim.cfg Normal file
View File

@ -0,0 +1 @@
nimcache = "build/nimcache/$projectName"

319
nimble.lock Normal file
View File

@ -0,0 +1,319 @@
{
"version": 2,
"packages": {
"unittest2": {
"version": "0.2.5",
"vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189",
"url": "https://github.com/status-im/nim-unittest2",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9"
}
},
"bearssl": {
"version": "0.2.6",
"vcsRevision": "11e798b62b8e6beabe958e048e9e24c7e0f9ee63",
"url": "https://github.com/status-im/nim-bearssl",
"downloadMethod": "git",
"dependencies": [
"unittest2"
],
"checksums": {
"sha1": "7e068f119664cf47ad0cfb74ef4c56fb6b616523"
}
},
"bearssl_pkey_decoder": {
"version": "0.1.0",
"vcsRevision": "21dd3710df9345ed2ad8bf8f882761e07863b8e0",
"url": "https://github.com/vacp2p/bearssl_pkey_decoder",
"downloadMethod": "git",
"dependencies": [
"bearssl"
],
"checksums": {
"sha1": "21b42e2e6ddca6c875d3fc50f36a5115abf51714"
}
},
"results": {
"version": "0.5.1",
"vcsRevision": "df8113dda4c2d74d460a8fa98252b0b771bf1f27",
"url": "https://github.com/arnetheduck/nim-results",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "a9c011f74bc9ed5c91103917b9f382b12e82a9e7"
}
},
"stew": {
"version": "0.4.2",
"vcsRevision": "b66168735d6f3841c5239c3169d3fe5fe98b1257",
"url": "https://github.com/status-im/nim-stew",
"downloadMethod": "git",
"dependencies": [
"results",
"unittest2"
],
"checksums": {
"sha1": "928e82cb8d2f554e8f10feb2349ee9c32fee3a8c"
}
},
"faststreams": {
"version": "0.5.0",
"vcsRevision": "ce27581a3e881f782f482cb66dc5b07a02bd615e",
"url": "https://github.com/status-im/nim-faststreams",
"downloadMethod": "git",
"dependencies": [
"stew",
"unittest2"
],
"checksums": {
"sha1": "ee61e507b805ae1df7ec936f03f2d101b0d72383"
}
},
"serialization": {
"version": "0.5.2",
"vcsRevision": "b0f2fa32960ea532a184394b0f27be37bd80248b",
"url": "https://github.com/status-im/nim-serialization",
"downloadMethod": "git",
"dependencies": [
"faststreams",
"unittest2",
"stew"
],
"checksums": {
"sha1": "fa35c1bb76a0a02a2379fe86eaae0957c7527cb8"
}
},
"json_serialization": {
"version": "0.4.4",
"vcsRevision": "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44",
"url": "https://github.com/status-im/nim-json-serialization",
"downloadMethod": "git",
"dependencies": [
"faststreams",
"serialization",
"stew",
"results"
],
"checksums": {
"sha1": "8b3115354104858a0ac9019356fb29720529c2bd"
}
},
"testutils": {
"version": "0.8.0",
"vcsRevision": "e4d37dc1652d5c63afb89907efb5a5e812261797",
"url": "https://github.com/status-im/nim-testutils",
"downloadMethod": "git",
"dependencies": [
"unittest2"
],
"checksums": {
"sha1": "d1678f50aa47d113b4e77d41eec2190830b523fa"
}
},
"chronicles": {
"version": "0.12.2",
"vcsRevision": "27ec507429a4eb81edc20f28292ee8ec420be05b",
"url": "https://github.com/status-im/nim-chronicles",
"downloadMethod": "git",
"dependencies": [
"faststreams",
"serialization",
"json_serialization",
"testutils"
],
"checksums": {
"sha1": "02febb20d088120b2836d3306cfa21f434f88f65"
}
},
"httputils": {
"version": "0.4.0",
"vcsRevision": "c53852d9e24205b6363bba517fa8ee7bde823691",
"url": "https://github.com/status-im/nim-http-utils",
"downloadMethod": "git",
"dependencies": [
"stew",
"results",
"unittest2"
],
"checksums": {
"sha1": "298bc5b6fe4e5aa9c3b7e2ebfa17191675020f10"
}
},
"chronos": {
"version": "4.0.4",
"vcsRevision": "0646c444fce7c7ed08ef6f2c9a7abfd172ffe655",
"url": "https://github.com/status-im/nim-chronos",
"downloadMethod": "git",
"dependencies": [
"results",
"stew",
"bearssl",
"httputils",
"unittest2"
],
"checksums": {
"sha1": "455802a90204d8ad6b31d53f2efff8ebfe4c834a"
}
},
"dnsclient": {
"version": "0.3.4",
"vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8",
"url": "https://github.com/ba0f3/dnsclient.nim",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40"
}
},
"jwt": {
"version": "0.2",
"vcsRevision": "18f8378de52b241f321c1f9ea905456e89b95c6f",
"url": "https://github.com/vacp2p/nim-jwt.git",
"downloadMethod": "git",
"dependencies": [
"bearssl",
"bearssl_pkey_decoder"
],
"checksums": {
"sha1": "bcfd6fc9c5e10a52b87117219b7ab5c98136bc8e"
}
},
"nimcrypto": {
"version": "0.7.3",
"vcsRevision": "b3dbc9c4d08e58c5b7bfad6dc7ef2ee52f2f4c08",
"url": "https://github.com/cheatfate/nimcrypto",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "f72b90fe3f4da09efa482de4f8729e7ee4abea2f"
}
},
"metrics": {
"version": "0.1.2",
"vcsRevision": "11d0cddfb0e711aa2a8c75d1892ae24a64c299fc",
"url": "https://github.com/status-im/nim-metrics",
"downloadMethod": "git",
"dependencies": [
"chronos",
"results",
"stew"
],
"checksums": {
"sha1": "5cdac99d85d3c146d170e85064c88fb28f377842"
}
},
"secp256k1": {
"version": "0.6.0.3.2",
"vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15",
"url": "https://github.com/status-im/nim-secp256k1",
"downloadMethod": "git",
"dependencies": [
"stew",
"results",
"nimcrypto"
],
"checksums": {
"sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10"
}
},
"zlib": {
"version": "0.1.0",
"vcsRevision": "e680f269fb01af2c34a2ba879ff281795a5258fe",
"url": "https://github.com/status-im/nim-zlib",
"downloadMethod": "git",
"dependencies": [
"stew",
"results"
],
"checksums": {
"sha1": "bbde4f5a97a84b450fef7d107461e5f35cf2b47f"
}
},
"websock": {
"version": "0.2.1",
"vcsRevision": "35ae76f1559e835c80f9c1a3943bf995d3dd9eb5",
"url": "https://github.com/status-im/nim-websock",
"downloadMethod": "git",
"dependencies": [
"chronos",
"httputils",
"chronicles",
"stew",
"nimcrypto",
"bearssl",
"results",
"zlib"
],
"checksums": {
"sha1": "1cb5efa10cd389bc01d0707c242ae010c76a03cd"
}
},
"lsquic": {
"version": "0.0.1",
"vcsRevision": "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f",
"url": "https://github.com/vacp2p/nim-lsquic",
"downloadMethod": "git",
"dependencies": [
"zlib",
"stew",
"chronos",
"nimcrypto",
"unittest2",
"chronicles"
],
"checksums": {
"sha1": "f465fa994346490d0924d162f53d9b5aec62f948"
}
},
"libp2p": {
"version": "1.15.2",
"vcsRevision": "ca48c3718246bb411ff0e354a70cb82d9a28de0d",
"url": "https://github.com/vacp2p/nim-libp2p",
"downloadMethod": "git",
"dependencies": [
"nimcrypto",
"dnsclient",
"bearssl",
"chronicles",
"chronos",
"metrics",
"secp256k1",
"stew",
"websock",
"unittest2",
"results",
"lsquic",
"jwt"
],
"checksums": {
"sha1": "3b2cdc7e00261eb4210ca3d44ec3bd64c2b7bbba"
}
},
"stint": {
"version": "0.8.2",
"vcsRevision": "470b7892561b5179ab20bd389a69217d6213fe58",
"url": "https://github.com/status-im/nim-stint",
"downloadMethod": "git",
"dependencies": [
"stew",
"unittest2"
],
"checksums": {
"sha1": "d8f871fd617e7857192d4609fe003b48942a8ae5"
}
},
"taskpools": {
"version": "0.1.0",
"vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb",
"url": "https://github.com/status-im/nim-taskpools",
"downloadMethod": "git",
"dependencies": [],
"checksums": {
"sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81"
}
}
},
"tasks": {}
}

View File

@ -9,27 +9,15 @@ nix develop
## Building
To build a Codex you can use:
To simply build you can use:
```sh
nix build '.?submodules=1#default'
nix build '.#libsds'
```
The `?submodules=1` part should eventually not be necessary.
For more details see:
https://github.com/NixOS/nix/issues/4423
It can be also done without even cloning the repo:
```sh
nix build 'git+https://github.com/waku-org/nim-sds?submodules=1#'
```
## Running
```sh
nix run 'git+https://github.com/waku-org/nim-sds?submodules=1#''
```
## Testing
```sh
nix flake check ".?submodules=1#"
nix build github:waku-org/nim-sds
nix build github:waku-org/nim-sds#libsds-ios
nix build github:waku-org/nim-sds#libsds-android-arm64"
```
Or as a flake input.

View File

@ -1,8 +1,6 @@
{
pkgs,
src ? ../.,
# Nimbus-build-system package.
nim ? null,
# Options: 0,1,2
verbosity ? 2,
# Make targets
@ -11,9 +9,6 @@
stableSystems ? ["x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "x86_64-windows"],
}:
assert pkgs.lib.assertMsg ((src.submodules or true) == true)
"Unable to build without submodules. Append '?submodules=1#' to the URI.";
let
inherit (pkgs) stdenv lib writeScriptBin callPackage;
inherit (lib) any match substring optionals optionalString;
@ -27,6 +22,10 @@ let
revision = substring 0 8 (src.rev or src.dirtyRev or "00000000");
version = tools.findKeyValue "^version = \"([a-f0-9.-]+)\"$" ../sds.nimble;
nimbleDeps = callPackage ./deps.nix {
inherit src version revision;
};
in stdenv.mkDerivation {
pname = "nim-sds";
inherit src;
@ -39,27 +38,28 @@ in stdenv.mkDerivation {
};
buildInputs = with pkgs; [
openssl gmp zip
openssl gmp zip nim git nimble
];
# Dependencies that should only exist in the build environment.
nativeBuildInputs = with pkgs; [
nim cmake which patchelf
nim cmake which patchelf nimbleDeps
] ++ optionals stdenv.isLinux [
pkgs.lsb-release
];
makeFlags = targets ++ [
"V=${toString verbosity}"
# Built from nimbus-build-system via flake.
"USE_SYSTEM_NIM=1"
];
# Provide dependencies via Nimble deps derivation.
configurePhase = ''
# Avoid /tmp write errors.
export XDG_CACHE_HOME=$TMPDIR/cache
patchShebangs . vendor/nimbus-build-system/scripts
make nimbus-build-system-nimble-dir
export NIMBLE_DIR=$NIX_BUILD_TOP/nimbledeps
cp -r ${nimbleDeps}/nimbledeps $NIMBLE_DIR
cp ${nimbleDeps}/nimble.paths ./
chmod 775 -R $NIMBLE_DIR
# Fix relative paths to absolute paths
sed -i "s|./nimbledeps|$NIMBLE_DIR|g" nimble.paths
'';
installPhase = let
@ -74,7 +74,7 @@ in stdenv.mkDerivation {
zip -r libwaku.aar *
'' else ''
mkdir -p $out/lib -p $out/include
cp build/* $out/lib/
cp build/lib* $out/lib/
cp library/libsds.h $out/include/
'';

55
nix/deps.nix Normal file
View File

@ -0,0 +1,55 @@
{ pkgs, stdenv, src, version, revision }:
stdenv.mkDerivation {
pname = "nim-sds-nimble-deps";
version = "${version}-${revision}";
inherit src;
nativeBuildInputs = with pkgs; [
jq rsync git nimble cacert moreutils
];
configurePhase = ''
export XDG_CACHE_HOME=$TMPDIR
export NIMBLE_DIR=$NIX_BUILD_TOP/nimbledir
export HOME=$TMPDIR
'';
buildPhase = ''
nimble --version
nimble --silent --localdeps setup
nimble --silent --localdeps install -y --depsOnly
'';
installPhase = ''
mkdir -p $out/nimbledeps
cp nimble.paths $out/nimble.paths
rsync -ra \
--prune-empty-dirs \
--include='*/' \
--include='*.json' \
--include='*.nim' \
--include='*.nimble' \
--exclude='*' \
$NIMBLE_DIR/pkgs2 $out/nimbledeps
'';
fixupPhase = ''
# Replace build path with deterministic $out.
sed "s|$NIMBLE_DIR|./nimbledeps|g" $out/nimble.paths \
| sort | sponge $out/nimble.paths
# Nimble does not maintain order of files list.
for META_FILE in $(find $out -name nimblemeta.json); do
jq '.metaData.files |= sort' $META_FILE | sponge $META_FILE
done
'';
# Make this a fixed-output derivation to allows internet access for Nimble.
outputHash = "sha256-OnirsXLj4HMVTbk+b4fcC+1K9MSMJyae6I7JO16WDno=";
outputHashAlgo = "sha256";
outputHashMode = "recursive";
}

View File

@ -10,8 +10,8 @@
androidenv.composeAndroidPackages {
cmdLineToolsVersion = "9.0";
toolsVersion = "26.1.1";
platformToolsVersion = "34.0.5";
buildToolsVersions = [ "34.0.0" ];
platformToolsVersion = "35.0.2";
buildToolsVersions = [ "35.0.0" ];
platformVersions = [ "34" ];
cmakeVersions = [ "3.22.1" ];
ndkVersion = "27.2.12479018";

View File

@ -1,16 +0,0 @@
# Package
version = "0.1.0"
author = "Waku Team"
description = "E2E Reliability Protocol API"
license = "MIT"
srcDir = "src"
# Dependencies
requires "nim >= 2.0.8"
requires "chronicles"
requires "libp2p"
# Tasks
task test, "Run the test suite":
exec "nim c -r tests/test_bloom.nim"
exec "nim c -r tests/test_reliability.nim"

View File

@ -1,6 +1,8 @@
import std/[times, locks, tables, sets, options]
import chronos, results, chronicles
import ./[message, protobuf, reliability_utils, rolling_bloom_filter]
import sds/[message, protobuf, sds_utils, rolling_bloom_filter]
export message, protobuf, sds_utils, rolling_bloom_filter
proc newReliabilityManager*(
config: ReliabilityConfig = defaultConfig()

View File

@ -1,43 +1,45 @@
mode = ScriptMode.Verbose
import strutils, os
# Package
version = "0.1.0"
author = "Waku Team"
description = "E2E Reliability Protocol API"
version = "0.2.4"
author = "Logos Messaging Team"
description = "E2E Scalable Data Sync API"
license = "MIT"
srcDir = "src"
srcDir = "sds"
# Dependencies
requires "nim >= 2.2.4",
"chronicles", "chronos", "stew", "stint", "metrics", "libp2p", "results"
requires "nim >= 2.2.6"
requires "chronos >= 4.0.4"
requires "libp2p >= 1.15.1"
requires "chronicles"
requires "stew"
requires "stint"
requires "metrics"
requires "results"
requires "taskpools >= 0.1.0" ## This should be removed when using nim-ffi dependency
proc buildLibrary(
outLibNameAndExt: string,
name: string,
srcDir = "./",
params = "",
extra_params = "",
`type` = "static",
) =
if not dirExists "build":
mkDir "build"
# allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
var extra_params = params
for i in 2 ..< paramCount():
extra_params &= " " & paramStr(i)
if `type` == "static":
exec "nim c" & " --out:build/" & outLibNameAndExt &
" --threads:on --app:staticlib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds --skipParentCfg:on " &
" --threads:on --app:staticlib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds " &
extra_params & " " & srcDir & name & ".nim"
else:
when defined(windows):
exec "nim c" & " --out:build/" & outLibNameAndExt &
" --threads:on --app:lib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds --skipParentCfg:off " &
" --threads:on --app:lib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds " &
extra_params & " " & srcDir & name & ".nim"
else:
exec "nim c" & " --out:build/" & outLibNameAndExt &
" --threads:on --app:lib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds --skipParentCfg:on " &
" --threads:on --app:lib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds " &
extra_params & " " & srcDir & name & ".nim"
proc getArch(): string =
@ -169,17 +171,13 @@ task libsdsIOS, "Build the mobile bindings for iOS":
buildMobileIOS srcDir, sdkPath
### Mobile Android
proc buildMobileAndroid(srcDir = ".", params = "") =
proc buildMobileAndroid(srcDir = ".", extra_params = "") =
let cpu = getArch()
let outDir = "build/"
if not dirExists outDir:
mkDir outDir
var extra_params = params
for i in 2 ..< paramCount():
extra_params &= " " & paramStr(i)
exec "nim c" & " --out:" & outDir &
"/libsds.so --threads:on --app:lib --opt:size --noMain --mm:refc --nimMainPrefix:libsds " &
"-d:chronicles_sinks=textlines[dynamic] --header --passL:-L" & outdir &

View File

@ -22,7 +22,7 @@ proc hashN(item: string, n: int, maxValue: int): int =
hashB = abs(hash(item & " b")) mod maxValue # string concatenation
abs((hashA + n * hashB)) mod maxValue
# # Use bit rotation for second hash instead of string concatenation if speed if preferred over FP-rate
# # Rotate left by 21 bits (lower the rotation, higher the speed but higher the FP-rate too)
# # Rotate left by 21 bits (lower the rotation, higher the speed but higher the FP-rate too)
# hashB = abs(
# ((h shl 21) or (h shr (sizeof(int) * 8 - 21)))
# ) mod maxValue

View File

@ -1,6 +1,6 @@
import libp2p/protobuf/minprotobuf
import endians
import ../src/[message, protobufutil, bloom, reliability_utils]
import sds/[message, protobufutil, bloom, sds_utils]
proc encode*(msg: SdsMessage): ProtoBuffer =
var pb = initProtoBuffer()

1
tests/nim.cfg Normal file
View File

@ -0,0 +1 @@
path = "../"

View File

@ -1,5 +1,5 @@
import unittest, results, strutils
import ../src/bloom
import sds/bloom
from random import rand, randomize
suite "bloom filter":

View File

@ -1,5 +1,5 @@
import unittest, results, chronos, std/[times, options, tables]
import ../src/[reliability, message, protobuf, reliability_utils, rolling_bloom_filter]
import sds
const testChannel = "testChannel"

@ -1 +0,0 @@
Subproject commit a8fb38a10bcb548df78e9a70bd77b26bb50abd12

1
vendor/nim-chronos vendored

@ -1 +0,0 @@
Subproject commit b55e2816eb45f698ddaca8d8473e401502562db2

@ -1 +0,0 @@
Subproject commit e214b3992a31acece6a9aada7d0a1ad37c928f3b

@ -1 +0,0 @@
Subproject commit 2b08c774afaafd600cf4c6f994cf78b8aa090c0c

@ -1 +0,0 @@
Subproject commit 2b1c5eb11df3647a2cee107cd4cce3593cbb8bcf

1
vendor/nim-libp2p vendored

@ -1 +0,0 @@
Subproject commit ac25da6cea158768bbc060b7be2fbe004206f3bb

1
vendor/nim-results vendored

@ -1 +0,0 @@
Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27

@ -1 +0,0 @@
Subproject commit 548d0adc9797a10b2db7f788b804330306293088

1
vendor/nim-stew vendored

@ -1 +0,0 @@
Subproject commit d7a6868ba84165e7fdde427af9a1fc3f5f5cc151

@ -1 +0,0 @@
Subproject commit 7b74a716a40249720fd7da428113147942b9642d

@ -1 +0,0 @@
Subproject commit 0e7a764edae92b224326b3700f062702489ce2b4