deep refactor to support Nimble

.gitignore

@@ -15,3 +15,6 @@ nimbus-build-system.paths
 
 # Nimble packages
 /vendor/.nimble
+nimble.develop
+nimble.paths
+nimbledeps

.gitmodules

@@ -1,55 +0,0 @@
-[submodule "vendor/nimbus-build-system"]
-	path = vendor/nimbus-build-system
-	url = https://github.com/status-im/nimbus-build-system.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-chronos"]
-	path = vendor/nim-chronos
-	url = https://github.com/status-im/nim-chronos.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-results"]
-	path = vendor/nim-results
-	url = https://github.com/arnetheduck/nim-results.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-stew"]
-	path = vendor/nim-stew
-	url = https://github.com/status-im/nim-stew.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-chronicles"]
-	path = vendor/nim-chronicles
-	url = https://github.com/status-im/nim-chronicles.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-faststreams"]
-	path = vendor/nim-faststreams
-	url = https://github.com/status-im/nim-faststreams.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-json-serialization"]
-	path = vendor/nim-json-serialization
-	url = https://github.com/status-im/nim-json-serialization.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-serialization"]
-	path = vendor/nim-serialization
-	url = https://github.com/status-im/nim-serialization.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-taskpools"]
-	path = vendor/nim-taskpools
-	url = https://github.com/status-im/nim-taskpools.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-confutils"]
-	path = vendor/nim-confutils
-	url = https://github.com/status-im/nim-confutils.git
-	ignore = untracked
-	branch = master
-[submodule "vendor/nim-libp2p"]
-	path = vendor/nim-libp2p
-	url = https://github.com/vacp2p/nim-libp2p.git
-	ignore = untracked
-	branch = master

Makefile

@@ -1,49 +1,13 @@
 .PHONY: libsds deps
 
-export BUILD_SYSTEM_DIR := vendor/nimbus-build-system
 LINK_PCRE := 0
 
-# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
--include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
-
-nimbus-build-system-nimble-dir:
-	NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \
-	PWD_CMD="$(PWD)" \
-	$(CURDIR)/scripts/generate_nimble_links.sh
-
-ifeq ($(NIM_PARAMS),)
-# "variables.mk" was not included, so we update the submodules.
-GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
-.DEFAULT:
-	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
-		$(GIT_SUBMODULE_UPDATE); \
-		echo
-# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
-# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
-#
-# After restarting, it will execute its original goal, so we don't have to start a child Make here
-# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
-
-else # "variables.mk" was included. Business as usual until the end of this file.
-
 # default target, because it's the first one that doesn't start with '.'
 all: | libsds
 
-sds.nims:
-	ln -s sds.nimble $@
-
-update: | update-common
-	rm -rf sds.nims && \
-		$(MAKE) sds.nims $(HANDLE_OUTPUT)
-
 clean:
 	rm -rf build
 
-deps: | deps-common sds.nims
-
-# must be included after the default target
--include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
-
 ## Git version
 GIT_VERSION ?= $(shell git describe --abbrev=6 --always --tags)
 ## Compilation parameters. If defined in the CLI the assignments won't be executed
@@ -78,8 +42,8 @@ else ifeq ($(detected_OS),Linux)
 	BUILD_COMMAND := $(BUILD_COMMAND)Linux
 endif
 
-libsds: | deps
-	$(ENV_SCRIPT) nim $(BUILD_COMMAND) $(NIM_PARAMS) sds.nims
+libsds:
+	nim $(BUILD_COMMAND) $(NIM_PARAMS) sds.nims
 
 #####################
 ## Mobile Bindings ##
@@ -171,8 +135,6 @@ else
 	$(error Unsupported ARCH '$(ARCH)'. Please set ARCH to one of: arm64, arm, amd64, x86)
 endif
 
-endif
-
 # Target iOS
 
 libsds-ios: | deps

config.nims

@@ -0,0 +1,4 @@
+# begin Nimble config (version 2)
+when withDir(thisDir(), system.fileExists("nimble.paths")):
+  include "nimble.paths"
+# end Nimble config

env.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
-# and we fall back to a Zsh-specific special var to also support Zsh.
-REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
-ABS_PATH="$(cd ${REL_PATH}; pwd)"
-source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh

go.mod

@@ -1,3 +0,0 @@
-module sds-bindings
-
-go 1.22.5

library/events/json_message_ready_event.nim

@@ -1,5 +1,5 @@
 import std/json
-import ./json_base_event, ../../src/[message]
+import ./json_base_event, sds/[message]
 
 type JsonMessageReadyEvent* = ref object of JsonEvent
   messageId*: SdsMessageID

library/events/json_message_sent_event.nim

@@ -1,5 +1,5 @@
 import std/json
-import ./json_base_event, ../../src/[message]
+import ./json_base_event, sds/[message]
 
 type JsonMessageSentEvent* = ref object of JsonEvent
   messageId*: SdsMessageID

library/events/json_missing_dependencies_event.nim

@@ -1,5 +1,5 @@
 import std/json
-import ./json_base_event, ../../src/[message], std/base64
+import ./json_base_event, sds/[message], std/base64
 
 type JsonMissingDependenciesEvent* = ref object of JsonEvent
   messageId*: SdsMessageID

library/libsds.nim

@@ -13,7 +13,7 @@ import
   ./sds_thread/inter_thread_communication/sds_thread_request,
   ./sds_thread/inter_thread_communication/requests/
     [sds_lifecycle_request, sds_message_request, sds_dependencies_request],
-  ../src/[reliability_utils, message],
+  sds,
   ./events/[
     json_message_ready_event, json_message_sent_event, json_missing_dependencies_event,
     json_periodic_sync_event,

library/nim.cfg

@@ -0,0 +1 @@
+path = "../"

library/sds_thread/inter_thread_communication/requests/sds_dependencies_request.nim

@@ -1,8 +1,8 @@
 import std/[json, strutils, net, sequtils]
 import chronos, chronicles, results
 
-import ../../../alloc
-import ../../../../src/[reliability_utils, reliability]
+import library/alloc
+import sds
 
 type SdsDependenciesMsgType* = enum
   MARK_DEPENDENCIES_MET

library/sds_thread/inter_thread_communication/requests/sds_lifecycle_request.nim

@@ -1,8 +1,8 @@
 import std/json
 import chronos, chronicles, results
 
-import ../../../alloc
-import ../../../../src/[reliability_utils, reliability]
+import library/alloc
+import sds
 
 type SdsLifecycleMsgType* = enum
   CREATE_RELIABILITY_MANAGER

library/sds_thread/inter_thread_communication/requests/sds_message_request.nim

@@ -1,8 +1,8 @@
 import std/[json, strutils, net, sequtils, base64]
 import chronos, chronicles, results
 
-import ../../../alloc
-import ../../../../src/[reliability_utils, reliability, message]
+import library/alloc
+import sds
 
 type SdsMessageMsgType* = enum
   WRAP_MESSAGE

library/sds_thread/inter_thread_communication/sds_thread_request.nim

@@ -7,7 +7,7 @@ import chronos, chronos/threadsync
 import
   ../../ffi_types,
   ./requests/[sds_lifecycle_request, sds_message_request, sds_dependencies_request],
-  ../../../src/[reliability_utils]
+  sds/sds_utils
 
 type RequestType* {.pure.} = enum
   LIFECYCLE

library/sds_thread/sds_thread.nim

@@ -7,7 +7,7 @@ import chronicles, chronos, chronos/threadsync, taskpools/channels_spsc_single,
 import
   ../ffi_types,
   ./inter_thread_communication/sds_thread_request,
-  ../../src/[reliability_utils]
+  sds/sds_utils
 
 type SdsContext* = object
   thread: Thread[(ptr SdsContext)]

nim.cfg

@@ -0,0 +1 @@
+nimcache = "build/nimcache/$projectName"

reliability.nimble

@@ -1,16 +0,0 @@
-# Package
-version = "0.1.0"
-author = "Waku Team"
-description = "E2E Reliability Protocol API"
-license = "MIT"
-srcDir = "src"
-
-# Dependencies
-requires "nim >= 2.0.8"
-requires "chronicles"
-requires "libp2p"
-
-# Tasks
-task test, "Run the test suite":
-  exec "nim c -r tests/test_bloom.nim"
-  exec "nim c -r tests/test_reliability.nim"

sds.nim

@@ -1,6 +1,8 @@
 import std/[times, locks, tables, sets, options]
 import chronos, results, chronicles
-import ./[message, protobuf, reliability_utils, rolling_bloom_filter]
+import sds/[message, protobuf, sds_utils, rolling_bloom_filter]
+
+export message, protobuf, sds_utils, rolling_bloom_filter
 
 proc newReliabilityManager*(
     config: ReliabilityConfig = defaultConfig()

sds.nimble

@@ -1,31 +1,33 @@
-mode = ScriptMode.Verbose
-
 import strutils, os
 
 # Package
-version = "0.1.0"
-author = "Waku Team"
-description = "E2E Reliability Protocol API"
+version = "0.2.4"
+author = "Logos Messaging Team"
+description = "E2E Scalable Data Sync API"
 license = "MIT"
-srcDir = "src"
+srcDir = "sds"
 
 # Dependencies
-requires "nim >= 2.2.4",
-  "chronicles", "chronos", "stew", "stint", "metrics", "libp2p", "results"
+requires "nim >= 2.2.6"
+requires "chronos >= 4.0.4"
+requires "libp2p >= 1.15.1"
+requires "chronicles"
+requires "stew"
+requires "stint"
+requires "metrics"
+requires "results"
+requires "taskpools >= 0.1.0" ## This should be removed when using nim-ffi dependency
 
 proc buildLibrary(
     outLibNameAndExt: string,
     name: string,
     srcDir = "./",
-    params = "",
+    extra_params = "",
     `type` = "static",
 ) =
   if not dirExists "build":
     mkDir "build"
-  # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
-  var extra_params = params
-  for i in 2 ..< paramCount():
-    extra_params &= " " & paramStr(i)
+
   if `type` == "static":
     exec "nim c" & " --out:build/" & outLibNameAndExt &
       " --threads:on --app:staticlib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds --skipParentCfg:on " &
@@ -169,17 +171,13 @@ task libsdsIOS, "Build the mobile bindings for iOS":
   buildMobileIOS srcDir, sdkPath
 
 ### Mobile Android
-proc buildMobileAndroid(srcDir = ".", params = "") =
+proc buildMobileAndroid(srcDir = ".", extra_params = "") =
   let cpu = getArch()
 
   let outDir = "build/"
   if not dirExists outDir:
     mkDir outDir
 
-  var extra_params = params
-  for i in 2 ..< paramCount():
-    extra_params &= " " & paramStr(i)
-
   exec "nim c" & " --out:" & outDir &
     "/libsds.so --threads:on --app:lib --opt:size --noMain --mm:refc --nimMainPrefix:libsds " &
     "-d:chronicles_sinks=textlines[dynamic] --header --passL:-L" & outdir &

sds/bloom.nim

@@ -22,7 +22,7 @@ proc hashN(item: string, n: int, maxValue: int): int =
     hashB = abs(hash(item & " b")) mod maxValue # string concatenation
   abs((hashA + n * hashB)) mod maxValue
   #   # Use bit rotation for second hash instead of string concatenation if speed if preferred over FP-rate
-  #   # Rotate left by 21 bits (lower the rotation, higher the speed but higher the FP-rate too) 
+  #   # Rotate left by 21 bits (lower the rotation, higher the speed but higher the FP-rate too)
   #   hashB = abs(
   #     ((h shl 21) or (h shr (sizeof(int) * 8 - 21)))
   #   ) mod maxValue

sds/protobuf.nim

@@ -1,6 +1,6 @@
 import libp2p/protobuf/minprotobuf
 import endians
-import ../src/[message, protobufutil, bloom, reliability_utils]
+import sds/[message, protobufutil, bloom, sds_utils]
 
 proc encode*(msg: SdsMessage): ProtoBuffer =
   var pb = initProtoBuffer()

tests/nim.cfg

@@ -0,0 +1 @@
+path = "../"

tests/test_bloom.nim

@@ -1,5 +1,5 @@
 import unittest, results, strutils
-import ../src/bloom
+import sds/bloom
 from random import rand, randomize
 
 suite "bloom filter":

tests/test_reliability.nim

@@ -1,5 +1,5 @@
 import unittest, results, chronos, std/[times, options, tables]
-import ../src/[reliability, message, protobuf, reliability_utils, rolling_bloom_filter]
+import sds
 
 const testChannel = "testChannel"
 

vendor/nim-chronicles

@@ -1 +0,0 @@
-Subproject commit a8fb38a10bcb548df78e9a70bd77b26bb50abd12

vendor/nim-chronos

@@ -1 +0,0 @@
-Subproject commit b55e2816eb45f698ddaca8d8473e401502562db2

vendor/nim-confutils

@@ -1 +0,0 @@
-Subproject commit e214b3992a31acece6a9aada7d0a1ad37c928f3b

vendor/nim-faststreams

@@ -1 +0,0 @@
-Subproject commit 2b08c774afaafd600cf4c6f994cf78b8aa090c0c

vendor/nim-json-serialization

@@ -1 +0,0 @@
-Subproject commit 2b1c5eb11df3647a2cee107cd4cce3593cbb8bcf

vendor/nim-libp2p

@@ -1 +0,0 @@
-Subproject commit ac25da6cea158768bbc060b7be2fbe004206f3bb

vendor/nim-results

@@ -1 +0,0 @@
-Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27

vendor/nim-serialization

@@ -1 +0,0 @@
-Subproject commit 548d0adc9797a10b2db7f788b804330306293088

vendor/nim-stew

@@ -1 +0,0 @@
-Subproject commit d7a6868ba84165e7fdde427af9a1fc3f5f5cc151

vendor/nim-taskpools

@@ -1 +0,0 @@
-Subproject commit 7b74a716a40249720fd7da428113147942b9642d

vendor/nimbus-build-system

@@ -1 +0,0 @@
-Subproject commit 0be0663e1af76e869837226a4ef3e586fcc737d3