Compare commits

..

8 Commits

Author SHA1 Message Date
Siddarth Kumar
fb8039c5a5
chore: fix iOS build
otherwise iOS linker fails with
  Undefined symbols for architecture arm64
2025-12-23 19:47:02 +04:00
Igor Sirotin
8d33a7f7da
feat: thread pool (#40)
* feat: thread pool

* proper pass ARCH in Makefile when building for Android

---------

Co-authored-by: Ivan Folgueira Bande <ivansete@status.im>
2025-12-22 18:10:45 +00:00
Ivan Folgueira Bande
e67639ee08
get arch from uname -m if ARCH env var is not set 2025-12-17 16:02:53 +01:00
Ivan FB
ac31e5adf2
Merge pull request #37 from logos-messaging/fix/buildForAppleSilicon
fix: Add condition to check hostCpu and then build based on that
2025-12-15 12:06:17 +01:00
Khushboo Mehta
13c3c348fa fix: Add condition to check hostCpu and then build based on that 2025-12-15 12:01:51 +01:00
Ivan FB
0b4d3cc03f
Merge pull request #38 from logos-messaging/rm-log
rm UNWRAP_MESSAGE failed error
2025-12-10 09:51:41 +01:00
Ivan Folgueira Bande
191928adc6
rm UNWRAP_MESSAGE failed error 2025-12-09 11:45:35 +01:00
Ivan Folgueira Bande
ae445d5585
rename ANDROID_NDK_HOME to ANDROID_NDK_ROOT 2025-11-28 19:34:51 +01:00
7 changed files with 106 additions and 44 deletions

View File

@ -94,16 +94,16 @@ libsds: | deps
ANDROID_TARGET ?= 30 ANDROID_TARGET ?= 30
ifeq ($(detected_OS),Darwin) ifeq ($(detected_OS),Darwin)
ANDROID_TOOLCHAIN_DIR := $(ANDROID_NDK_HOME)/toolchains/llvm/prebuilt/darwin-x86_64 ANDROID_TOOLCHAIN_DIR := $(ANDROID_NDK_ROOT)/toolchains/llvm/prebuilt/darwin-x86_64
else else
ANDROID_TOOLCHAIN_DIR := $(ANDROID_NDK_HOME)/toolchains/llvm/prebuilt/linux-x86_64 ANDROID_TOOLCHAIN_DIR := $(ANDROID_NDK_ROOT)/toolchains/llvm/prebuilt/linux-x86_64
endif endif
# Fixes "clang: not found" errors # Fixes "clang: not found" errors
PATH := $(ANDROID_TOOLCHAIN_DIR)/bin:$(PATH) PATH := $(ANDROID_TOOLCHAIN_DIR)/bin:$(PATH)
libsds-android-precheck: libsds-android-precheck:
ifndef ANDROID_NDK_HOME ifndef ANDROID_NDK_ROOT
$(error ANDROID_NDK_HOME is not set) $(error ANDROID_NDK_ROOT is not set)
endif endif
build-libsds-for-android-arch: NIM_PARAMS := $(NIM_PARAMS) --passC="--sysroot=$(ANDROID_TOOLCHAIN_DIR)/sysroot" build-libsds-for-android-arch: NIM_PARAMS := $(NIM_PARAMS) --passC="--sysroot=$(ANDROID_TOOLCHAIN_DIR)/sysroot"
@ -115,7 +115,7 @@ build-libsds-for-android-arch: NIM_PARAMS := $(NIM_PARAMS) --passC="-I$(ANDROID_
build-libsds-for-android-arch: NIM_PARAMS := $(NIM_PARAMS) --passL="-L$(ANDROID_TOOLCHAIN_DIR)/sysroot/usr/lib/$(ARCH_DIRNAME)/$(ANDROID_TARGET)" build-libsds-for-android-arch: NIM_PARAMS := $(NIM_PARAMS) --passL="-L$(ANDROID_TOOLCHAIN_DIR)/sysroot/usr/lib/$(ARCH_DIRNAME)/$(ANDROID_TARGET)"
build-libsds-for-android-arch: build-libsds-for-android-arch:
CC=$(ANDROID_TOOLCHAIN_DIR)/bin/$(ANDROID_ARCH)$(ANDROID_TARGET)-clang \ CC=$(ANDROID_TOOLCHAIN_DIR)/bin/$(ANDROID_ARCH)$(ANDROID_TARGET)-clang \
CPU=$(CPU) ABIDIR=$(ABIDIR) \ ARCH=$(ARCH) ABIDIR=$(ABIDIR) \
ARCH_DIRNAME=$(ARCH_DIRNAME) \ ARCH_DIRNAME=$(ARCH_DIRNAME) \
ANDROID_ARCH=$(ANDROID_ARCH) \ ANDROID_ARCH=$(ANDROID_ARCH) \
ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) \ ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) \
@ -123,37 +123,37 @@ build-libsds-for-android-arch:
nim libsdsAndroid $(NIM_PARAMS) sds.nims nim libsdsAndroid $(NIM_PARAMS) sds.nims
libsds-android-arm64: ANDROID_ARCH=aarch64-linux-android libsds-android-arm64: ANDROID_ARCH=aarch64-linux-android
libsds-android-arm64: CPU=arm64 libsds-android-arm64: ARCH=arm64
libsds-android-arm64: ABIDIR=arm64-v8a libsds-android-arm64: ABIDIR=arm64-v8a
libsds-android-arm64: ARCH_DIRNAME=aarch64-linux-android libsds-android-arm64: ARCH_DIRNAME=aarch64-linux-android
libsds-android-arm64: | libsds-android-precheck build deps libsds-android-arm64: | libsds-android-precheck build deps
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \ $(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
CPU=$(CPU) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
libsds-android-amd64: ANDROID_ARCH=x86_64-linux-android libsds-android-amd64: ANDROID_ARCH=x86_64-linux-android
libsds-android-amd64: CPU=amd64 libsds-android-amd64: ARCH=amd64
libsds-android-amd64: ABIDIR=x86_64 libsds-android-amd64: ABIDIR=x86_64
libsds-android-amd64: ARCH_DIRNAME=x86_64-linux-android libsds-android-amd64: ARCH_DIRNAME=x86_64-linux-android
libsds-android-amd64: | libsds-android-precheck build deps libsds-android-amd64: | libsds-android-precheck build deps
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \ $(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
CPU=$(CPU) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
libsds-android-x86: ANDROID_ARCH=i686-linux-android libsds-android-x86: ANDROID_ARCH=i686-linux-android
libsds-android-x86: CPU=i386 libsds-android-x86: ARCH=i386
libsds-android-x86: ABIDIR=x86 libsds-android-x86: ABIDIR=x86
libsds-android-x86: ARCH_DIRNAME=i686-linux-android libsds-android-x86: ARCH_DIRNAME=i686-linux-android
libsds-android-x86: | libsds-android-precheck build deps libsds-android-x86: | libsds-android-precheck build deps
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \ $(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
CPU=$(CPU) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
libsds-android-arm: ANDROID_ARCH=armv7a-linux-androideabi libsds-android-arm: ANDROID_ARCH=armv7a-linux-androideabi
libsds-android-arm: CPU=arm libsds-android-arm: ARCH=arm
libsds-android-arm: ABIDIR=armeabi-v7a libsds-android-arm: ABIDIR=armeabi-v7a
libsds-android-arm: ARCH_DIRNAME=arm-linux-androideabi libsds-android-arm: ARCH_DIRNAME=arm-linux-androideabi
libsds-android-arm: | libsds-android-precheck build deps libsds-android-arm: | libsds-android-precheck build deps
# cross-rs target architecture name does not match the one used in android # cross-rs target architecture name does not match the one used in android
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \ $(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
CPU=$(CPU) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) \ ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) \
libsds-android: libsds-android:
ifeq ($(ARCH),arm64) ifeq ($(ARCH),arm64)

View File

@ -30,8 +30,8 @@ unzip android-ndk-r27c-linux.zip
Then, add the following to your ~/.bashrc file: Then, add the following to your ~/.bashrc file:
```code ```code
export ANDROID_NDK_HOME=$HOME/android-ndk-r27c export ANDROID_NDK_ROOT=$HOME/android-ndk-r27c
export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH export PATH=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH
``` ```
Then, use one of the following commands, according to the current architecture: Then, use one of the following commands, according to the current architecture:

View File

@ -5,7 +5,7 @@
when defined(linux): when defined(linux):
{.passl: "-Wl,-soname,libsds.so".} {.passl: "-Wl,-soname,libsds.so".}
import std/[typetraits, tables, atomics], chronos, chronicles import std/[typetraits, tables, atomics, locks], chronos, chronicles
import import
./sds_thread/sds_thread, ./sds_thread/sds_thread,
./alloc, ./alloc,
@ -57,6 +57,29 @@ template callEventCallback(ctx: ptr SdsContext, eventName: string, body: untyped
RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData
) )
var
ctxPool: seq[ptr SdsContext]
ctxPoolLock: Lock
proc acquireCtx(callback: SdsCallBack, userData: pointer): ptr SdsContext =
ctxPoolLock.acquire()
defer: ctxPoolLock.release()
if ctxPool.len > 0:
result = ctxPool.pop()
else:
result = sds_thread.createSdsThread().valueOr:
let msg = "Error in createSdsThread: " & $error
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
return nil
proc releaseCtx(ctx: ptr SdsContext) =
ctxPoolLock.acquire()
defer: ctxPoolLock.release()
ctx.userData = nil
ctx.eventCallback = nil
ctx.eventUserData = nil
ctxPool.add(ctx)
proc handleRequest( proc handleRequest(
ctx: ptr SdsContext, ctx: ptr SdsContext,
requestType: RequestType, requestType: RequestType,
@ -140,10 +163,9 @@ proc SdsNewReliabilityManager(
echo "error: missing callback in NewReliabilityManager" echo "error: missing callback in NewReliabilityManager"
return nil return nil
## Create the SDS thread that will keep waiting for req from the main thread. ## Create or reuse the SDS thread that will keep waiting for req from the main thread.
var ctx = sds_thread.createSdsThread().valueOr: var ctx = acquireCtx(callback, userData)
let msg = "Error in createSdsThread: " & $error if ctx.isNil():
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
return nil return nil
ctx.userData = userData ctx.userData = userData
@ -183,14 +205,20 @@ proc SdsCleanupReliabilityManager(
initializeLibrary() initializeLibrary()
checkLibsdsParams(ctx, callback, userData) checkLibsdsParams(ctx, callback, userData)
sds_thread.destroySdsThread(ctx).isOkOr: let resetRes = handleRequest(
let msg = "libsds error: " & $error ctx,
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) RequestType.LIFECYCLE,
SdsLifecycleRequest.createShared(SdsLifecycleMsgType.RESET_RELIABILITY_MANAGER),
callback,
userData,
)
if resetRes == RET_ERR:
return RET_ERR return RET_ERR
## always need to invoke the callback although we don't retrieve value to the caller releaseCtx(ctx)
callback(RET_OK, nil, 0, userData)
# handleRequest already invoked the callback; nothing else to signal here.
return RET_OK return RET_OK
proc SdsResetReliabilityManager( proc SdsResetReliabilityManager(

View File

@ -63,7 +63,6 @@ proc process*(
let messageBytes = self.message.toSeq() let messageBytes = self.message.toSeq()
let (unwrappedMessage, missingDeps, channelId) = unwrapReceivedMessage(rm[], messageBytes).valueOr: let (unwrappedMessage, missingDeps, channelId) = unwrapReceivedMessage(rm[], messageBytes).valueOr:
error "UNWRAP_MESSAGE failed", error = error
return err("error processing UNWRAP_MESSAGE request: " & $error) return err("error processing UNWRAP_MESSAGE request: " & $error)
let res = SdsUnwrapResponse(message: unwrappedMessage, missingDeps: missingDeps, channelId: channelId) let res = SdsUnwrapResponse(message: unwrappedMessage, missingDeps: missingDeps, channelId: channelId)

View File

@ -48,7 +48,7 @@ in stdenv.mkDerivation rec {
]; ];
ANDROID_SDK_ROOT = optionalString isAndroidBuild pkgs.androidPkgs.sdk; ANDROID_SDK_ROOT = optionalString isAndroidBuild pkgs.androidPkgs.sdk;
ANDROID_NDK_HOME = optionalString isAndroidBuild pkgs.androidPkgs.ndk; ANDROID_NDK_ROOT = optionalString isAndroidBuild pkgs.androidPkgs.ndk;
NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${version}"; NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${version}";
XDG_CACHE_HOME = "/tmp"; XDG_CACHE_HOME = "/tmp";

View File

@ -8,13 +8,13 @@ mkShell {
export ANDROID_HOME="${androidPkgs.sdk}" export ANDROID_HOME="${androidPkgs.sdk}"
export ANDROID_NDK_ROOT="${androidPkgs.ndk}" export ANDROID_NDK_ROOT="${androidPkgs.ndk}"
export ANDROID_SDK_ROOT="$ANDROID_HOME" export ANDROID_SDK_ROOT="$ANDROID_HOME"
export ANDROID_NDK_HOME="${androidPkgs.ndk}" export ANDROID_NDK_ROOT="${androidPkgs.ndk}"
export PATH="$ANDROID_NDK_ROOT:$PATH" export PATH="$ANDROID_NDK_ROOT:$PATH"
export PATH="$ANDROID_SDK_ROOT/tools:$PATH" export PATH="$ANDROID_SDK_ROOT/tools:$PATH"
export PATH="$ANDROID_SDK_ROOT/tools/bin:$PATH" export PATH="$ANDROID_SDK_ROOT/tools/bin:$PATH"
export PATH="$ANDROID_SDK_ROOT/platform-tools:$PATH" export PATH="$ANDROID_SDK_ROOT/platform-tools:$PATH"
export PATH="$(echo $ANDROID_SDK_ROOT/cmdline-tools/*/bin):$PATH" export PATH="$(echo $ANDROID_SDK_ROOT/cmdline-tools/*/bin):$PATH"
export PATH="$(echo $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/*/bin):$PATH" export PATH="$(echo $ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/*/bin):$PATH"
''; '';
} }

View File

@ -1,5 +1,7 @@
mode = ScriptMode.Verbose mode = ScriptMode.Verbose
import strutils, os
# Package # Package
version = "0.1.0" version = "0.1.0"
author = "Waku Team" author = "Waku Team"
@ -38,6 +40,12 @@ proc buildLibrary(
" --threads:on --app:lib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds --skipParentCfg:on " & " --threads:on --app:lib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds --skipParentCfg:on " &
extra_params & " " & srcDir & name & ".nim" extra_params & " " & srcDir & name & ".nim"
proc getArch(): string =
let arch = getEnv("ARCH")
if arch != "": return $arch
let (archFromUname, _) = gorgeEx("uname -m")
return $archFromUname
# Tasks # Tasks
task test, "Run the test suite": task test, "Run the test suite":
exec "nim c -r tests/test_bloom.nim" exec "nim c -r tests/test_bloom.nim"
@ -62,9 +70,14 @@ task libsdsDynamicLinux, "Generate bindings":
task libsdsDynamicMac, "Generate bindings": task libsdsDynamicMac, "Generate bindings":
let outLibNameAndExt = "libsds.dylib" let outLibNameAndExt = "libsds.dylib"
let name = "libsds" let name = "libsds"
let arch = getArch()
let sdkPath = staticExec("xcrun --show-sdk-path").strip()
let archFlags = (if arch == "arm64": "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
else: "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"")
buildLibrary outLibNameAndExt, buildLibrary outLibNameAndExt,
name, "library/", name, "library/",
"""-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """, archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE",
"dynamic" "dynamic"
task libsdsStaticWindows, "Generate bindings": task libsdsStaticWindows, "Generate bindings":
@ -86,9 +99,14 @@ task libsdsStaticLinux, "Generate bindings":
task libsdsStaticMac, "Generate bindings": task libsdsStaticMac, "Generate bindings":
let outLibNameAndExt = "libsds.a" let outLibNameAndExt = "libsds.a"
let name = "libsds" let name = "libsds"
let arch = getArch()
let sdkPath = staticExec("xcrun --show-sdk-path").strip()
let archFlags = (if arch == "arm64": "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
else: "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"")
buildLibrary outLibNameAndExt, buildLibrary outLibNameAndExt,
name, "library/", name, "library/",
"""-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """, archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE",
"static" "static"
# Build Mobile iOS # Build Mobile iOS
@ -96,35 +114,52 @@ proc buildMobileIOS(srcDir = ".", sdkPath = "") =
echo "Building iOS libsds library" echo "Building iOS libsds library"
let outDir = "build" let outDir = "build"
let nimcacheDir = outDir & "/nimcache"
if not dirExists outDir: if not dirExists outDir:
mkDir outDir mkDir outDir
if sdkPath.len == 0: if sdkPath.len == 0:
quit "Error: Xcode/iOS SDK not found" quit "Error: Xcode/iOS SDK not found"
# The output static library
let cFile = outDir & "/nimcache/@mlibsds.nim.c"
let oFile = outDir & "/libsds.o"
let aFile = outDir & "/libsds.a" let aFile = outDir & "/libsds.a"
let aFileTmp = outDir & "/libsds_tmp.a"
let arch = getArch()
# 1) Generate C sources from Nim (no linking) # 1) Generate C sources from Nim (no linking)
# Use unique symbol prefix to avoid conflicts with other Nim libraries
exec "nim c" & exec "nim c" &
" --nimcache:build/nimcache --os:ios --cpu:arm64" & " --nimcache:" & nimcacheDir & " --os:ios --cpu:" & arch &
" --compileOnly:on" & " --compileOnly:on" &
" --noMain --mm:refc" & " --noMain --mm:orc" &
" --threads:on --opt:size --header" & " --threads:on --opt:size --header" &
" --nimMainPrefix:libsds --skipParentCfg:on" & " --nimMainPrefix:libsds --skipParentCfg:on" &
" --cc:clang" & " --cc:clang" &
" --out:" & cFile & " " & " -d:useMalloc" &
srcDir & "/libsds.nim" " " & srcDir & "/libsds.nim"
exec "clang -arch arm64" & # 2) Compile all generated C files to object files with hidden visibility
" -isysroot " & sdkPath & # This prevents symbol conflicts with other Nim libraries (e.g., libnim_status_client)
let clangFlags = "-arch " & arch & " -isysroot " & sdkPath &
" -I./vendor/nimbus-build-system/vendor/Nim/lib/" & " -I./vendor/nimbus-build-system/vendor/Nim/lib/" &
" -fembed-bitcode -c " & cFile & " -fembed-bitcode -miphoneos-version-min=16.0 -O2" &
" -o " & oFile " -fvisibility=hidden"
exec "ar rcs " & aFile & " " & oFile var objectFiles: seq[string] = @[]
for cFile in listFiles(nimcacheDir):
if cFile.endsWith(".c"):
let oFile = cFile.changeFileExt("o")
exec "clang " & clangFlags & " -c " & cFile & " -o " & oFile
objectFiles.add(oFile)
# 3) Create static library from all object files
exec "ar rcs " & aFileTmp & " " & objectFiles.join(" ")
# 4) Use libtool to localize all non-public symbols
# Keep only Sds* functions as global, hide everything else to prevent conflicts
# with nim runtime symbols from libnim_status_client
let keepSymbols = "_Sds*:_libsdsNimMain:_libsdsDatInit*:_libsdsInit*:_NimMainModule__libsds*"
exec "xcrun libtool -static -o " & aFile & " " & aFileTmp &
" -exported_symbols_list /dev/stdin <<< '" & keepSymbols & "' 2>/dev/null || cp " & aFileTmp & " " & aFile
echo "✔ iOS library created: " & aFile echo "✔ iOS library created: " & aFile
@ -135,7 +170,7 @@ task libsdsIOS, "Build the mobile bindings for iOS":
### Mobile Android ### Mobile Android
proc buildMobileAndroid(srcDir = ".", params = "") = proc buildMobileAndroid(srcDir = ".", params = "") =
let cpu = getEnv("CPU") let cpu = getArch()
let outDir = "build/" let outDir = "build/"
if not dirExists outDir: if not dirExists outDir: