Compare commits

..

7 Commits

Author SHA1 Message Date
Siddarth Kumar
fb8039c5a5
chore: fix iOS build
otherwise iOS linker fails with
  Undefined symbols for architecture arm64
2025-12-23 19:47:02 +04:00
Igor Sirotin
8d33a7f7da
feat: thread pool (#40)
* feat: thread pool

* proper pass ARCH in Makefile when building for Android

---------

Co-authored-by: Ivan Folgueira Bande <ivansete@status.im>
2025-12-22 18:10:45 +00:00
Ivan Folgueira Bande
e67639ee08
get arch from uname -m if ARCH env var is not set 2025-12-17 16:02:53 +01:00
Ivan FB
ac31e5adf2
Merge pull request #37 from logos-messaging/fix/buildForAppleSilicon
fix: Add condition to check hostCpu and then build based on that
2025-12-15 12:06:17 +01:00
Khushboo Mehta
13c3c348fa fix: Add condition to check hostCpu and then build based on that 2025-12-15 12:01:51 +01:00
Ivan FB
0b4d3cc03f
Merge pull request #38 from logos-messaging/rm-log
rm UNWRAP_MESSAGE failed error
2025-12-10 09:51:41 +01:00
Ivan Folgueira Bande
191928adc6
rm UNWRAP_MESSAGE failed error 2025-12-09 11:45:35 +01:00
4 changed files with 97 additions and 35 deletions

View File

@ -115,7 +115,7 @@ build-libsds-for-android-arch: NIM_PARAMS := $(NIM_PARAMS) --passC="-I$(ANDROID_
build-libsds-for-android-arch: NIM_PARAMS := $(NIM_PARAMS) --passL="-L$(ANDROID_TOOLCHAIN_DIR)/sysroot/usr/lib/$(ARCH_DIRNAME)/$(ANDROID_TARGET)" build-libsds-for-android-arch: NIM_PARAMS := $(NIM_PARAMS) --passL="-L$(ANDROID_TOOLCHAIN_DIR)/sysroot/usr/lib/$(ARCH_DIRNAME)/$(ANDROID_TARGET)"
build-libsds-for-android-arch: build-libsds-for-android-arch:
CC=$(ANDROID_TOOLCHAIN_DIR)/bin/$(ANDROID_ARCH)$(ANDROID_TARGET)-clang \ CC=$(ANDROID_TOOLCHAIN_DIR)/bin/$(ANDROID_ARCH)$(ANDROID_TARGET)-clang \
CPU=$(CPU) ABIDIR=$(ABIDIR) \ ARCH=$(ARCH) ABIDIR=$(ABIDIR) \
ARCH_DIRNAME=$(ARCH_DIRNAME) \ ARCH_DIRNAME=$(ARCH_DIRNAME) \
ANDROID_ARCH=$(ANDROID_ARCH) \ ANDROID_ARCH=$(ANDROID_ARCH) \
ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) \ ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) \
@ -123,37 +123,37 @@ build-libsds-for-android-arch:
nim libsdsAndroid $(NIM_PARAMS) sds.nims nim libsdsAndroid $(NIM_PARAMS) sds.nims
libsds-android-arm64: ANDROID_ARCH=aarch64-linux-android libsds-android-arm64: ANDROID_ARCH=aarch64-linux-android
libsds-android-arm64: CPU=arm64 libsds-android-arm64: ARCH=arm64
libsds-android-arm64: ABIDIR=arm64-v8a libsds-android-arm64: ABIDIR=arm64-v8a
libsds-android-arm64: ARCH_DIRNAME=aarch64-linux-android libsds-android-arm64: ARCH_DIRNAME=aarch64-linux-android
libsds-android-arm64: | libsds-android-precheck build deps libsds-android-arm64: | libsds-android-precheck build deps
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \ $(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
CPU=$(CPU) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
libsds-android-amd64: ANDROID_ARCH=x86_64-linux-android libsds-android-amd64: ANDROID_ARCH=x86_64-linux-android
libsds-android-amd64: CPU=amd64 libsds-android-amd64: ARCH=amd64
libsds-android-amd64: ABIDIR=x86_64 libsds-android-amd64: ABIDIR=x86_64
libsds-android-amd64: ARCH_DIRNAME=x86_64-linux-android libsds-android-amd64: ARCH_DIRNAME=x86_64-linux-android
libsds-android-amd64: | libsds-android-precheck build deps libsds-android-amd64: | libsds-android-precheck build deps
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \ $(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
CPU=$(CPU) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
libsds-android-x86: ANDROID_ARCH=i686-linux-android libsds-android-x86: ANDROID_ARCH=i686-linux-android
libsds-android-x86: CPU=i386 libsds-android-x86: ARCH=i386
libsds-android-x86: ABIDIR=x86 libsds-android-x86: ABIDIR=x86
libsds-android-x86: ARCH_DIRNAME=i686-linux-android libsds-android-x86: ARCH_DIRNAME=i686-linux-android
libsds-android-x86: | libsds-android-precheck build deps libsds-android-x86: | libsds-android-precheck build deps
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \ $(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
CPU=$(CPU) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME)
libsds-android-arm: ANDROID_ARCH=armv7a-linux-androideabi libsds-android-arm: ANDROID_ARCH=armv7a-linux-androideabi
libsds-android-arm: CPU=arm libsds-android-arm: ARCH=arm
libsds-android-arm: ABIDIR=armeabi-v7a libsds-android-arm: ABIDIR=armeabi-v7a
libsds-android-arm: ARCH_DIRNAME=arm-linux-androideabi libsds-android-arm: ARCH_DIRNAME=arm-linux-androideabi
libsds-android-arm: | libsds-android-precheck build deps libsds-android-arm: | libsds-android-precheck build deps
# cross-rs target architecture name does not match the one used in android # cross-rs target architecture name does not match the one used in android
$(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \ $(MAKE) build-libsds-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) \
CPU=$(CPU) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) \ ARCH=$(ARCH) ABIDIR=$(ABIDIR) ARCH_DIRNAME=$(ARCH_DIRNAME) \
libsds-android: libsds-android:
ifeq ($(ARCH),arm64) ifeq ($(ARCH),arm64)

View File

@ -5,7 +5,7 @@
when defined(linux): when defined(linux):
{.passl: "-Wl,-soname,libsds.so".} {.passl: "-Wl,-soname,libsds.so".}
import std/[typetraits, tables, atomics], chronos, chronicles import std/[typetraits, tables, atomics, locks], chronos, chronicles
import import
./sds_thread/sds_thread, ./sds_thread/sds_thread,
./alloc, ./alloc,
@ -57,6 +57,29 @@ template callEventCallback(ctx: ptr SdsContext, eventName: string, body: untyped
RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData
) )
var
ctxPool: seq[ptr SdsContext]
ctxPoolLock: Lock
proc acquireCtx(callback: SdsCallBack, userData: pointer): ptr SdsContext =
ctxPoolLock.acquire()
defer: ctxPoolLock.release()
if ctxPool.len > 0:
result = ctxPool.pop()
else:
result = sds_thread.createSdsThread().valueOr:
let msg = "Error in createSdsThread: " & $error
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
return nil
proc releaseCtx(ctx: ptr SdsContext) =
ctxPoolLock.acquire()
defer: ctxPoolLock.release()
ctx.userData = nil
ctx.eventCallback = nil
ctx.eventUserData = nil
ctxPool.add(ctx)
proc handleRequest( proc handleRequest(
ctx: ptr SdsContext, ctx: ptr SdsContext,
requestType: RequestType, requestType: RequestType,
@ -140,10 +163,9 @@ proc SdsNewReliabilityManager(
echo "error: missing callback in NewReliabilityManager" echo "error: missing callback in NewReliabilityManager"
return nil return nil
## Create the SDS thread that will keep waiting for req from the main thread. ## Create or reuse the SDS thread that will keep waiting for req from the main thread.
var ctx = sds_thread.createSdsThread().valueOr: var ctx = acquireCtx(callback, userData)
let msg = "Error in createSdsThread: " & $error if ctx.isNil():
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
return nil return nil
ctx.userData = userData ctx.userData = userData
@ -183,14 +205,20 @@ proc SdsCleanupReliabilityManager(
initializeLibrary() initializeLibrary()
checkLibsdsParams(ctx, callback, userData) checkLibsdsParams(ctx, callback, userData)
sds_thread.destroySdsThread(ctx).isOkOr: let resetRes = handleRequest(
let msg = "libsds error: " & $error ctx,
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) RequestType.LIFECYCLE,
SdsLifecycleRequest.createShared(SdsLifecycleMsgType.RESET_RELIABILITY_MANAGER),
callback,
userData,
)
if resetRes == RET_ERR:
return RET_ERR return RET_ERR
## always need to invoke the callback although we don't retrieve value to the caller releaseCtx(ctx)
callback(RET_OK, nil, 0, userData)
# handleRequest already invoked the callback; nothing else to signal here.
return RET_OK return RET_OK
proc SdsResetReliabilityManager( proc SdsResetReliabilityManager(

View File

@ -63,7 +63,6 @@ proc process*(
let messageBytes = self.message.toSeq() let messageBytes = self.message.toSeq()
let (unwrappedMessage, missingDeps, channelId) = unwrapReceivedMessage(rm[], messageBytes).valueOr: let (unwrappedMessage, missingDeps, channelId) = unwrapReceivedMessage(rm[], messageBytes).valueOr:
error "UNWRAP_MESSAGE failed", error = error
return err("error processing UNWRAP_MESSAGE request: " & $error) return err("error processing UNWRAP_MESSAGE request: " & $error)
let res = SdsUnwrapResponse(message: unwrappedMessage, missingDeps: missingDeps, channelId: channelId) let res = SdsUnwrapResponse(message: unwrappedMessage, missingDeps: missingDeps, channelId: channelId)

View File

@ -1,5 +1,7 @@
mode = ScriptMode.Verbose mode = ScriptMode.Verbose
import strutils, os
# Package # Package
version = "0.1.0" version = "0.1.0"
author = "Waku Team" author = "Waku Team"
@ -38,6 +40,12 @@ proc buildLibrary(
" --threads:on --app:lib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds --skipParentCfg:on " & " --threads:on --app:lib --opt:size --noMain --mm:refc --header --nimMainPrefix:libsds --skipParentCfg:on " &
extra_params & " " & srcDir & name & ".nim" extra_params & " " & srcDir & name & ".nim"
proc getArch(): string =
let arch = getEnv("ARCH")
if arch != "": return $arch
let (archFromUname, _) = gorgeEx("uname -m")
return $archFromUname
# Tasks # Tasks
task test, "Run the test suite": task test, "Run the test suite":
exec "nim c -r tests/test_bloom.nim" exec "nim c -r tests/test_bloom.nim"
@ -62,9 +70,14 @@ task libsdsDynamicLinux, "Generate bindings":
task libsdsDynamicMac, "Generate bindings": task libsdsDynamicMac, "Generate bindings":
let outLibNameAndExt = "libsds.dylib" let outLibNameAndExt = "libsds.dylib"
let name = "libsds" let name = "libsds"
let arch = getArch()
let sdkPath = staticExec("xcrun --show-sdk-path").strip()
let archFlags = (if arch == "arm64": "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
else: "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"")
buildLibrary outLibNameAndExt, buildLibrary outLibNameAndExt,
name, "library/", name, "library/",
"""-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """, archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE",
"dynamic" "dynamic"
task libsdsStaticWindows, "Generate bindings": task libsdsStaticWindows, "Generate bindings":
@ -86,9 +99,14 @@ task libsdsStaticLinux, "Generate bindings":
task libsdsStaticMac, "Generate bindings": task libsdsStaticMac, "Generate bindings":
let outLibNameAndExt = "libsds.a" let outLibNameAndExt = "libsds.a"
let name = "libsds" let name = "libsds"
let arch = getArch()
let sdkPath = staticExec("xcrun --show-sdk-path").strip()
let archFlags = (if arch == "arm64": "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
else: "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"")
buildLibrary outLibNameAndExt, buildLibrary outLibNameAndExt,
name, "library/", name, "library/",
"""-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """, archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE",
"static" "static"
# Build Mobile iOS # Build Mobile iOS
@ -96,35 +114,52 @@ proc buildMobileIOS(srcDir = ".", sdkPath = "") =
echo "Building iOS libsds library" echo "Building iOS libsds library"
let outDir = "build" let outDir = "build"
let nimcacheDir = outDir & "/nimcache"
if not dirExists outDir: if not dirExists outDir:
mkDir outDir mkDir outDir
if sdkPath.len == 0: if sdkPath.len == 0:
quit "Error: Xcode/iOS SDK not found" quit "Error: Xcode/iOS SDK not found"
# The output static library
let cFile = outDir & "/nimcache/@mlibsds.nim.c"
let oFile = outDir & "/libsds.o"
let aFile = outDir & "/libsds.a" let aFile = outDir & "/libsds.a"
let aFileTmp = outDir & "/libsds_tmp.a"
let arch = getArch()
# 1) Generate C sources from Nim (no linking) # 1) Generate C sources from Nim (no linking)
# Use unique symbol prefix to avoid conflicts with other Nim libraries
exec "nim c" & exec "nim c" &
" --nimcache:build/nimcache --os:ios --cpu:arm64" & " --nimcache:" & nimcacheDir & " --os:ios --cpu:" & arch &
" --compileOnly:on" & " --compileOnly:on" &
" --noMain --mm:refc" & " --noMain --mm:orc" &
" --threads:on --opt:size --header" & " --threads:on --opt:size --header" &
" --nimMainPrefix:libsds --skipParentCfg:on" & " --nimMainPrefix:libsds --skipParentCfg:on" &
" --cc:clang" & " --cc:clang" &
" --out:" & cFile & " " & " -d:useMalloc" &
srcDir & "/libsds.nim" " " & srcDir & "/libsds.nim"
exec "clang -arch arm64" & # 2) Compile all generated C files to object files with hidden visibility
" -isysroot " & sdkPath & # This prevents symbol conflicts with other Nim libraries (e.g., libnim_status_client)
let clangFlags = "-arch " & arch & " -isysroot " & sdkPath &
" -I./vendor/nimbus-build-system/vendor/Nim/lib/" & " -I./vendor/nimbus-build-system/vendor/Nim/lib/" &
" -fembed-bitcode -c " & cFile & " -fembed-bitcode -miphoneos-version-min=16.0 -O2" &
" -o " & oFile " -fvisibility=hidden"
exec "ar rcs " & aFile & " " & oFile var objectFiles: seq[string] = @[]
for cFile in listFiles(nimcacheDir):
if cFile.endsWith(".c"):
let oFile = cFile.changeFileExt("o")
exec "clang " & clangFlags & " -c " & cFile & " -o " & oFile
objectFiles.add(oFile)
# 3) Create static library from all object files
exec "ar rcs " & aFileTmp & " " & objectFiles.join(" ")
# 4) Use libtool to localize all non-public symbols
# Keep only Sds* functions as global, hide everything else to prevent conflicts
# with nim runtime symbols from libnim_status_client
let keepSymbols = "_Sds*:_libsdsNimMain:_libsdsDatInit*:_libsdsInit*:_NimMainModule__libsds*"
exec "xcrun libtool -static -o " & aFile & " " & aFileTmp &
" -exported_symbols_list /dev/stdin <<< '" & keepSymbols & "' 2>/dev/null || cp " & aFileTmp & " " & aFile
echo "✔ iOS library created: " & aFile echo "✔ iOS library created: " & aFile
@ -135,7 +170,7 @@ task libsdsIOS, "Build the mobile bindings for iOS":
### Mobile Android ### Mobile Android
proc buildMobileAndroid(srcDir = ".", params = "") = proc buildMobileAndroid(srcDir = ".", params = "") =
let cpu = getEnv("CPU") let cpu = getArch()
let outDir = "build/" let outDir = "build/"
if not dirExists outDir: if not dirExists outDir: