Init library

This commit is contained in:
Arnaud 2025-09-09 20:30:03 +02:00 committed by Eric
parent f791a960f2
commit fee0c80db8
No known key found for this signature in database
14 changed files with 1197 additions and 0 deletions

View File

@ -232,6 +232,7 @@ format:
$(NPH) *.nim
$(NPH) codex/
$(NPH) tests/
$(NPH) library/
clean-nph:
rm -f $(NPH)
@ -242,4 +243,21 @@ print-nph-path:
clean: | clean-nph
################
## C Bindings ##
################
.PHONY: libcodex
STATIC ?= 0
libcodex: deps
rm -f build/libcodex*
ifeq ($(STATIC), 1)
echo -e $(BUILD_MSG) "build/$@.a" && \
$(ENV_SCRIPT) nim libcodexStatic $(NIM_PARAMS) codex.nims
else
echo -e $(BUILD_MSG) "build/$@.so" && \
$(ENV_SCRIPT) nim libcodexDynamic $(NIM_PARAMS) codex.nims
endif
endif # "variables.mk" was not included

View File

@ -53,6 +53,52 @@ To get acquainted with Codex, consider:
The client exposes a REST API that can be used to interact with the clients. Overview of the API can be found on [api.codex.storage](https://api.codex.storage).
## Bindings
Codex provides a C API that can be wrapped by other languages. The bindings is located in the `library` folder.
Currently, only a Go binding is included.
### Build the C library
```bash
make libcodex
```
This produces the shared library under `build/`.
### Run the Go example
Build the Go example:
```bash
go build -o codex-go examples/golang/codex.go
```
Export the library path:
```bash
export LD_LIBRARY_PATH=build
```
Run the example:
```bash
./codex-go
```
### Static vs Dynamic build
By default, Codex builds a dynamic library (`libcodex.so`), which you can load at runtime.
If you prefer a static library (`libcodex.a`), set the `STATIC` flag:
```bash
# Build dynamic (default)
make libcodex
# Build static
make STATIC=1 libcodex
```
## Contributing and development
Feel free to dive in, contributions are welcomed! Open an issue or submit PRs.

View File

@ -25,6 +25,20 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
exec(cmd)
proc buildLibrary(name: string, srcDir = "./", params = "", `type` = "static") =
if not dirExists "build":
mkDir "build"
# allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
var extra_params = params
if `type` == "static":
exec "nim c" & " --out:build/" & name &
".a --threads:on --app:staticlib --opt:size --noMain --mm:refc --header --d:metrics --nimMainPrefix:libcodex --skipParentCfg:on -d:noSignalHandler " &
extra_params & " " & srcDir & name & ".nim"
else:
exec "nim c" & " --out:build/" & name &
".so --threads:on --app:lib --opt:size --noMain --mm:refc --header --d:metrics --nimMainPrefix:libcodex --skipParentCfg:on -d:noSignalHandler -d:LeopardCmakeFlags=\"-DCMAKE_POSITION_INDEPENDENT_CODE=ON\"" &
extra_params & " " & srcDir & name & ".nim"
proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
buildBinary name, srcDir, params
exec "build/" & name
@ -121,3 +135,11 @@ task showCoverage, "open coverage html":
echo " ======== Opening HTML coverage report in browser... ======== "
if findExe("open") != "":
exec("open coverage/report/index.html")
task libcodexDynamic, "Generate bindings":
let name = "libcodex"
buildLibrary name, "library/", "", "dynamic"
task libcodextatic, "Generate bindings":
let name = "libcodex"
buildLibrary name, "library/", "", "static"

24
examples/golang/README.md Normal file
View File

@ -0,0 +1,24 @@
## Pre-requisite
libcodex.so is needed to be compiled and present in build folder.
## Compilation
From the codex root folder:
```code
go build -o codex-go examples/golang/codex.go
```
## Run
From the codex root folder:
```code
export LD_LIBRARY_PATH=build
```
```code
./codex-go
```

296
examples/golang/codex.go Normal file
View File

@ -0,0 +1,296 @@
package main
/*
#cgo LDFLAGS: -L../../build/ -lcodex
#cgo LDFLAGS: -L../../ -Wl,-rpath,../../
#include "../../library/libcodex.h"
#include <stdio.h>
#include <stdlib.h>
void libcodexNimMain(void);
static void codex_host_init_once(void){
static int done;
if (!__atomic_exchange_n(&done, 1, __ATOMIC_SEQ_CST)) libcodexNimMain();
}
extern void globalEventCallback(int ret, char* msg, size_t len, void* userData);
typedef struct {
int ret;
char* msg;
size_t len;
} Resp;
static void* allocResp() {
return calloc(1, sizeof(Resp));
}
static void freeResp(void* resp) {
if (resp != NULL) {
free(resp);
}
}
static char* getMyCharPtr(void* resp) {
if (resp == NULL) {
return NULL;
}
Resp* m = (Resp*) resp;
return m->msg;
}
static size_t getMyCharLen(void* resp) {
if (resp == NULL) {
return 0;
}
Resp* m = (Resp*) resp;
return m->len;
}
static int getRet(void* resp) {
if (resp == NULL) {
return 0;
}
Resp* m = (Resp*) resp;
return m->ret;
}
// resp must be set != NULL in case interest on retrieving data from the callback
static void callback(int ret, char* msg, size_t len, void* resp) {
if (resp != NULL) {
Resp* m = (Resp*) resp;
m->ret = ret;
m->msg = msg;
m->len = len;
}
}
#define CODEX_CALL(call) \
do { \
int ret = call; \
if (ret != 0) { \
printf("Failed the call to: %s. Returned code: %d\n", #call, ret); \
exit(1); \
} \
} while (0)
static void* cGoCodexNew(const char* configJson, void* resp) {
void* ret = codex_new(configJson, (CodexCallback) callback, resp);
return ret;
}
static void cGoCodexStart(void* codexCtx, void* resp) {
CODEX_CALL(codex_start(codexCtx, (CodexCallback) callback, resp));
}
static void cGoCodexStop(void* codexCtx, void* resp) {
CODEX_CALL(codex_stop(codexCtx, (CodexCallback) callback, resp));
}
static void cGoCodexDestroy(void* codexCtx, void* resp) {
CODEX_CALL(codex_destroy(codexCtx, (CodexCallback) callback, resp));
}
static void cGoCodexSetEventCallback(void* codexCtx) {
// The 'globalEventCallback' Go function is shared amongst all possible Codex instances.
// Given that the 'globalEventCallback' is shared, we pass again the
// codexCtx instance but in this case is needed to pick up the correct method
// that will handle the event.
// In other words, for every call the libcodex makes to globalEventCallback,
// the 'userData' parameter will bring the context of the node that registered
// that globalEventCallback.
// This technique is needed because cgo only allows to export Go functions and not methods.
codex_set_event_callback(codexCtx, (CodexCallback) globalEventCallback, codexCtx);
}
*/
import "C"
import (
"encoding/json"
"errors"
"fmt"
"log"
"os"
"os/signal"
"syscall"
"unsafe"
)
type LogLevel string
const (
Trace LogLevel = "TRACE"
Debug LogLevel = "DEBUG"
Info LogLevel = "INFO"
Notice LogLevel = "NOTICE"
Warn LogLevel = "WARN"
Error LogLevel = "ERROR"
Fatal LogLevel = "FATAL"
)
type LogFormat string
const (
LogFormatAuto LogFormat = "auto"
LogFormatColors LogFormat = "colors"
LogFormatNoColors LogFormat = "nocolors"
LogFormatJSON LogFormat = "json"
)
type RepoKind string
const (
FS RepoKind = "fs"
SQLite RepoKind = "sqlite"
LevelDb RepoKind = "leveldb"
)
type CodexConfig struct {
LogLevel LogLevel `json:"log-level,omitempty"`
LogFormat LogFormat `json:"log-format,omitempty"`
MetricsEnabled bool `json:"metrics,omitempty"`
MetricsAddress string `json:"metrics-address,omitempty"`
DataDir string `json:"data-dir,omitempty"`
ListenAddrs []string `json:"listen-addrs,omitempty"`
Nat string `json:"nat,omitempty"`
DiscoveryPort int `json:"disc-port,omitempty"`
NetPrivKeyFile string `json:"net-privkey,omitempty"`
BootstrapNodes []byte `json:"bootstrap-node,omitempty"`
MaxPeers int `json:"max-peers,omitempty"`
NumThreads int `json:"num-threads,omitempty"`
AgentString string `json:"agent-string,omitempty"`
RepoKind RepoKind `json:"repo-kind,omitempty"`
StorageQuota int `json:"storage-quota,omitempty"`
BlockTtl int `json:"block-ttl,omitempty"`
BlockMaintenanceInterval int `json:"block-mi,omitempty"`
BlockMaintenanceNumberOfBlocks int `json:"block-mn,omitempty"`
CacheSize int `json:"cache-size,omitempty"`
LogFile string `json:"log-file,omitempty"`
}
type CodexNode struct {
ctx unsafe.Pointer
}
func CodexNew(config CodexConfig) (*CodexNode, error) {
jsonConfig, err := json.Marshal(config)
if err != nil {
return nil, err
}
var cJsonConfig = C.CString(string(jsonConfig))
var resp = C.allocResp()
defer C.free(unsafe.Pointer(cJsonConfig))
defer C.freeResp(resp)
ctx := C.cGoCodexNew(cJsonConfig, resp)
if C.getRet(resp) == C.RET_OK {
return &CodexNode{ctx: ctx}, nil
}
errMsg := "error CodexNew: " + C.GoStringN(C.getMyCharPtr(resp), C.int(C.getMyCharLen(resp)))
return nil, errors.New(errMsg)
}
func (self *CodexNode) CodexStart() error {
var resp = C.allocResp()
defer C.freeResp(resp)
C.cGoCodexStart(self.ctx, resp)
if C.getRet(resp) == C.RET_OK {
return nil
}
errMsg := "error CodexStart: " + C.GoStringN(C.getMyCharPtr(resp), C.int(C.getMyCharLen(resp)))
return errors.New(errMsg)
}
func (self *CodexNode) CodexStop() error {
var resp = C.allocResp()
defer C.freeResp(resp)
C.cGoCodexStop(self.ctx, resp)
if C.getRet(resp) == C.RET_OK {
return nil
}
errMsg := "error CodexStop: " + C.GoStringN(C.getMyCharPtr(resp), C.int(C.getMyCharLen(resp)))
return errors.New(errMsg)
}
func (self *CodexNode) CodexDestroy() error {
var resp = C.allocResp()
defer C.freeResp(resp)
C.cGoCodexDestroy(self.ctx, resp)
if C.getRet(resp) == C.RET_OK {
return nil
}
errMsg := "error CodexDestroy: " + C.GoStringN(C.getMyCharPtr(resp), C.int(C.getMyCharLen(resp)))
return errors.New(errMsg)
}
//export globalEventCallback
func globalEventCallback(callerRet C.int, msg *C.char, len C.size_t, userData unsafe.Pointer) {
// This is shared among all Golang instances
self := CodexNode{ctx: userData}
self.MyEventCallback(callerRet, msg, len)
}
func (self *CodexNode) MyEventCallback(callerRet C.int, msg *C.char, len C.size_t) {
fmt.Println("Event received:", C.GoStringN(msg, C.int(len)))
}
func (self *CodexNode) CodexSetEventCallback() {
// Notice that the events for self node are handled by the 'MyEventCallback' method
C.cGoCodexSetEventCallback(self.ctx)
}
func main() {
config := CodexConfig{
LogLevel: Info,
}
log.Println("Starting Codex...")
node, err := CodexNew(config)
if err != nil {
fmt.Println("Error happened:", err.Error())
return
}
node.CodexSetEventCallback()
err = node.CodexStart()
if err != nil {
fmt.Println("Error happened:", err.Error())
return
}
log.Println("Codex started...")
// Wait for a SIGINT or SIGTERM signal
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
<-ch
log.Println("Stopping the node...")
err = node.CodexStop()
if err != nil {
fmt.Println("Error happened:", err.Error())
return
}
err = node.CodexDestroy()
if err != nil {
fmt.Println("Error happened:", err.Error())
return
}
}

37
library/README.md Normal file
View File

@ -0,0 +1,37 @@
# Codex Library
Codex exposes a C binding that serves as a stable contract, making it straightforward to integrate Codex into other languages such as Go.
The implementation was inspired by [nim-library-template](https://github.com/logos-co/nim-library-template)
and by the [nwaku](https://github.com/waku-org/nwaku/tree/master/library) library.
The source code contains detailed comments to explain the threading and callback flow.
The diagram below summarizes the lifecycle: context creation, request execution, and shutdown.
```mermaid
sequenceDiagram
autonumber
actor App as App/User
participant Go as Go Wrapper
participant C as C API (libcodex.h)
participant Ctx as CodexContext
participant Thr as Worker Thread
participant Eng as CodexServer
App->>Go: Start
Go->>C: codex_start_node
C->>Ctx: enqueue request
C->>Ctx: fire signal
Ctx->>Thr: wake worker
Thr->>Ctx: dequeue request
Thr-->>Ctx: ACK
Ctx-->>C: forward ACK
C-->>Go: RET OK
Go->>App: Unblock
Thr->>Eng: execute (async)
Eng-->>Thr: result ready
Thr-->>Ctx: callback
Ctx-->>C: forward callback
C-->>Go: forward callback
Go-->>App: done
```

42
library/alloc.nim Normal file
View File

@ -0,0 +1,42 @@
## Can be shared safely between threads
type SharedSeq*[T] = tuple[data: ptr UncheckedArray[T], len: int]
proc alloc*(str: cstring): cstring =
# Byte allocation from the given address.
# There should be the corresponding manual deallocation with deallocShared !
if str.isNil():
var ret = cast[cstring](allocShared(1)) # Allocate memory for the null terminator
ret[0] = '\0' # Set the null terminator
return ret
let ret = cast[cstring](allocShared(len(str) + 1))
copyMem(ret, str, len(str) + 1)
return ret
proc alloc*(str: string): cstring =
## Byte allocation from the given address.
## There should be the corresponding manual deallocation with deallocShared !
var ret = cast[cstring](allocShared(str.len + 1))
let s = cast[seq[char]](str)
for i in 0 ..< str.len:
ret[i] = s[i]
ret[str.len] = '\0'
return ret
proc allocSharedSeq*[T](s: seq[T]): SharedSeq[T] =
let data = allocShared(sizeof(T) * s.len)
if s.len != 0:
copyMem(data, unsafeAddr s[0], s.len)
return (cast[ptr UncheckedArray[T]](data), s.len)
proc deallocSharedSeq*[T](s: var SharedSeq[T]) =
deallocShared(s.data)
s.len = 0
proc toSeq*[T](s: SharedSeq[T]): seq[T] =
## Creates a seq[T] from a SharedSeq[T]. No explicit dealloc is required
## as req[T] is a GC managed type.
var ret = newSeq[T]()
for i in 0 ..< s.len:
ret.add(s.data[i])
return ret

198
library/codex_context.nim Normal file
View File

@ -0,0 +1,198 @@
## This file defines the Codex context and its thread flow:
## 1. Client enqueues a request and signals the Codex thread.
## 2. The Codex thread dequeues the request and sends an ack (reqReceivedSignal).
## 3. The Codex thread executes the request asynchronously.
## 4. On completion, the Codex thread invokes the client callback with the result and userData.
{.pragma: exported, exportc, cdecl, raises: [].}
{.pragma: callback, cdecl, raises: [], gcsafe.}
{.passc: "-fPIC".}
import std/[options, locks, atomics]
import chronicles
import chronos
import chronos/threadsync
import taskpools/channels_spsc_single
import ./ffi_types
import ./codex_thread_requests/[codex_thread_request]
from ../codex/codex import CodexServer
type CodexContext* = object
thread: Thread[(ptr CodexContext)]
# This lock is only necessary while we use a SP Channel and while the signalling
# between threads assumes that there aren't concurrent requests.
# Rearchitecting the signaling + migrating to a MP Channel will allow us to receive
# requests concurrently and spare us the need of locks
lock: Lock
# Channel to send requests to the Codex thread.
# Requests will be popped from this channel.
reqChannel: ChannelSPSCSingle[ptr CodexThreadRequest]
# To notify the Codex thread that a request is ready
reqSignal: ThreadSignalPtr
# To notify the client thread that the request was received.
# It is acknowledgment signal (handshake).
reqReceivedSignal: ThreadSignalPtr
# Custom state attached by the client to a request,
# returned when its callback is invoked
userData*: pointer
# Function called by the library to notify the client of global events
eventCallback*: pointer
# Custom state attached by the client to the context,
# returned with every event callback
eventUserData*: pointer
# Set to false to stop the Codex thread (during codex_destroy)
running: Atomic[bool]
template callEventCallback(ctx: ptr CodexContext, eventName: string, body: untyped) =
## Template used to notify the client of global events
## Example: onConnectionChanged, onProofMissing, etc.
if isNil(ctx[].eventCallback):
error eventName & " - eventCallback is nil"
return
foreignThreadGc:
try:
let event = body
cast[CodexCallback](ctx[].eventCallback)(
RET_OK, unsafeAddr event[0], cast[csize_t](len(event)), ctx[].eventUserData
)
except Exception, CatchableError:
let msg =
"Exception " & eventName & " when calling 'eventCallBack': " &
getCurrentExceptionMsg()
cast[CodexCallback](ctx[].eventCallback)(
RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), ctx[].eventUserData
)
proc sendRequestToCodexThread*(
ctx: ptr CodexContext,
reqType: RequestType,
reqContent: pointer,
callback: CodexCallback,
userData: pointer,
timeout = InfiniteDuration,
): Result[void, string] =
ctx.lock.acquire()
defer:
ctx.lock.release()
let req = CodexThreadRequest.createShared(reqType, reqContent, callback, userData)
# Send the request to the Codex thread
let sentOk = ctx.reqChannel.trySend(req)
if not sentOk:
deallocShared(req)
return err("Couldn't send a request to the codex thread: " & $req[])
# Notify the Codex thread that a request is available
let fireSyncRes = ctx.reqSignal.fireSync()
if fireSyncRes.isErr():
deallocShared(req)
return err("failed fireSync: " & $fireSyncRes.error)
if fireSyncRes.get() == false:
deallocShared(req)
return err("Couldn't fireSync in time")
# Wait until the Codex Thread properly received the request
let res = ctx.reqReceivedSignal.waitSync(timeout)
if res.isErr():
deallocShared(req)
return err("Couldn't receive reqReceivedSignal signal")
## Notice that in case of "ok", the deallocShared(req) is performed by the Codex Thread in the
## process proc. See the 'codex_thread_request.nim' module for more details.
ok()
proc runCodex(ctx: ptr CodexContext) {.async.} =
var codex: CodexServer
while true:
# Wait until a request is available
await ctx.reqSignal.wait()
# If codex_destroy was called, exit the loop
if ctx.running.load == false:
break
var request: ptr CodexThreadRequest
# Pop a request from the channel
let recvOk = ctx.reqChannel.tryRecv(request)
if not recvOk:
error "codex thread could not receive a request"
continue
# Dispatch the request to be processed asynchronously
asyncSpawn CodexThreadRequest.process(request, addr codex)
# Notify the main thread that we picked up the request
let fireRes = ctx.reqReceivedSignal.fireSync()
if fireRes.isErr():
error "could not fireSync back to requester thread", error = fireRes.error
proc run(ctx: ptr CodexContext) {.thread.} =
waitFor runCodex(ctx)
proc createCodexContext*(): Result[ptr CodexContext, string] =
## This proc is called from the main thread and it creates
## the Codex working thread.
# Allocates a CodexContext in shared memory (for the main thread)
var ctx = createShared(CodexContext, 1)
# This signal is used by the main side to wake the Codex thread
# when a new request is enqueued.
ctx.reqSignal = ThreadSignalPtr.new().valueOr:
return err("couldn't create reqSignal ThreadSignalPtr")
# Used to let the caller know that the Codex thread has
# acknowledged / picked up a request (like a handshake).
ctx.reqReceivedSignal = ThreadSignalPtr.new().valueOr:
return err("couldn't create reqReceivedSignal ThreadSignalPtr")
# Protects shared state inside CodexContext
ctx.lock.initLock()
# Codex thread will loop until codex_destroy is called
ctx.running.store(true)
try:
createThread(ctx.thread, run, ctx)
except ValueError, ResourceExhaustedError:
freeShared(ctx)
return err("failed to create the Codex thread: " & getCurrentExceptionMsg())
return ok(ctx)
proc destroyCodexContext*(ctx: ptr CodexContext): Result[void, string] =
# Signal the Codex thread to stop
ctx.running.store(false)
# Wake the worker up if it's waiting
let signaledOnTime = ctx.reqSignal.fireSync().valueOr:
return err("error in destroyCodexContext: " & $error)
if not signaledOnTime:
return err("failed to signal reqSignal on time in destroyCodexContext")
# Wait for the thread to finish
joinThread(ctx.thread)
# Clean up
ctx.lock.deinitLock()
?ctx.reqSignal.close()
?ctx.reqReceivedSignal.close()
freeShared(ctx)
return ok()

View File

@ -0,0 +1,81 @@
## This file contains the base message request type that will be handled.
## The requests are created by the main thread and processed by
## the Codex Thread.
import std/json
import results
import chronos
import ../ffi_types
import ./requests/node_lifecycle_request
from ../../codex/codex import CodexServer
type RequestType* {.pure.} = enum
LIFECYCLE
type CodexThreadRequest* = object
reqType: RequestType
# Request payloed
reqContent: pointer
# Callback to notify the client thread of the result
callback: CodexCallback
# Custom state attached by the client to the request,
# returned when its callback is invoked.
userData: pointer
proc createShared*(
T: type CodexThreadRequest,
reqType: RequestType,
reqContent: pointer,
callback: CodexCallback,
userData: pointer,
): ptr type T =
var ret = createShared(T)
ret[].reqType = reqType
ret[].reqContent = reqContent
ret[].callback = callback
ret[].userData = userData
return ret
proc handleRes[T: string | void](
res: Result[T, string], request: ptr CodexThreadRequest
) =
## Handles the Result responses, which can either be Result[string, string] or
## Result[void, string].
defer:
deallocShared(request)
if res.isErr():
foreignThreadGc:
let msg = "libcodex error: handleRes fireSyncRes error: " & $res.error
request[].callback(
RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData
)
return
foreignThreadGc:
var msg: cstring = ""
when T is string:
msg = res.get().cstring()
request[].callback(
RET_OK, unsafeAddr msg[0], cast[csize_t](len(msg)), request[].userData
)
return
proc process*(
T: type CodexThreadRequest, request: ptr CodexThreadRequest, codex: ptr CodexServer
) {.async.} =
## Processes the request in the Codex thread.
## Dispatch to the appropriate request handler based on reqType.
let retFut =
case request[].reqType
of LIFECYCLE:
cast[ptr NodeLifecycleRequest](request[].reqContent).process(codex)
handleRes(await retFut, request)
proc `$`*(self: CodexThreadRequest): string =
return $self.reqType

View File

@ -0,0 +1,165 @@
## This file contains the lifecycle request type that will be handled.
import std/[options, json, strutils, net, os]
import confutils/defs
import codexdht/discv5/spr
import stew/shims/parseutils
import contractabi/address
import chronos
import chronicles
import results
import confutils
import confutils/std/net
import libp2p
import json_serialization
import json_serialization/std/[options, net]
import ../../alloc
import ../../../codex/conf
import ../../../codex/utils
import ../../../codex/utils/[keyutils, fileutils]
from ../../../codex/codex import CodexServer, new, start, stop
type NodeLifecycleMsgType* = enum
CREATE_NODE
START_NODE
STOP_NODE
proc readValue*[T: InputFile | InputDir | OutPath | OutDir | OutFile](
r: var JsonReader, val: var T
) =
val = T(r.readValue(string))
proc readValue*(r: var JsonReader, val: var MultiAddress) =
val = MultiAddress.init(r.readValue(string)).get()
proc readValue*(r: var JsonReader, val: var NatConfig) =
let res = NatConfig.parse(r.readValue(string))
if res.isErr:
raise
newException(SerializationError, "Cannot parse the NAT config: " & res.error())
val = res.get()
proc readValue*(r: var JsonReader, val: var SignedPeerRecord) =
let res = SignedPeerRecord.parse(r.readValue(string))
if res.isErr:
raise
newException(SerializationError, "Cannot parse the signed peer: " & res.error())
val = res.get()
proc readValue*(r: var JsonReader, val: var ThreadCount) =
let res = ThreadCount.parse(r.readValue(string))
if res.isErr:
raise
newException(SerializationError, "Cannot parse the thread count: " & res.error())
val = res.get()
proc readValue*(r: var JsonReader, val: var NBytes) =
let res = NBytes.parse(r.readValue(string))
if res.isErr:
raise newException(SerializationError, "Cannot parse the NBytes: " & res.error())
val = res.get()
proc readValue*(r: var JsonReader, val: var Duration) =
var dur: Duration
let input = r.readValue(string)
let count = parseDuration(input, dur)
if count == 0:
raise newException(SerializationError, "Cannot parse the duration: " & input)
val = dur
proc readValue*(r: var JsonReader, val: var EthAddress) =
val = EthAddress.init(r.readValue(string)).get()
type NodeLifecycleRequest* = object
operation: NodeLifecycleMsgType
configJson: cstring
proc createShared*(
T: type NodeLifecycleRequest, op: NodeLifecycleMsgType, configJson: cstring = ""
): ptr type T =
var ret = createShared(T)
ret[].operation = op
ret[].configJson = configJson.alloc()
return ret
proc destroyShared(self: ptr NodeLifecycleRequest) =
deallocShared(self[].configJson)
deallocShared(self)
proc createCodex(configJson: cstring): Future[Result[CodexServer, string]] {.async.} =
var conf = CodexConf.load(
version = codexFullVersion,
envVarsPrefix = "codex",
cmdLine = @[],
secondarySources = proc(
config: CodexConf, sources: auto
) {.gcsafe, raises: [ConfigurationError].} =
if configJson.len > 0:
sources.addConfigFileContent(Json, $(configJson))
,
)
conf.setupLogging()
conf.setupMetrics()
if not (checkAndCreateDataDir((conf.dataDir).string)):
# We are unable to access/create data folder or data folder's
# permissions are insecure.
return err(
"Unable to access/create data folder or data folder's permissions are insecure."
)
if not (checkAndCreateDataDir((conf.dataDir / "repo"))):
# We are unable to access/create data folder or data folder's
# permissions are insecure.
return err(
"Unable to access/create data folder or data folder's permissions are insecure."
)
debug "Repo dir initialized", dir = conf.dataDir / "repo"
let keyPath =
if isAbsolute(conf.netPrivKeyFile):
conf.netPrivKeyFile
else:
conf.dataDir / conf.netPrivKeyFile
let privateKey = setupKey(keyPath).expect("Should setup private key!")
let server =
try:
CodexServer.new(conf, privateKey)
except Exception as exc:
return err("Failed to start Codex: " & exc.msg)
return ok(server)
proc process*(
self: ptr NodeLifecycleRequest, codex: ptr CodexServer
): Future[Result[string, string]] {.async.} =
defer:
destroyShared(self)
case self.operation
of CREATE_NODE:
codex[] = (
await createCodex(
self.configJson # , self.appCallbacks
)
).valueOr:
error "CREATE_NODE failed", error = error
return err($error)
of START_NODE:
try:
await codex[].start()
except Exception as e:
error "START_NODE failed", error = e.msg
return err(e.msg)
of STOP_NODE:
try:
await codex[].stop()
except Exception as e:
error "STOP_NODE failed", error = e.msg
return err(e.msg)
return ok("")

View File

@ -0,0 +1,14 @@
# JSON Event definition
#
# This file defines de JsonEvent type, which serves as the base
# for all event types in the library
#
# Reference specification:
# https://github.com/vacp2p/rfc/blob/master/content/docs/rfcs/36/README.md#jsonsignal-type
type JsonEvent* = ref object of RootObj
eventType* {.requiresInit.}: string
method `$`*(jsonEvent: JsonEvent): string {.base.} =
discard
# All events should implement this

35
library/ffi_types.nim Normal file
View File

@ -0,0 +1,35 @@
# FFI Types and Utilities
#
# This file defines the core types and utilities for the library's foreign
# function interface (FFI), enabling interoperability with external code.
################################################################################
### Exported types
type CodexCallback* = proc(
callerRet: cint, msg: ptr cchar, len: csize_t, userData: pointer
) {.cdecl, gcsafe, raises: [].}
const RET_OK*: cint = 0
const RET_ERR*: cint = 1
const RET_MISSING_CALLBACK*: cint = 2
### End of exported types
################################################################################
################################################################################
### FFI utils
template foreignThreadGc*(body: untyped) =
when declared(setupForeignThreadGc):
setupForeignThreadGc()
body
when declared(tearDownForeignThreadGc):
tearDownForeignThreadGc()
type onDone* = proc()
### End of FFI utils
################################################################################

54
library/libcodex.h Normal file
View File

@ -0,0 +1,54 @@
/**
* libcodex.h - C Interface for Example Library
*
* This header provides the public API for libcodex
*
* To see the auto-generated header by Nim, run `make libcodex` from the
* repository root. The generated file will be created at:
* nimcache/release/libcodex/libcodex.h
*/
#ifndef __libcodex__
#define __libcodex__
#include <stddef.h>
#include <stdint.h>
// The possible returned values for the functions that return int
#define RET_OK 0
#define RET_ERR 1
#define RET_MISSING_CALLBACK 2
#ifdef __cplusplus
extern "C" {
#endif
typedef void (*CodexCallback) (int callerRet, const char* msg, size_t len, void* userData);
void* codex_new(
const char* configJson,
CodexCallback callback,
void* userData);
int codex_start(void* ctx,
CodexCallback callback,
void* userData);
int codex_stop(void* ctx,
CodexCallback callback,
void* userData);
// Destroys an instance of a codex node created with codex_new
int codex_destroy(void* ctx,
CodexCallback callback,
void* userData);
void codex_set_event_callback(void* ctx,
CodexCallback callback,
void* userData);
#ifdef __cplusplus
}
#endif
#endif /* __libcodex__ */

165
library/libcodex.nim Normal file
View File

@ -0,0 +1,165 @@
# libcodex.nim - C-exported interface for the Codex shared library
#
# This file implements the public C API for libcodex.
# It acts as the bridge between C programs and the internal Nim implementation.
#
# This file defines:
# - Initialization logic for the Nim runtime (once per process)
# - Thread-safe exported procs callable from C
# - Callback registration and invocation for asynchronous communication
# cdecl is C declaration calling convention.
# Its the standard way C compilers expect functions to behave:
# 1- Caller cleans up the stack after the call
# 2- Symbol names are exported in a predictable way
# In other termes, it is a glue that makes Nim functions callable as normal C functions.
{.pragma: exported, exportc, cdecl, raises: [].}
{.pragma: callback, cdecl, raises: [], gcsafe.}
# Ensure code is position-independent so it can be built into a shared library (.so).
# In other terms, the code that can run no matter where its placed in memory.
{.passc: "-fPIC".}
when defined(linux):
# Define the canonical name for this library
{.passl: "-Wl,-soname,libcodex.so".}
import std/[atomics]
import chronicles
import chronos
import ./codex_context
import ./codex_thread_requests/codex_thread_request
import ./codex_thread_requests/requests/node_lifecycle_request
import ./ffi_types
template checkLibcodexParams*(
ctx: ptr CodexContext, callback: CodexCallback, userData: pointer
) =
if not isNil(ctx):
ctx[].userData = userData
if isNil(callback):
return RET_MISSING_CALLBACK
proc handleRequest(
ctx: ptr CodexContext,
requestType: RequestType,
content: pointer,
callback: CodexCallback,
userData: pointer,
): cint =
codex_context.sendRequestToCodexThread(ctx, requestType, content, callback, userData).isOkOr:
let msg = "libcodex error: " & $error
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
return RET_ERR
return RET_OK
# From Nim doc:
# "the C targets require you to initialize Nim's internals, which is done calling a NimMain function."
# "The name NimMain can be influenced via the --nimMainPrefix:prefix switch."
# "Use --nimMainPrefix:MyLib and the function to call is named MyLibNimMain."
proc libcodexNimMain() {.importc.}
# Atomic flag to prevent multiple initializations
var initialized: Atomic[bool]
if defined(android):
# Redirect chronicles to Android System logs
when compiles(defaultChroniclesStream.outputs[0].writer):
defaultChroniclesStream.outputs[0].writer = proc(
logLevel: LogLevel, msg: LogOutputStr
) {.raises: [].} =
echo logLevel, msg
# Initializes the Nim runtime and foreign-thread GC
proc initializeLibrary() {.exported.} =
if not initialized.exchange(true):
## Every Nim library must call `<prefix>NimMain()` once
libcodexNimMain()
when declared(setupForeignThreadGc):
setupForeignThreadGc()
when declared(nimGC_setStackBottom):
var locals {.volatile, noinit.}: pointer
locals = addr(locals)
nimGC_setStackBottom(locals)
proc codex_new(
configJson: cstring, callback: CodexCallback, userData: pointer
): pointer {.dynlib, exportc, cdecl.} =
initializeLibrary()
if isNil(callback):
error "Missing callback in codex_new"
return nil
var ctx = codex_context.createCodexContext().valueOr:
let msg = "Error in createCodexContext: " & $error
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
return nil
ctx.userData = userData
let retCode = handleRequest(
ctx,
RequestType.LIFECYCLE,
NodeLifecycleRequest.createShared(
NodeLifecycleMsgType.CREATE_NODE, configJson # , appCallbacks
),
callback,
userData,
)
if retCode == RET_ERR:
return nil
return ctx
proc codex_destroy(
ctx: ptr CodexContext, callback: COdexCallback, userData: pointer
): cint {.dynlib, exportc.} =
initializeLibrary()
checkLibcodexParams(ctx, callback, userData)
codex_context.destroyCodexContext(ctx).isOkOr:
let msg = "libcodex error: " & $error
callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData)
return RET_ERR
## always need to invoke the callback although we don't retrieve value to the caller
callback(RET_OK, nil, 0, userData)
return RET_OK
proc codex_start(
ctx: ptr CodexContext, callback: CodexCallback, userData: pointer
): cint {.dynlib, exportc.} =
initializeLibrary()
checkLibcodexParams(ctx, callback, userData)
handleRequest(
ctx,
RequestType.LIFECYCLE,
NodeLifecycleRequest.createShared(NodeLifecycleMsgType.START_NODE),
callback,
userData,
)
proc codex_stop(
ctx: ptr CodexContext, callback: CodexCallback, userData: pointer
): cint {.dynlib, exportc.} =
initializeLibrary()
checkLibcodexParams(ctx, callback, userData)
handleRequest(
ctx,
RequestType.LIFECYCLE,
NodeLifecycleRequest.createShared(NodeLifecycleMsgType.STOP_NODE),
callback,
userData,
)
proc codex_set_event_callback(
ctx: ptr CodexContext, callback: CodexCallback, userData: pointer
) {.dynlib, exportc.} =
initializeLibrary()
ctx[].eventCallback = cast[pointer](callback)
ctx[].eventUserData = userData