From 5cc5cfe82ec684217f7c7b28101a14371229276c Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 16 Feb 2024 14:14:43 +0530 Subject: [PATCH 01/24] chore: add C/C++ gitignore --- .gitignore | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..93a4daf --- /dev/null +++ b/.gitignore @@ -0,0 +1,57 @@ +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +# ignore vscode files +.vscode/ + + From d3463be25da1c939c35cfb06d8b39b501a42858f Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 16 Feb 2024 14:16:12 +0530 Subject: [PATCH 02/24] chore: draft makefile to compile c++ code --- cpp/Makefile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 cpp/Makefile diff --git a/cpp/Makefile b/cpp/Makefile new file mode 100644 index 0000000..d90464a --- /dev/null +++ b/cpp/Makefile @@ -0,0 +1,19 @@ +# Build a shared library of negentropy + +#TODO: Need to add compilation flags based on OS +INCS = -I./ -I/opt/homebrew/include/ +TARGET = libnegentropy.so + +.PHONY: all clean + +all: negentropy-cpp shared-lib + +shared-lib: negentropy.o + g++ -o $(TARGET) negentropy.o -shared + +negentropy-cpp: + g++ negentropy.h --std=c++20 -o negentropy.o $(INCS) + + +clean: + rm -f negentropy.o \ No newline at end of file From d8d171baa2a96a2db0f9e051325993d9abde4f11 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 16 Feb 2024 15:15:59 +0530 Subject: [PATCH 03/24] feat: add C wrapper and makefile --- cpp/Makefile | 12 ++++++--- cpp/negentropy_wrapper.h | 56 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 cpp/negentropy_wrapper.h diff --git a/cpp/Makefile b/cpp/Makefile index d90464a..c37e418 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -1,12 +1,16 @@ # Build a shared library of negentropy #TODO: Need to add compilation flags based on OS -INCS = -I./ -I/opt/homebrew/include/ +INCS = -I./ -I/opt/homebrew/include/ TARGET = libnegentropy.so -.PHONY: all clean +.PHONY: all clean install-deps -all: negentropy-cpp shared-lib +all: negentropy-cpp c-wrapper shared-lib + +#TODO: Need to add compilation flags based on OS +install-deps: + brew install lmdb openssl shared-lib: negentropy.o g++ -o $(TARGET) negentropy.o -shared @@ -14,6 +18,8 @@ shared-lib: negentropy.o negentropy-cpp: g++ negentropy.h --std=c++20 -o negentropy.o $(INCS) +c-wrapper: + g++ negentropy_wrapper.h --std=c++20 -o negentropy_wrapper.o $(INCS) -I../test/cpp/lmdbxx/include/ clean: rm -f negentropy.o \ No newline at end of file diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h new file mode 100644 index 0000000..b59d90f --- /dev/null +++ b/cpp/negentropy_wrapper.h @@ -0,0 +1,56 @@ +#pragma once + +#ifdef __cplusplus +#define EXTERNC extern "C" +#else +#define EXTERNC +#endif + +#include "negentropy.h" +#include "negentropy/storage/BTreeLMDB.h" + +//This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. +//TODO: Do error handling by catching exceptions + +EXTERNC void* storage_new(){ + negentropy::storage::BTreeLMDB* storage; + //storage = new negentropy::storage::BTreeLMDB(); + return storage; +} + +EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit){ + negentropy::storage::BTreeLMDB* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); + + Negentropy* ne; + try{ + ne = new Negentropy(*lmdbStorage, frameSizeLimit); + }catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + return ne; +} + +EXTERNC const char* negentropy_initiate(void* negentropy){ + Negentropy* ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::string* output = new std::string(); + try { + *output = ngn_inst->initiate(); + } catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + return output->c_str(); +} + +EXTERNC void negentropy_setinitiator(void* negentropy){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + ngn_inst->setInitiator(); + +} + From d9d750c8c6d12a1147d8dceddefe0e39c5b7a567 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 16 Feb 2024 15:23:50 +0530 Subject: [PATCH 04/24] chore: move lmdbxx submodule under cpp folder --- .gitmodules | 2 +- cpp/Makefile | 6 +++--- {test/cpp => cpp/vendor}/lmdbxx | 0 test/cpp/Makefile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename {test/cpp => cpp/vendor}/lmdbxx (100%) diff --git a/.gitmodules b/.gitmodules index 2834d9b..8b45d4c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,5 +2,5 @@ path = test/cpp/hoytech-cpp url = https://github.com/hoytech/hoytech-cpp.git [submodule "test/cpp/lmdbxx"] - path = test/cpp/lmdbxx + path = cpp/vendor/lmdbxx url = https://github.com/hoytech/lmdbxx.git diff --git a/cpp/Makefile b/cpp/Makefile index c37e418..ae6c627 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -19,7 +19,7 @@ negentropy-cpp: g++ negentropy.h --std=c++20 -o negentropy.o $(INCS) c-wrapper: - g++ negentropy_wrapper.h --std=c++20 -o negentropy_wrapper.o $(INCS) -I../test/cpp/lmdbxx/include/ + g++ negentropy_wrapper.h --std=c++20 -o negentropy_wrapper.o $(INCS) -I./vendor/lmdbxx/include/ -clean: - rm -f negentropy.o \ No newline at end of file +clean: + rm -f negentropy.o diff --git a/test/cpp/lmdbxx b/cpp/vendor/lmdbxx similarity index 100% rename from test/cpp/lmdbxx rename to cpp/vendor/lmdbxx diff --git a/test/cpp/Makefile b/test/cpp/Makefile index db2070e..6b0d5c4 100644 --- a/test/cpp/Makefile +++ b/test/cpp/Makefile @@ -2,7 +2,7 @@ W = -Wall OPT = -g -O2 STD = -std=c++20 CXXFLAGS = $(STD) $(OPT) $(W) -fPIC $(XCXXFLAGS) -INCS = -I../../cpp/ -I./hoytech-cpp/ -Ilmdbxx/include/ +INCS = -I../../cpp/ -I./hoytech-cpp/ -I../cpp/vendor/lmdbxx/include/ #-I/opt/homebrew/include/ -L/opt/homebrew/lib/ DEPS = ../../cpp/negentropy.h ../../cpp/negentropy/* ../../cpp/negentropy/storage/* ../../cpp/negentropy/storage/btree/* From d4e6451d67b931cf86dc7b5ad3b9fe5974e4aadf Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 16 Feb 2024 17:34:49 +0530 Subject: [PATCH 05/24] feat: add reconcile methods --- cpp/negentropy_wrapper.h | 78 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index b59d90f..f766da2 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -9,17 +9,35 @@ #include "negentropy.h" #include "negentropy/storage/BTreeLMDB.h" + +#define length(array) ((sizeof(array)) / (sizeof(array[0]))) + //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions -EXTERNC void* storage_new(){ +EXTERNC void* storage_new(const char* db_path, const char* name){ negentropy::storage::BTreeLMDB* storage; +/* + auto env = lmdb::env::create(); + env.set_max_dbs(64); + env.open(db_path, 0); + + lmdb::dbi btreeDbi; + + { + auto txn = lmdb::txn::begin(env); + btreeDbi = negentropy::storage::BTreeLMDB::setupDB(txn, name); + txn.commit(); + } */ + //TODO: Finish constructor //storage = new negentropy::storage::BTreeLMDB(); return storage; } EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit){ + //TODO: Make these typecasts into macros?? negentropy::storage::BTreeLMDB* lmdbStorage; + //TODO: reinterpret cast is risky, need to use more safe type conversion. lmdbStorage = reinterpret_cast(storage); Negentropy* ne; @@ -54,3 +72,61 @@ EXTERNC void negentropy_setinitiator(void* negentropy){ } + +EXTERNC bool storage_insert(void* storage, uint64_t createdAt, const char* id){ + negentropy::storage::BTreeLMDB* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); + + //TODO: Error handling. Is it required? + //How does out of memory get handled? + return lmdbStorage->insert(createdAt, id); +} + + +EXTERNC bool storage_erase(void* storage, uint64_t createdAt, const char* id){ + negentropy::storage::BTreeLMDB* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); + + //TODO: Error handling + return lmdbStorage->erase(createdAt, id); +} + + +EXTERNC const char* reconcile(void* negentropy, const char* query){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::string* output = new std::string(); + try { + *output = ngn_inst->reconcile(std::string_view(query)); + } catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + return output->c_str(); +} + +EXTERNC const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], + uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::optional* output; + std::vector haveIds(have_ids, have_ids+have_ids_len); + + std::vector needIds(need_ids, need_ids+need_ids_len); + + try { + *output = ngn_inst->reconcile(std::string_view(query), haveIds, needIds); + } catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + if (output->has_value()) { + //TODO: Figure out diff between error and this. + return NULL; + }else { + + return output->value().c_str(); + } +} \ No newline at end of file From 27a7a64e095c52517112c88e4d9c0bf450cbd5a5 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Fri, 16 Feb 2024 17:35:02 +0530 Subject: [PATCH 06/24] chore: in progress Makefile --- cpp/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/Makefile b/cpp/Makefile index ae6c627..c561c11 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -13,7 +13,7 @@ install-deps: brew install lmdb openssl shared-lib: negentropy.o - g++ -o $(TARGET) negentropy.o -shared + g++ -shared -fPIC -o $(TARGET) negentropy.o negentropy_wrapper.o negentropy-cpp: g++ negentropy.h --std=c++20 -o negentropy.o $(INCS) @@ -22,4 +22,4 @@ c-wrapper: g++ negentropy_wrapper.h --std=c++20 -o negentropy_wrapper.o $(INCS) -I./vendor/lmdbxx/include/ clean: - rm -f negentropy.o + rm -f negentropy.o negentropy_wrapper.o libnegentropy.so From b3e3904356bb0312f512e873d01e4a7f8c1db5a7 Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Fri, 16 Feb 2024 15:25:39 +0100 Subject: [PATCH 07/24] Some refactoring to allow the creation of shared library --- cpp/Makefile | 16 +++-- cpp/negentropy_wrapper.c | 121 +++++++++++++++++++++++++++++++++++++ cpp/negentropy_wrapper.h | 125 ++++----------------------------------- 3 files changed, 140 insertions(+), 122 deletions(-) create mode 100644 cpp/negentropy_wrapper.c diff --git a/cpp/Makefile b/cpp/Makefile index c561c11..c40517c 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -6,20 +6,18 @@ TARGET = libnegentropy.so .PHONY: all clean install-deps -all: negentropy-cpp c-wrapper shared-lib +all: precompiled-header shared-lib #TODO: Need to add compilation flags based on OS install-deps: brew install lmdb openssl -shared-lib: negentropy.o - g++ -shared -fPIC -o $(TARGET) negentropy.o negentropy_wrapper.o +# Generate 'negentropy.h.gch' +precompiled-header: + g++ --std=c++20 -Wall -fexceptions -g negentropy.h -I./ -I./vendor/lmdbxx/include/ -negentropy-cpp: - g++ negentropy.h --std=c++20 -o negentropy.o $(INCS) - -c-wrapper: - g++ negentropy_wrapper.h --std=c++20 -o negentropy_wrapper.o $(INCS) -I./vendor/lmdbxx/include/ +shared-lib: negentropy.h.gch + g++ --std=c++20 -I./ -I./vendor/lmdbxx/include/ -shared -fPIC -o $(TARGET) negentropy_wrapper.c clean: - rm -f negentropy.o negentropy_wrapper.o libnegentropy.so + rm -f $(TARGET) negentropy.h.gch libnegentropy.so diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c new file mode 100644 index 0000000..c2d18a1 --- /dev/null +++ b/cpp/negentropy_wrapper.c @@ -0,0 +1,121 @@ + +#include "negentropy_wrapper.h" + +//This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. +//TODO: Do error handling by catching exceptions + +void* storage_new(const char* db_path, const char* name){ + negentropy::storage::BTreeLMDB* storage; +/* + auto env = lmdb::env::create(); + env.set_max_dbs(64); + env.open(db_path, 0); + + lmdb::dbi btreeDbi; + + { + auto txn = lmdb::txn::begin(env); + btreeDbi = negentropy::storage::BTreeLMDB::setupDB(txn, name); + txn.commit(); + } */ + //TODO: Finish constructor + //storage = new negentropy::storage::BTreeLMDB(); + return storage; +} + +void* negentropy_new(void* storage, uint64_t frameSizeLimit){ + //TODO: Make these typecasts into macros?? + negentropy::storage::BTreeLMDB* lmdbStorage; + //TODO: reinterpret cast is risky, need to use more safe type conversion. + lmdbStorage = reinterpret_cast(storage); + + Negentropy* ne; + try{ + ne = new Negentropy(*lmdbStorage, frameSizeLimit); + }catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + return ne; +} + +const char* negentropy_initiate(void* negentropy){ + Negentropy* ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::string* output = new std::string(); + try { + *output = ngn_inst->initiate(); + } catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + return output->c_str(); +} + +void negentropy_setinitiator(void* negentropy){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + ngn_inst->setInitiator(); + +} + + +bool storage_insert(void* storage, uint64_t createdAt, const char* id){ + negentropy::storage::BTreeLMDB* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); + + //TODO: Error handling. Is it required? + //How does out of memory get handled? + return lmdbStorage->insert(createdAt, id); +} + + +bool storage_erase(void* storage, uint64_t createdAt, const char* id){ + negentropy::storage::BTreeLMDB* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); + + //TODO: Error handling + return lmdbStorage->erase(createdAt, id); +} + + +const char* reconcile(void* negentropy, const char* query){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::string* output = new std::string(); + try { + *output = ngn_inst->reconcile(std::string_view(query)); + } catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + return output->c_str(); +} + +const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], + uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::optional* output; + std::vector haveIds(have_ids, have_ids+have_ids_len); + + std::vector needIds(need_ids, need_ids+need_ids_len); + + try { + *output = ngn_inst->reconcile(std::string_view(query), haveIds, needIds); + } catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + if (output->has_value()) { + //TODO: Figure out diff between error and this. + return NULL; + }else { + + return output->value().c_str(); + } +} \ No newline at end of file diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index f766da2..e18ab5f 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -1,132 +1,31 @@ -#pragma once -#ifdef __cplusplus -#define EXTERNC extern "C" -#else -#define EXTERNC -#endif +#ifndef _NEGENTROPY_WRAPPER_H +#define _NEGENTROPY_WRAPPER_H #include "negentropy.h" #include "negentropy/storage/BTreeLMDB.h" - #define length(array) ((sizeof(array)) / (sizeof(array[0]))) //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions -EXTERNC void* storage_new(const char* db_path, const char* name){ - negentropy::storage::BTreeLMDB* storage; -/* - auto env = lmdb::env::create(); - env.set_max_dbs(64); - env.open(db_path, 0); +void* storage_new(const char* db_path, const char* name); - lmdb::dbi btreeDbi; +void* negentropy_new(void* storage, uint64_t frameSizeLimit); - { - auto txn = lmdb::txn::begin(env); - btreeDbi = negentropy::storage::BTreeLMDB::setupDB(txn, name); - txn.commit(); - } */ - //TODO: Finish constructor - //storage = new negentropy::storage::BTreeLMDB(); - return storage; -} +const char* negentropy_initiate(void* negentropy); -EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit){ - //TODO: Make these typecasts into macros?? - negentropy::storage::BTreeLMDB* lmdbStorage; - //TODO: reinterpret cast is risky, need to use more safe type conversion. - lmdbStorage = reinterpret_cast(storage); +void negentropy_setinitiator(void* negentropy); - Negentropy* ne; - try{ - ne = new Negentropy(*lmdbStorage, frameSizeLimit); - }catch(negentropy::err e){ - //TODO:Find a way to return this error - return NULL; - } - return ne; -} +bool storage_insert(void* storage, uint64_t createdAt, const char* id); -EXTERNC const char* negentropy_initiate(void* negentropy){ - Negentropy* ngn_inst; - ngn_inst = reinterpret_cast*>(negentropy); +bool storage_erase(void* storage, uint64_t createdAt, const char* id); - std::string* output = new std::string(); - try { - *output = ngn_inst->initiate(); - } catch(negentropy::err e){ - //TODO:Find a way to return this error - return NULL; - } - return output->c_str(); -} +const char* reconcile(void* negentropy, const char* query); -EXTERNC void negentropy_setinitiator(void* negentropy){ - Negentropy *ngn_inst; - ngn_inst = reinterpret_cast*>(negentropy); +const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], + uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len); - ngn_inst->setInitiator(); +#endif -} - - -EXTERNC bool storage_insert(void* storage, uint64_t createdAt, const char* id){ - negentropy::storage::BTreeLMDB* lmdbStorage; - lmdbStorage = reinterpret_cast(storage); - - //TODO: Error handling. Is it required? - //How does out of memory get handled? - return lmdbStorage->insert(createdAt, id); -} - - -EXTERNC bool storage_erase(void* storage, uint64_t createdAt, const char* id){ - negentropy::storage::BTreeLMDB* lmdbStorage; - lmdbStorage = reinterpret_cast(storage); - - //TODO: Error handling - return lmdbStorage->erase(createdAt, id); -} - - -EXTERNC const char* reconcile(void* negentropy, const char* query){ - Negentropy *ngn_inst; - ngn_inst = reinterpret_cast*>(negentropy); - - std::string* output = new std::string(); - try { - *output = ngn_inst->reconcile(std::string_view(query)); - } catch(negentropy::err e){ - //TODO:Find a way to return this error - return NULL; - } - return output->c_str(); -} - -EXTERNC const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], - uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len){ - Negentropy *ngn_inst; - ngn_inst = reinterpret_cast*>(negentropy); - - std::optional* output; - std::vector haveIds(have_ids, have_ids+have_ids_len); - - std::vector needIds(need_ids, need_ids+need_ids_len); - - try { - *output = ngn_inst->reconcile(std::string_view(query), haveIds, needIds); - } catch(negentropy::err e){ - //TODO:Find a way to return this error - return NULL; - } - if (output->has_value()) { - //TODO: Figure out diff between error and this. - return NULL; - }else { - - return output->value().c_str(); - } -} \ No newline at end of file From ceaf285f39f55a3cb68a2497b51ad601b489375f Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Sat, 17 Feb 2024 07:00:02 +0530 Subject: [PATCH 08/24] chore: fix for mac compilation --- cpp/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/Makefile b/cpp/Makefile index c40517c..9009576 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -1,7 +1,7 @@ # Build a shared library of negentropy #TODO: Need to add compilation flags based on OS -INCS = -I./ -I/opt/homebrew/include/ +INCS = -I./ -I/opt/homebrew/include/ -I./vendor/lmdbxx/include/ TARGET = libnegentropy.so .PHONY: all clean install-deps @@ -14,10 +14,10 @@ install-deps: # Generate 'negentropy.h.gch' precompiled-header: - g++ --std=c++20 -Wall -fexceptions -g negentropy.h -I./ -I./vendor/lmdbxx/include/ + g++ --std=c++20 -Wall -fexceptions -g negentropy.h $(INCS) shared-lib: negentropy.h.gch - g++ --std=c++20 -I./ -I./vendor/lmdbxx/include/ -shared -fPIC -o $(TARGET) negentropy_wrapper.c + g++ --std=c++20 $(INCS) -shared -fPIC -o $(TARGET) negentropy_wrapper.c clean: rm -f $(TARGET) negentropy.h.gch libnegentropy.so From 07a159d82f6372b2755c7176814953040ce8ee53 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Sat, 17 Feb 2024 07:18:22 +0530 Subject: [PATCH 09/24] feat: use BTreeMem to start with and also make C wrapper extern --- cpp/negentropy_wrapper.c | 38 +++++++++++++++++++------------------- cpp/negentropy_wrapper.h | 18 +++++++++--------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index c2d18a1..d744fea 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -5,7 +5,7 @@ //TODO: Do error handling by catching exceptions void* storage_new(const char* db_path, const char* name){ - negentropy::storage::BTreeLMDB* storage; + negentropy::storage::BTreeMem* storage; /* auto env = lmdb::env::create(); env.set_max_dbs(64); @@ -15,23 +15,23 @@ void* storage_new(const char* db_path, const char* name){ { auto txn = lmdb::txn::begin(env); - btreeDbi = negentropy::storage::BTreeLMDB::setupDB(txn, name); + btreeDbi = negentropy::storage::BTreeMem::setupDB(txn, name); txn.commit(); } */ //TODO: Finish constructor - //storage = new negentropy::storage::BTreeLMDB(); + storage = new negentropy::storage::BTreeMem(); return storage; } void* negentropy_new(void* storage, uint64_t frameSizeLimit){ //TODO: Make these typecasts into macros?? - negentropy::storage::BTreeLMDB* lmdbStorage; + negentropy::storage::BTreeMem* lmdbStorage; //TODO: reinterpret cast is risky, need to use more safe type conversion. - lmdbStorage = reinterpret_cast(storage); + lmdbStorage = reinterpret_cast(storage); - Negentropy* ne; + Negentropy* ne; try{ - ne = new Negentropy(*lmdbStorage, frameSizeLimit); + ne = new Negentropy(*lmdbStorage, frameSizeLimit); }catch(negentropy::err e){ //TODO:Find a way to return this error return NULL; @@ -40,8 +40,8 @@ void* negentropy_new(void* storage, uint64_t frameSizeLimit){ } const char* negentropy_initiate(void* negentropy){ - Negentropy* ngn_inst; - ngn_inst = reinterpret_cast*>(negentropy); + Negentropy* ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); std::string* output = new std::string(); try { @@ -54,8 +54,8 @@ const char* negentropy_initiate(void* negentropy){ } void negentropy_setinitiator(void* negentropy){ - Negentropy *ngn_inst; - ngn_inst = reinterpret_cast*>(negentropy); + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); ngn_inst->setInitiator(); @@ -63,8 +63,8 @@ void negentropy_setinitiator(void* negentropy){ bool storage_insert(void* storage, uint64_t createdAt, const char* id){ - negentropy::storage::BTreeLMDB* lmdbStorage; - lmdbStorage = reinterpret_cast(storage); + negentropy::storage::BTreeMem* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); //TODO: Error handling. Is it required? //How does out of memory get handled? @@ -73,8 +73,8 @@ bool storage_insert(void* storage, uint64_t createdAt, const char* id){ bool storage_erase(void* storage, uint64_t createdAt, const char* id){ - negentropy::storage::BTreeLMDB* lmdbStorage; - lmdbStorage = reinterpret_cast(storage); + negentropy::storage::BTreeMem* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); //TODO: Error handling return lmdbStorage->erase(createdAt, id); @@ -82,8 +82,8 @@ bool storage_erase(void* storage, uint64_t createdAt, const char* id){ const char* reconcile(void* negentropy, const char* query){ - Negentropy *ngn_inst; - ngn_inst = reinterpret_cast*>(negentropy); + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); std::string* output = new std::string(); try { @@ -97,8 +97,8 @@ const char* reconcile(void* negentropy, const char* query){ const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len){ - Negentropy *ngn_inst; - ngn_inst = reinterpret_cast*>(negentropy); + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); std::optional* output; std::vector haveIds(have_ids, have_ids+have_ids_len); diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index e18ab5f..d145025 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -3,28 +3,28 @@ #define _NEGENTROPY_WRAPPER_H #include "negentropy.h" -#include "negentropy/storage/BTreeLMDB.h" +#include "negentropy/storage/BTreeMem.h" #define length(array) ((sizeof(array)) / (sizeof(array[0]))) //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions -void* storage_new(const char* db_path, const char* name); +extern "C" void* storage_new(const char* db_path, const char* name); -void* negentropy_new(void* storage, uint64_t frameSizeLimit); +extern "C" void* negentropy_new(void* storage, uint64_t frameSizeLimit); -const char* negentropy_initiate(void* negentropy); +extern "C" const char* negentropy_initiate(void* negentropy); -void negentropy_setinitiator(void* negentropy); +extern "C" void negentropy_setinitiator(void* negentropy); -bool storage_insert(void* storage, uint64_t createdAt, const char* id); +extern "C" bool storage_insert(void* storage, uint64_t createdAt, const char* id); -bool storage_erase(void* storage, uint64_t createdAt, const char* id); +extern "C" bool storage_erase(void* storage, uint64_t createdAt, const char* id); -const char* reconcile(void* negentropy, const char* query); +extern "C" const char* reconcile(void* negentropy, const char* query); -const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], +extern "C" const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len); #endif From 68ca46408e9c2767e4a43e238ba939452a66dfc6 Mon Sep 17 00:00:00 2001 From: Prem Prathi Date: Sat, 17 Feb 2024 06:35:13 +0000 Subject: [PATCH 10/24] chore: move header inclusions to c file --- cpp/negentropy_wrapper.c | 3 ++- cpp/negentropy_wrapper.h | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index d744fea..32e2b1f 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -1,4 +1,5 @@ - +#include "negentropy.h" +#include "negentropy/storage/BTreeMem.h" #include "negentropy_wrapper.h" //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index d145025..9ee90e6 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -2,10 +2,6 @@ #ifndef _NEGENTROPY_WRAPPER_H #define _NEGENTROPY_WRAPPER_H -#include "negentropy.h" -#include "negentropy/storage/BTreeMem.h" - -#define length(array) ((sizeof(array)) / (sizeof(array[0]))) //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions From e4edec4e5a7b3c09852292a16d98d86a1c3b1459 Mon Sep 17 00:00:00 2001 From: Prem Prathi Date: Sat, 17 Feb 2024 06:36:08 +0000 Subject: [PATCH 11/24] chore: example testing c wrapper --- cpp/example/Makefile | 4 ++++ cpp/example/test.c | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 cpp/example/Makefile create mode 100644 cpp/example/test.c diff --git a/cpp/example/Makefile b/cpp/example/Makefile new file mode 100644 index 0000000..fb38278 --- /dev/null +++ b/cpp/example/Makefile @@ -0,0 +1,4 @@ +.PHONY: all + +all: + g++ --std=c++20 test.c -I../ -lnegentropy -lcrypto -L../ diff --git a/cpp/example/test.c b/cpp/example/test.c new file mode 100644 index 0000000..ae877ec --- /dev/null +++ b/cpp/example/test.c @@ -0,0 +1,22 @@ +#include +#include + +#include "../negentropy_wrapper.h" + +int main(){ + void* st = storage_new("",""); + if(st == NULL){ + perror("failed to create storage"); + } + void* ngn_inst = negentropy_new(st, 153600); + if(ngn_inst == NULL){ + perror("failed to create negentropy instance"); + } + + const char* output = negentropy_initiate(ngn_inst); + if(ngn_inst == NULL){ + perror("failed to initiate negentropy instance"); + } + printf("initiated negentropy successfully with output %s \n", output); + +} \ No newline at end of file From 2f09569d1d4bc708745c12bdb6f33d97bc293709 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Sun, 18 Feb 2024 08:00:37 +0530 Subject: [PATCH 12/24] chore: fix compilation error in macOS --- cpp/Makefile | 2 +- cpp/negentropy_wrapper.c | Bin 3947 -> 16753 bytes 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 cpp/negentropy_wrapper.c diff --git a/cpp/Makefile b/cpp/Makefile index 9009576..b0553fd 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -17,7 +17,7 @@ precompiled-header: g++ --std=c++20 -Wall -fexceptions -g negentropy.h $(INCS) shared-lib: negentropy.h.gch - g++ --std=c++20 $(INCS) -shared -fPIC -o $(TARGET) negentropy_wrapper.c + g++ --std=c++20 $(INCS) -shared -fPIC -o $(TARGET) negentropy_wrapper.c -lcrypto -lssl -L/opt/homebrew/lib/ clean: rm -f $(TARGET) negentropy.h.gch libnegentropy.so diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c old mode 100644 new mode 100755 index 32e2b1fcc882b744f9ccf31aed40df678d78887e..606a50b2934f4c2e2495b7344f38f2a295caa4e1 GIT binary patch literal 16753 zcmeI3O=uHQ6op@YS_E5&B2*DI1*z1OBsMPULK($MgHj_6m?C8`%~&N(W}Hl^k%HY6 zv_(W$Rdgq~P&fV{3PLwkaiK&grXnt+E~JRK60&H$cYc~74KB3H^Wfy=op z?<+r+nuPEO8jYk4q<~L|2@zmRY#~jPWI5TLO0rWkr26XyvRP&@&9ZDL7mRA{sywN> zXMU?)`^C7sg8S+r$?~yy@6qnVai=%QuBw^O+l}z#^u(3Tq$p`K$?fNrAti6&Zws<{HdfDTCjWoxKx!JH2oF zvm?Kw&s#pt3`?=d@VPW=P%mnR8jgiCMOvB9T6ugf|By%F)iCbMITy39gP-Z|*q`69GN4fNIdACwv-5Z&8z5^^Vd&lWw-7f z{rK0#YvsFvo0oSc8hbasd41z&=fM5B?81R>nf*)sr%pV)^x#oSef9m!jQ?Y)l=U7@ zH7t)k8M`>~>E%?f&q5WVYH43Y~i%Tft4FiPC)QU=3(2;)dhuQ&zQ*Qj z2eZ9oU5C=_&sMAV6<0tX1}`uC4s3)3T_LPkAxc~t>u;~#rf?+yr6i*jYs=calOLd9x~R-E-WJ$tE?PC4?F299;8V$3 z+$ROCYbYuv=TPQZ$8@zI^W$R68a(@J2J|;ZlBV89ng%=uvzN2g9PD}4Yl^Jh!Ou`P zWuB(=oTi0DrZG9Yv}b5(sZeJPyUog6B~PD38=}KD8KU}$cI-ROSebLkH5KD3=YGS5 zX+pKG^=&)2rsp)xam(9;TzVhoWH3P|@63-yX$H z<|`X*sM0Ima#cBOWnU1T97-NZ>wkZD48gXfNMsx8csV;!euFfSzde^hu^+ zBdMtG_$R#K6tS;vi1Ri3fQFum{vSFln9_!mRLX`GQmE(8ZCazNlVmV8dgofkp%lzZ zG~|{YTDMpu$mIueS7JcO&Y$TaS;4D5bhBJS(?VH4TP>Vz8L_;|bDAB4U zb=fo05J5;-W@w{2Rue2*=D6Q^^ZU(>cVV~uvPp|rG-><+l#H0GxoKKQs4*%}od>5* z_-$JN8HA=u# zl9HU|LexR(40TXypPcXFNZ8!nQvRZ+cNi}pc7sKI>5`5XPEmAhgi{(A%Z%Zb?YAI< zzfLafC0vunYHHGtJ(pNIiCwLJ6MHBHD#Kt98)%Sp@c>)WPJ7pflfH|ia1CWGR1qWu zgXPT>Qj|=E&i?-(8jITm7MJcRRATXyNZ$%x2VllZPnq?xM|t?CD){|cE7H3ELIp=u zBE#8^(;Vr5_>t7{WH~c4LT9{GU+CkBx=xoWwnOvgAAc@fE1RCa?{65n!rNJmG(E<# znW-n(9gc@ZbNmOVm#{xBSp6*G1Pw~=!g4ZdcXnuI-pMHvu^nDhF#B6+L`+?(+>KVJxVa%66>o>37vxG|GhA z<`v)eG*6pbq~x0o From 89eb17227da64cb3869e55c502460814fb4bd197 Mon Sep 17 00:00:00 2001 From: Prem Prathi Date: Mon, 19 Feb 2024 04:55:15 +0000 Subject: [PATCH 13/24] chore: fix wrongly committed file --- cpp/negentropy_wrapper.c | Bin 16753 -> 3950 bytes 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 cpp/negentropy_wrapper.c diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c old mode 100755 new mode 100644 index 606a50b2934f4c2e2495b7344f38f2a295caa4e1..e3269f84513b28547eb9bf76b6a65f04bb8edd1c GIT binary patch literal 3950 zcmds4O>f&q5WVYH43Y~i%Tft4FiPC)QU=3(2;)dhuQ&zQ*Qj z2eZ9oU5C=_&sMAV6<0ui3|?ON9oPs7x7p{vcw1nnxoFjlK0Cq7IrwC< z7WYX(I~t0L$vKpH)-hcz$icYSvIfuongM;rSklznNYjAGVD@shnu9&hdQGvlJNOyu zrp(inp3}6D$TTKrm-Y-TEfwmlVYgYCtK{jEXhU?^CPP#|(T;t`87p%Rxu$Yl<=k($ zFioh|wZ3f!*YupGIc|BIkW25woD3%DP14P7GPmn-1ykCvl1kaILJIZ#xlL?qa28(#64;c$<>DgVTxD+`RU1ZBP4=i8yAE@ZHH13kCy`lym|#c<}2wtzb+NbLvg-!se+%2H6?KEs748x zN>Y-uT!=bIouLj&?UVCe90{AdTgqSb^bX_Y!)~ysFJ02n!YPW5jc`f>W0^6$vi%lh z@Yl(ey@YGhSWQj(vF8#?C$X#5Z(j2DH=_#{5_9zekR0Y3ZYeic3U#Q@S zN@O_Oahf9?5I>SSo-Ai(hUko!>I;25QP=5G#dc`k{NvAsYh}~Z_kD(uE4-c6NYi5+ zo0)oo-QjpxG{=8%dI|gUg4NF=PSBv_E-WXbc4voX=AE1(5!>N46(gXtuqy_Q^RQ$Q z@bmpw?HIU>odBDgC9R#X6x+G~zQ330E>iO8T+zdK=RV&+62^jxscgsUo_<%IMWal( zZC>$hPxG|7MM}QeK#qEk*!mxi^t+J;ey-eTZ7BWau!uDc<>QR>`B)j^@G&fhF~wLr H(mnn+e?u0R literal 16753 zcmeI3O=uHQ6op@YS_E5&B2*DI1*z1OBsMPULK($MgHj_6m?C8`%~&N(W}Hl^k%HY6 zv_(W$Rdgq~P&fV{3PLwkaiK&grXnt+E~JRK60&H$cYc~74KB3H^Wfy=op z?<+r+nuPEO8jYk4q<~L|2@zmRY#~jPWI5TLO0rWkr26XyvRP&@&9ZDL7mRA{sywN> zXMU?)`^C7sg8S+r$?~yy@6qnVai=%QuBw^O+l}z#^u(3Tq$p`K$?fNrAti6&Zws<{HdfDTCjWoxKx!JH2oF zvm?Kw&s#pt3`?=d@VPW=P%mnR8jgiCMOvB9T6ugf|By%F)iCbMITy39gP-Z|*q`69GN4fNIdACwv-5Z&8z5^^Vd&lWw-7f z{rK0#YvsFvo0oSc8hbasd41z&=fM5B?81R>nf*)sr%pV)^x#oSef9m!jQ?Y)l=U7@ zH7t)k8M`>~>E%? Date: Mon, 19 Feb 2024 15:10:31 +0530 Subject: [PATCH 14/24] fix: protect extern C under ifdef cplusplus --- cpp/negentropy_wrapper.h | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index 9ee90e6..8cc58d2 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -2,25 +2,30 @@ #ifndef _NEGENTROPY_WRAPPER_H #define _NEGENTROPY_WRAPPER_H +#ifdef __cplusplus +#define EXTERNC extern "C" +#else +#define EXTERNC +#endif //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions -extern "C" void* storage_new(const char* db_path, const char* name); +EXTERNC void* storage_new(const char* db_path, const char* name); -extern "C" void* negentropy_new(void* storage, uint64_t frameSizeLimit); +EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit); -extern "C" const char* negentropy_initiate(void* negentropy); +EXTERNC const char* negentropy_initiate(void* negentropy); -extern "C" void negentropy_setinitiator(void* negentropy); +EXTERNC void negentropy_setinitiator(void* negentropy); -extern "C" bool storage_insert(void* storage, uint64_t createdAt, const char* id); +EXTERNC bool storage_insert(void* storage, uint64_t createdAt, const char* id); -extern "C" bool storage_erase(void* storage, uint64_t createdAt, const char* id); +EXTERNC bool storage_erase(void* storage, uint64_t createdAt, const char* id); -extern "C" const char* reconcile(void* negentropy, const char* query); +EXTERNC const char* reconcile(void* negentropy, const char* query); -extern "C" const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], +EXTERNC const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len); #endif From a736a26532e0311402700f35d37cc10088c5f316 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Tue, 20 Feb 2024 17:27:38 +0530 Subject: [PATCH 15/24] fix: update reconcile function logic where out params are passed --- cpp/negentropy_wrapper.c | 30 ++++++++++++++++++++++-------- cpp/negentropy_wrapper.h | 6 +++--- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index e3269f8..5974aac 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -84,13 +84,13 @@ bool storage_erase(void* storage, uint64_t createdAt, const char* id){ } -const char* reconcile(void* negentropy, const char* query){ +const char* reconcile(void* negentropy, const char* query, uint64_t query_len){ Negentropy *ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); std::string* output = new std::string(); try { - *output = ngn_inst->reconcile(std::string_view(query)); + *output = ngn_inst->reconcile(std::string_view(query, query_len)); } catch(negentropy::err e){ //TODO:Find a way to return this error return NULL; @@ -98,18 +98,32 @@ const char* reconcile(void* negentropy, const char* query){ return output->c_str(); } -const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], - uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len){ +char *convert(const std::string & s) +{ + char *pc = new char[s.size()+1]; + std::strcpy(pc, s.c_str()); + return pc; +} + +const char* reconcile_with_ids(void* negentropy, const char* query, uint64_t query_len, char* have_ids[], + uint64_t *have_ids_len, char* need_ids[], uint64_t *need_ids_len){ Negentropy *ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); std::optional* output; - std::vector haveIds(have_ids, have_ids+have_ids_len); - - std::vector needIds(need_ids, need_ids+need_ids_len); + std::vector haveIds; + std::vector needIds; try { - *output = ngn_inst->reconcile(std::string_view(query), haveIds, needIds); + *output = ngn_inst->reconcile(std::string_view(query, query_len), haveIds, needIds); + + *have_ids_len = haveIds.size(); + *need_ids_len = needIds.size(); + //TODO: Optimize to not copy and rather return memory reference. + + std::transform(haveIds.begin(), haveIds.end(), have_ids, convert); + std::transform(needIds.begin(), needIds.end(), need_ids, convert); + } catch(negentropy::err e){ //TODO:Find a way to return this error return NULL; diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index 8cc58d2..68fcff5 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -23,10 +23,10 @@ EXTERNC bool storage_insert(void* storage, uint64_t createdAt, const char* id); EXTERNC bool storage_erase(void* storage, uint64_t createdAt, const char* id); -EXTERNC const char* reconcile(void* negentropy, const char* query); +EXTERNC const char* reconcile(void* negentropy, const char* query, uint64_t query_len); -EXTERNC const char* reconcile_with_ids(void* negentropy, const char* query, const char* have_ids[], - uint64_t have_ids_len, const char* need_ids[], uint64_t need_ids_len); +EXTERNC const char* reconcile_with_ids(void* negentropy, const char* query, uint64_t query_len, const char* have_ids[], + uint64_t *have_ids_len, const char* need_ids[], uint64_t *need_ids_len); #endif From b467a099401c876b4f517a8a5eb783b6556ca05e Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Wed, 21 Feb 2024 13:45:24 +0530 Subject: [PATCH 16/24] chore: modify to use struct and fix makefile --- cpp/Makefile | 4 ++-- cpp/negentropy_wrapper.c | 22 ++++++++++++---------- cpp/negentropy_wrapper.h | 13 +++++++++---- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/cpp/Makefile b/cpp/Makefile index b0553fd..d7626fc 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -4,7 +4,7 @@ INCS = -I./ -I/opt/homebrew/include/ -I./vendor/lmdbxx/include/ TARGET = libnegentropy.so -.PHONY: all clean install-deps +.PHONY: all clean install-deps precompiled-header shared-lib all: precompiled-header shared-lib @@ -16,7 +16,7 @@ install-deps: precompiled-header: g++ --std=c++20 -Wall -fexceptions -g negentropy.h $(INCS) -shared-lib: negentropy.h.gch +shared-lib: g++ --std=c++20 $(INCS) -shared -fPIC -o $(TARGET) negentropy_wrapper.c -lcrypto -lssl -L/opt/homebrew/lib/ clean: diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index 5974aac..0499d63 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -1,3 +1,5 @@ +#include + #include "negentropy.h" #include "negentropy/storage/BTreeMem.h" #include "negentropy_wrapper.h" @@ -5,7 +7,7 @@ //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions - +using namespace std; void* storage_new(const char* db_path, const char* name){ negentropy::storage::BTreeMem* storage; @@ -65,32 +67,32 @@ void negentropy_setinitiator(void* negentropy){ } -bool storage_insert(void* storage, uint64_t createdAt, const char* id){ +bool storage_insert(void* storage, uint64_t createdAt, buffer* id){ negentropy::storage::BTreeMem* lmdbStorage; lmdbStorage = reinterpret_cast(storage); - + std::cout << "inserting entry in storage, createdAt:" << createdAt << ",id:" << std::string_view(id->data, id->len) << "length is:"<< id->len << std::endl; //TODO: Error handling. Is it required? //How does out of memory get handled? - return lmdbStorage->insert(createdAt, id); + return lmdbStorage->insert(createdAt, std::string_view(id->data, id->len)); } -bool storage_erase(void* storage, uint64_t createdAt, const char* id){ +bool storage_erase(void* storage, uint64_t createdAt, buffer* id){ negentropy::storage::BTreeMem* lmdbStorage; lmdbStorage = reinterpret_cast(storage); //TODO: Error handling - return lmdbStorage->erase(createdAt, id); + return lmdbStorage->erase(createdAt, std::string_view(id->data, id->len)); } -const char* reconcile(void* negentropy, const char* query, uint64_t query_len){ +const char* reconcile(void* negentropy, buffer* query){ Negentropy *ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); std::string* output = new std::string(); try { - *output = ngn_inst->reconcile(std::string_view(query, query_len)); + *output = ngn_inst->reconcile(std::string_view(query->data, query->len)); } catch(negentropy::err e){ //TODO:Find a way to return this error return NULL; @@ -105,7 +107,7 @@ char *convert(const std::string & s) return pc; } -const char* reconcile_with_ids(void* negentropy, const char* query, uint64_t query_len, char* have_ids[], +const char* reconcile_with_ids(void* negentropy, buffer* query, char* have_ids[], uint64_t *have_ids_len, char* need_ids[], uint64_t *need_ids_len){ Negentropy *ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); @@ -115,7 +117,7 @@ const char* reconcile_with_ids(void* negentropy, const char* query, uint64_t que std::vector needIds; try { - *output = ngn_inst->reconcile(std::string_view(query, query_len), haveIds, needIds); + *output = ngn_inst->reconcile(std::string_view(query->data, query->len), haveIds, needIds); *have_ids_len = haveIds.size(); *need_ids_len = needIds.size(); diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index 68fcff5..bc979aa 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -8,6 +8,11 @@ #define EXTERNC #endif +typedef struct _buffer_{ + uint64_t len ; + char* data; +}buffer; + //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions @@ -19,13 +24,13 @@ EXTERNC const char* negentropy_initiate(void* negentropy); EXTERNC void negentropy_setinitiator(void* negentropy); -EXTERNC bool storage_insert(void* storage, uint64_t createdAt, const char* id); +EXTERNC bool storage_insert(void* storage, uint64_t createdAt, buffer* id); -EXTERNC bool storage_erase(void* storage, uint64_t createdAt, const char* id); +EXTERNC bool storage_erase(void* storage, uint64_t createdAt, buffer* id); -EXTERNC const char* reconcile(void* negentropy, const char* query, uint64_t query_len); +EXTERNC const char* reconcile(void* negentropy, buffer* query); -EXTERNC const char* reconcile_with_ids(void* negentropy, const char* query, uint64_t query_len, const char* have_ids[], +EXTERNC const char* reconcile_with_ids(void* negentropy, buffer* query, const char* have_ids[], uint64_t *have_ids_len, const char* need_ids[], uint64_t *need_ids_len); #endif From 34fd921b89577cee691ba5e69bed6c06554bd736 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Thu, 22 Feb 2024 15:19:27 +0530 Subject: [PATCH 17/24] chore: experimenting with callback --- cpp/negentropy.h | 3 ++- cpp/negentropy_wrapper.c | 24 ++++++++++++++---------- cpp/negentropy_wrapper.h | 7 +++---- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/cpp/negentropy.h b/cpp/negentropy.h index d3de53e..9a8e4ee 100644 --- a/cpp/negentropy.h +++ b/cpp/negentropy.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "negentropy/encoding.h" #include "negentropy/types.h" @@ -49,7 +50,7 @@ struct Negentropy { std::string output; output.push_back(PROTOCOL_VERSION); - + std::cout << "storage size" << storage.size() << std::endl; output += splitRange(0, storage.size(), Bound(MAX_U64)); return output; diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index 0499d63..501e86d 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -7,8 +7,6 @@ //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions -using namespace std; - void* storage_new(const char* db_path, const char* name){ negentropy::storage::BTreeMem* storage; /* @@ -44,18 +42,23 @@ void* negentropy_new(void* storage, uint64_t frameSizeLimit){ return ne; } -const char* negentropy_initiate(void* negentropy){ +void negentropy_initiate(void* negentropy, void (*callback)(const char* buf, size_t len)){ Negentropy* ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); - std::string* output = new std::string(); + std::string output; try { - *output = ngn_inst->initiate(); + output = ngn_inst->initiate(); + std::cout << "output of initiate is, len:" << output.size() << std::hex << output << std::endl; } catch(negentropy::err e){ //TODO:Find a way to return this error - return NULL; + callback(NULL,0); + return ; } - return output->c_str(); + callback(output.c_str(), output.size()); + //TODO: Avoid copy and use a callback + //memcpy(buf, output, output.size()) + return ; } void negentropy_setinitiator(void* negentropy){ @@ -70,7 +73,7 @@ void negentropy_setinitiator(void* negentropy){ bool storage_insert(void* storage, uint64_t createdAt, buffer* id){ negentropy::storage::BTreeMem* lmdbStorage; lmdbStorage = reinterpret_cast(storage); - std::cout << "inserting entry in storage, createdAt:" << createdAt << ",id:" << std::string_view(id->data, id->len) << "length is:"<< id->len << std::endl; + std::cout << "inserting entry in storage, createdAt:" << createdAt << ",id:" << std::hex << id->data << " length is:"<< id->len << std::endl; //TODO: Error handling. Is it required? //How does out of memory get handled? return lmdbStorage->insert(createdAt, std::string_view(id->data, id->len)); @@ -122,6 +125,7 @@ const char* reconcile_with_ids(void* negentropy, buffer* query, char* have_ids[ *have_ids_len = haveIds.size(); *need_ids_len = needIds.size(); //TODO: Optimize to not copy and rather return memory reference. + std::cout << "*have_ids_len:" << *have_ids_len << "*need_ids_len:" << *need_ids_len << "output has value" << output->has_value() << std::endl; std::transform(haveIds.begin(), haveIds.end(), have_ids, convert); std::transform(needIds.begin(), needIds.end(), need_ids, convert); @@ -130,11 +134,11 @@ const char* reconcile_with_ids(void* negentropy, buffer* query, char* have_ids[ //TODO:Find a way to return this error return NULL; } - if (output->has_value()) { + if (!output->has_value()) { //TODO: Figure out diff between error and this. return NULL; }else { - + std::cout << "output value" << output->value() << std::endl; return output->value().c_str(); } } diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index bc979aa..0416b54 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -1,4 +1,3 @@ - #ifndef _NEGENTROPY_WRAPPER_H #define _NEGENTROPY_WRAPPER_H @@ -20,7 +19,7 @@ EXTERNC void* storage_new(const char* db_path, const char* name); EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit); -EXTERNC const char* negentropy_initiate(void* negentropy); +EXTERNC void negentropy_initiate(void* negentropy, void *(const char* buf, size_t len)); EXTERNC void negentropy_setinitiator(void* negentropy); @@ -30,8 +29,8 @@ EXTERNC bool storage_erase(void* storage, uint64_t createdAt, buffer* id); EXTERNC const char* reconcile(void* negentropy, buffer* query); -EXTERNC const char* reconcile_with_ids(void* negentropy, buffer* query, const char* have_ids[], - uint64_t *have_ids_len, const char* need_ids[], uint64_t *need_ids_len); +EXTERNC const char* reconcile_with_ids(void* negentropy, buffer* query, char* have_ids[], + uint64_t *have_ids_len, char* need_ids[], uint64_t *need_ids_len); #endif From 97b7ef11560933df7f1b95bc542417a51395d71a Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Thu, 22 Feb 2024 16:47:07 +0530 Subject: [PATCH 18/24] chore: reverting callback changes --- cpp/negentropy_wrapper.c | 16 ++++++---------- cpp/negentropy_wrapper.h | 3 ++- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index 501e86d..9dc9277 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -42,23 +42,19 @@ void* negentropy_new(void* storage, uint64_t frameSizeLimit){ return ne; } -void negentropy_initiate(void* negentropy, void (*callback)(const char* buf, size_t len)){ +const char* negentropy_initiate(void* negentropy){ Negentropy* ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); - std::string output; + std::string* output = new std::string(); try { - output = ngn_inst->initiate(); - std::cout << "output of initiate is, len:" << output.size() << std::hex << output << std::endl; + *output = ngn_inst->initiate(); + std::cout << "output of initiate is, len:" << output->size() << std::hex << *output << std::endl; } catch(negentropy::err e){ //TODO:Find a way to return this error - callback(NULL,0); - return ; + return NULL; } - callback(output.c_str(), output.size()); - //TODO: Avoid copy and use a callback - //memcpy(buf, output, output.size()) - return ; + return output->c_str(); } void negentropy_setinitiator(void* negentropy){ diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index 0416b54..4210816 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -1,3 +1,4 @@ + #ifndef _NEGENTROPY_WRAPPER_H #define _NEGENTROPY_WRAPPER_H @@ -19,7 +20,7 @@ EXTERNC void* storage_new(const char* db_path, const char* name); EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit); -EXTERNC void negentropy_initiate(void* negentropy, void *(const char* buf, size_t len)); +EXTERNC const char* negentropy_initiate(void* negentropy); EXTERNC void negentropy_setinitiator(void* negentropy); From c38e6e2ed8d924d3e8ad16357cf9025eff1d1df5 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Mon, 4 Mar 2024 16:28:35 +0530 Subject: [PATCH 19/24] chore: add logging and fix some issues --- cpp/example/Makefile | 2 +- cpp/example/test.c | 73 +++++++++++++++++++++++++++++++++++----- cpp/negentropy_wrapper.c | 41 +++++++++++++++------- cpp/negentropy_wrapper.h | 6 ++-- cpp/vendor/lmdbxx | 2 +- 5 files changed, 98 insertions(+), 26 deletions(-) diff --git a/cpp/example/Makefile b/cpp/example/Makefile index fb38278..2dd32ed 100644 --- a/cpp/example/Makefile +++ b/cpp/example/Makefile @@ -1,4 +1,4 @@ .PHONY: all all: - g++ --std=c++20 test.c -I../ -lnegentropy -lcrypto -L../ + g++ --std=c++20 test.c -I../ -lnegentropy -lcrypto -L../ -L/opt/homebrew/lib/ -Wc++11-narrowing diff --git a/cpp/example/test.c b/cpp/example/test.c index ae877ec..516b23d 100644 --- a/cpp/example/test.c +++ b/cpp/example/test.c @@ -1,22 +1,79 @@ #include #include - +#include +#include +#include +#include +#include +#include #include "../negentropy_wrapper.h" + int main(){ - void* st = storage_new("",""); - if(st == NULL){ + void* st1 = storage_new("",""); + if(st1 == NULL){ perror("failed to create storage"); } - void* ngn_inst = negentropy_new(st, 153600); - if(ngn_inst == NULL){ + void* ngn_inst1 = negentropy_new(st1, 153600); + if(ngn_inst1 == NULL){ perror("failed to create negentropy instance"); } - const char* output = negentropy_initiate(ngn_inst); - if(ngn_inst == NULL){ + void* st2 = storage_new("",""); + if(st2 == NULL){ + perror("failed to create storage"); + } + void* ngn_inst2 = negentropy_new(st2, 153600); + if(ngn_inst2 == NULL){ + perror("failed to create negentropy instance"); + } + + unsigned char m1[] = {0x6a, 0xdf, 0xaa, 0xe0, 0x31, 0xeb, 0x61, 0xa8, \ + 0x3c, 0xff, 0x9c, 0xfd, 0xd2, 0xae, 0xf6, 0xed, \ + 0x63, 0xda, 0xcf, 0xaa, 0x96, 0xd0, 0x51, 0x26, \ + 0x7e, 0xf1, 0x0c, 0x8b, 0x61, 0xae, 0x35, 0xe9};//"61dfaae031eb61a83cff9cfdd2aef6ed63dacfaa96d051267ef10c8b61ae35e9"; + buffer b1 ; + b1.len = 32; + b1.data = m1; + +/* char m2[] = "28798d295c30c7e6d9a4a96cdda7e020f7aa7168cce063302ed19b856332959e"; + buffer b2 ; + b2.len = 32; + b2.data = m2; */ + + bool ret = storage_insert(st1,time(NULL),&b1); + if (ret){ + printf("inserted hash successfully in st1\n"); + } +/* + ret = storage_insert(st2,time(NULL),&b2); + if (ret){ + printf("inserted hash %s successfully in st2\n", m2); + } + + ret = storage_insert(st2,time(NULL),&b1); + if (ret){ + printf("inserted hash %s successfully in st2\n", m1); + } + */ + +/* std::string out; + negentropy_initiate(ngn_inst1, &out); + if(out.size() == 0){ perror("failed to initiate negentropy instance"); } - printf("initiated negentropy successfully with output %s \n", output); + printf("initiated negentropy successfully with output of len %zu \n", out.size()); + buffer b3 ; + b3.len = out.size(); + b3.data = (unsigned char*)malloc(b3.len); + memcpy(b3.data, out.c_str(),out.size()); + + const char* req2 = reconcile(ngn_inst2, &b3); */ + + //free(b3.data); + //b3.len = len; + //b3.data = (char*)malloc(len); + + //reconcile_with_ids(ngn_inst1, &b3, ) } \ No newline at end of file diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index 9dc9277..04eebc9 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -42,7 +42,7 @@ void* negentropy_new(void* storage, uint64_t frameSizeLimit){ return ne; } -const char* negentropy_initiate(void* negentropy){ +size_t negentropy_initiate(void* negentropy, buffer* out){ Negentropy* ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); @@ -52,9 +52,10 @@ const char* negentropy_initiate(void* negentropy){ std::cout << "output of initiate is, len:" << output->size() << std::hex << *output << std::endl; } catch(negentropy::err e){ //TODO:Find a way to return this error - return NULL; + return 0; } - return output->c_str(); + memcpy( out->data, output->c_str() ,output->size()); + return output->size(); } void negentropy_setinitiator(void* negentropy){ @@ -65,38 +66,52 @@ void negentropy_setinitiator(void* negentropy){ } +void printHexString(std::string_view toPrint){ + for (size_t i = 0; i < toPrint.size(); ++i) { + printf("%0hhx", toPrint[i]); + } + printf("\n"); +} bool storage_insert(void* storage, uint64_t createdAt, buffer* id){ negentropy::storage::BTreeMem* lmdbStorage; lmdbStorage = reinterpret_cast(storage); - std::cout << "inserting entry in storage, createdAt:" << createdAt << ",id:" << std::hex << id->data << " length is:"<< id->len << std::endl; + std::string_view data(reinterpret_cast< char const* >(id->data), id->len); + + std::cout << "inserting entry in storage, createdAt:" << createdAt << ",id:"; + printHexString(data); + //TODO: Error handling. Is it required? //How does out of memory get handled? - return lmdbStorage->insert(createdAt, std::string_view(id->data, id->len)); + return lmdbStorage->insert(createdAt, data); } bool storage_erase(void* storage, uint64_t createdAt, buffer* id){ negentropy::storage::BTreeMem* lmdbStorage; lmdbStorage = reinterpret_cast(storage); + std::string_view data(reinterpret_cast< char const* >(id->data), id->len); + + std::cout << "erasing entry from storage, createdAt:" << createdAt << ",id:"; + printHexString(data); //TODO: Error handling - return lmdbStorage->erase(createdAt, std::string_view(id->data, id->len)); + return lmdbStorage->erase(createdAt, data); } -const char* reconcile(void* negentropy, buffer* query){ +size_t reconcile(void* negentropy, buffer* query, buffer* output){ Negentropy *ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); - - std::string* output = new std::string(); + std::string* out = new std::string(); try { - *output = ngn_inst->reconcile(std::string_view(query->data, query->len)); + *out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len)); } catch(negentropy::err e){ //TODO:Find a way to return this error - return NULL; + return 0; } - return output->c_str(); + memcpy( output->data, out->c_str() ,out->size()); + return out->size(); } char *convert(const std::string & s) @@ -116,7 +131,7 @@ const char* reconcile_with_ids(void* negentropy, buffer* query, char* have_ids[ std::vector needIds; try { - *output = ngn_inst->reconcile(std::string_view(query->data, query->len), haveIds, needIds); + *output = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len), haveIds, needIds); *have_ids_len = haveIds.size(); *need_ids_len = needIds.size(); diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index 4210816..8974e1c 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -10,7 +10,7 @@ typedef struct _buffer_{ uint64_t len ; - char* data; + unsigned char* data; }buffer; //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. @@ -20,7 +20,7 @@ EXTERNC void* storage_new(const char* db_path, const char* name); EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit); -EXTERNC const char* negentropy_initiate(void* negentropy); +EXTERNC size_t negentropy_initiate(void* negentropy, buffer* output); EXTERNC void negentropy_setinitiator(void* negentropy); @@ -28,7 +28,7 @@ EXTERNC bool storage_insert(void* storage, uint64_t createdAt, buffer* id); EXTERNC bool storage_erase(void* storage, uint64_t createdAt, buffer* id); -EXTERNC const char* reconcile(void* negentropy, buffer* query); +EXTERNC size_t reconcile(void* negentropy, buffer* query,buffer* output); EXTERNC const char* reconcile_with_ids(void* negentropy, buffer* query, char* have_ids[], uint64_t *have_ids_len, char* need_ids[], uint64_t *need_ids_len); diff --git a/cpp/vendor/lmdbxx b/cpp/vendor/lmdbxx index d649a58..af64901 160000 --- a/cpp/vendor/lmdbxx +++ b/cpp/vendor/lmdbxx @@ -1 +1 @@ -Subproject commit d649a581d3cebfe7d8bd4d345bc2c1c4c2cc59a2 +Subproject commit af649014456719b16100c7da31d46378c91b0e7b From 8972f3ba693da2ef8913638c3e1850340741d2e9 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Tue, 5 Mar 2024 11:27:30 +0530 Subject: [PATCH 20/24] chore: example code to test C wrapper --- cpp/Makefile | 4 +- cpp/example/Makefile | 2 +- cpp/example/test.c | 65 +++++++++++++++++++++++++-------- cpp/negentropy_wrapper.c | 79 ++++++++++++++++++++++++---------------- cpp/negentropy_wrapper.h | 8 ++-- 5 files changed, 105 insertions(+), 53 deletions(-) diff --git a/cpp/Makefile b/cpp/Makefile index d7626fc..39f0719 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -14,10 +14,10 @@ install-deps: # Generate 'negentropy.h.gch' precompiled-header: - g++ --std=c++20 -Wall -fexceptions -g negentropy.h $(INCS) + g++ -O0 --std=c++20 -Wall -fexceptions -g negentropy.h $(INCS) shared-lib: - g++ --std=c++20 $(INCS) -shared -fPIC -o $(TARGET) negentropy_wrapper.c -lcrypto -lssl -L/opt/homebrew/lib/ + g++ -O0 -g -std=c++20 $(INCS) -shared -fPIC -o $(TARGET) negentropy_wrapper.c -lcrypto -lssl -L/opt/homebrew/lib/ clean: rm -f $(TARGET) negentropy.h.gch libnegentropy.so diff --git a/cpp/example/Makefile b/cpp/example/Makefile index 2dd32ed..070a18e 100644 --- a/cpp/example/Makefile +++ b/cpp/example/Makefile @@ -1,4 +1,4 @@ .PHONY: all all: - g++ --std=c++20 test.c -I../ -lnegentropy -lcrypto -L../ -L/opt/homebrew/lib/ -Wc++11-narrowing + g++ -g -O0 --std=c++20 test.c -I../ -lnegentropy -lcrypto -L../ -L/opt/homebrew/lib/ -Wc++11-narrowing diff --git a/cpp/example/test.c b/cpp/example/test.c index 516b23d..0fb7211 100644 --- a/cpp/example/test.c +++ b/cpp/example/test.c @@ -8,6 +8,27 @@ #include #include "../negentropy_wrapper.h" +void printHexBuffer(buffer buf){ + for (uint64_t i = 0; i < buf.len; ++i) { + printf("%0hhx", buf.data[i]); + } + printf("\n"); +} + +void rec_callback(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output){ + printf("needIds count:%llu , haveIds count: %llu \n",need_ids_len, have_ids_len); + + for (int i=0; i < need_ids_len ; i++) { + printf("need ID at %d :", i); + printHexBuffer(need_ids[i]); + } + + for (int j=0; j < have_ids_len ; j++) { + printf("need ID at %d :", j); + printHexBuffer(have_ids[j]); + } +} + int main(){ void* st1 = storage_new("",""); @@ -36,41 +57,55 @@ int main(){ b1.len = 32; b1.data = m1; -/* char m2[] = "28798d295c30c7e6d9a4a96cdda7e020f7aa7168cce063302ed19b856332959e"; + unsigned char m2[] = {0x28 ,0x79 ,0x8d ,0x29 ,0x5c ,0x30 ,0xc7 ,0xe6 \ + ,0xd9 ,0xa4 ,0xa9 ,0x6c ,0xdd ,0xa7 ,0xe0 ,0x20 \ + ,0xf7 ,0xaa ,0x71 ,0x68 ,0xcc ,0xe0 ,0x63 ,0x30 \ + ,0x2e ,0xd1 ,0x9b ,0x85 ,0x63 ,0x32 ,0x95 ,0x9e}; //28798d295c30c7e6d9a4a96cdda7e020f7aa7168cce063302ed19b856332959e buffer b2 ; b2.len = 32; - b2.data = m2; */ + b2.data = m2; bool ret = storage_insert(st1,time(NULL),&b1); if (ret){ printf("inserted hash successfully in st1\n"); } -/* + ret = storage_insert(st2,time(NULL),&b2); if (ret){ - printf("inserted hash %s successfully in st2\n", m2); + printf("inserted hash successfully in st2\n"); } ret = storage_insert(st2,time(NULL),&b1); if (ret){ - printf("inserted hash %s successfully in st2\n", m1); + printf("inserted hash successfully in st2\n"); } - */ + + buffer b4 ; + b4.len = 153600; + b4.data = (unsigned char*)malloc(153600); -/* std::string out; - negentropy_initiate(ngn_inst1, &out); - if(out.size() == 0){ + size_t outSize = negentropy_initiate(ngn_inst1, &b4); + if(outSize == 0){ perror("failed to initiate negentropy instance"); } - printf("initiated negentropy successfully with output of len %zu \n", out.size()); + printf("initiated negentropy successfully with output of len %zu \n", outSize); + b4.len = outSize; + buffer b3 ; - b3.len = out.size(); - b3.data = (unsigned char*)malloc(b3.len); - memcpy(b3.data, out.c_str(),out.size()); + b3.len = 153600; + b3.data = (unsigned char*)malloc(153600); - const char* req2 = reconcile(ngn_inst2, &b3); */ + outSize = reconcile(ngn_inst2, &b4, &b3); + if(outSize == 0){ + perror("nothing to reconcile"); + } + printf("reconcile returned with output of len %zu \n", outSize); + b3.len = outSize; - //free(b3.data); + outSize = reconcile_with_ids(ngn_inst1, &b3, &rec_callback); + + free(b3.data); + free(b4.data); //b3.len = len; //b3.data = (char*)malloc(len); diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index 04eebc9..a155c49 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -7,6 +7,13 @@ //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions +void printHexString(std::string_view toPrint){ + for (size_t i = 0; i < toPrint.size(); ++i) { + printf("%0hhx", toPrint[i]); + } + printf("\n"); +} + void* storage_new(const char* db_path, const char* name){ negentropy::storage::BTreeMem* storage; /* @@ -49,7 +56,8 @@ size_t negentropy_initiate(void* negentropy, buffer* out){ std::string* output = new std::string(); try { *output = ngn_inst->initiate(); - std::cout << "output of initiate is, len:" << output->size() << std::hex << *output << std::endl; + std::cout << "output of initiate is, len:" << output->size() << ", output:"; + printHexString(std::string_view(*output)); } catch(negentropy::err e){ //TODO:Find a way to return this error return 0; @@ -66,12 +74,6 @@ void negentropy_setinitiator(void* negentropy){ } -void printHexString(std::string_view toPrint){ - for (size_t i = 0; i < toPrint.size(); ++i) { - printf("%0hhx", toPrint[i]); - } - printf("\n"); -} bool storage_insert(void* storage, uint64_t createdAt, buffer* id){ negentropy::storage::BTreeMem* lmdbStorage; @@ -106,6 +108,8 @@ size_t reconcile(void* negentropy, buffer* query, buffer* output){ std::string* out = new std::string(); try { *out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len)); + std::cout << "reconcile output of reconcile is, len:" << out->size() << ", output:"; + printHexString(std::string_view(*out)); } catch(negentropy::err e){ //TODO:Find a way to return this error return 0; @@ -114,42 +118,53 @@ size_t reconcile(void* negentropy, buffer* query, buffer* output){ return out->size(); } -char *convert(const std::string & s) +void transform(std::vector &from_ids, buffer* to_ids) { - char *pc = new char[s.size()+1]; - std::strcpy(pc, s.c_str()); - return pc; + for (int i=0; i < from_ids.size(); i ++){ + to_ids[i].len = from_ids[i].size(); + to_ids[i].data = (unsigned char*)from_ids[i].c_str(); + } } -const char* reconcile_with_ids(void* negentropy, buffer* query, char* have_ids[], - uint64_t *have_ids_len, char* need_ids[], uint64_t *need_ids_len){ +int reconcile_with_ids(void* negentropy, buffer* query,reconcile_cbk cbk){ Negentropy *ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); - std::optional* output; - std::vector haveIds; - std::vector needIds; + std::optional out; + std::vector haveIds, needIds; + uint64_t have_ids_len, need_ids_len; + buffer* have_ids; + buffer* need_ids; try { - *output = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len), haveIds, needIds); + out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len), haveIds, needIds); - *have_ids_len = haveIds.size(); - *need_ids_len = needIds.size(); - //TODO: Optimize to not copy and rather return memory reference. - std::cout << "*have_ids_len:" << *have_ids_len << "*need_ids_len:" << *need_ids_len << "output has value" << output->has_value() << std::endl; + have_ids_len = haveIds.size(); + need_ids_len = needIds.size(); + have_ids = (buffer*)malloc(have_ids_len*sizeof(buffer)); + need_ids = (buffer*)malloc(need_ids_len*sizeof(buffer)); - std::transform(haveIds.begin(), haveIds.end(), have_ids, convert); - std::transform(needIds.begin(), needIds.end(), need_ids, convert); + std::cout << "have_ids_len:" << have_ids_len << "need_ids_len:" << need_ids_len << std::endl; + std::flush(std::cout); + transform(haveIds, have_ids); + transform(needIds, need_ids); + std::cout << "for debug" << std::endl; } catch(negentropy::err e){ - //TODO:Find a way to return this error - return NULL; + std::cout << "caught error "<< e.what() << std::endl; + //TODO:Find a way to return this error and cleanup partially allocated memory if any + return -1; } - if (!output->has_value()) { - //TODO: Figure out diff between error and this. - return NULL; - }else { - std::cout << "output value" << output->value() << std::endl; - return output->value().c_str(); + buffer output; + if (out) { + output.len = out.value().size(); + output.data = (unsigned char*)out.value().c_str(); + std::cout << "reconcile_with_ids output of reconcile is, len:" << out.value().size() << ", output:"; + printHexString(std::string_view(out.value())); + std::cout << "invoking callback" << std::endl; } -} + cbk(have_ids, have_ids_len, need_ids, need_ids_len, &output); + free(have_ids); + free(need_ids); + return 0; +} diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index 8974e1c..21e897a 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -28,10 +28,12 @@ EXTERNC bool storage_insert(void* storage, uint64_t createdAt, buffer* id); EXTERNC bool storage_erase(void* storage, uint64_t createdAt, buffer* id); -EXTERNC size_t reconcile(void* negentropy, buffer* query,buffer* output); +EXTERNC size_t reconcile(void* negentropy, buffer* query, buffer* output); + +EXTERNC typedef void (*reconcile_cbk)(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output); + +EXTERNC int reconcile_with_ids(void* negentropy, buffer* query, reconcile_cbk cbk); -EXTERNC const char* reconcile_with_ids(void* negentropy, buffer* query, char* have_ids[], - uint64_t *have_ids_len, char* need_ids[], uint64_t *need_ids_len); #endif From 315e1de72e924f3fce0cd2ac20427da2a5bc216b Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Tue, 5 Mar 2024 20:36:35 +0530 Subject: [PATCH 21/24] chore: callback integration in progress --- cpp/negentropy_wrapper.c | 15 +++++++++------ cpp/negentropy_wrapper.h | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index a155c49..f624a7f 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -126,7 +126,7 @@ void transform(std::vector &from_ids, buffer* to_ids) } } -int reconcile_with_ids(void* negentropy, buffer* query,reconcile_cbk cbk){ +int reconcile_with_ids(void* negentropy, buffer* query,reconcile_cbk cbk, char* outptr){ Negentropy *ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); @@ -145,25 +145,28 @@ int reconcile_with_ids(void* negentropy, buffer* query,reconcile_cbk cbk){ need_ids = (buffer*)malloc(need_ids_len*sizeof(buffer)); std::cout << "have_ids_len:" << have_ids_len << "need_ids_len:" << need_ids_len << std::endl; - std::flush(std::cout); transform(haveIds, have_ids); transform(needIds, need_ids); - std::cout << "for debug" << std::endl; } catch(negentropy::err e){ std::cout << "caught error "<< e.what() << std::endl; //TODO:Find a way to return this error and cleanup partially allocated memory if any return -1; } - buffer output; + buffer output = {0,NULL}; if (out) { output.len = out.value().size(); output.data = (unsigned char*)out.value().c_str(); std::cout << "reconcile_with_ids output of reconcile is, len:" << out.value().size() << ", output:"; printHexString(std::string_view(out.value())); - std::cout << "invoking callback" << std::endl; } - cbk(have_ids, have_ids_len, need_ids, need_ids_len, &output); + std::cout << "invoking callback" << std::endl; + std::flush(std::cout); + + cbk(have_ids, have_ids_len, need_ids, need_ids_len, &output, outptr); + std::cout << "invoked callback" << std::endl; + std::flush(std::cout); + free(have_ids); free(need_ids); return 0; diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index 21e897a..f3d0eac 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -30,9 +30,9 @@ EXTERNC bool storage_erase(void* storage, uint64_t createdAt, buffer* id); EXTERNC size_t reconcile(void* negentropy, buffer* query, buffer* output); -EXTERNC typedef void (*reconcile_cbk)(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output); +EXTERNC typedef void (*reconcile_cbk)(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output, char* outptr ); -EXTERNC int reconcile_with_ids(void* negentropy, buffer* query, reconcile_cbk cbk); +EXTERNC int reconcile_with_ids(void* negentropy, buffer* query, reconcile_cbk cbk, char* outptr); #endif From f23efbb66c1f7a4d165bc2f001dbf2a0b252821c Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Wed, 6 Mar 2024 05:45:33 +0530 Subject: [PATCH 22/24] chore: support a non-callback version of reconcile_with_ids --- cpp/example/test.c | 21 ++++++++++---- cpp/negentropy_wrapper.c | 62 ++++++++++++++++++++++++++++++++++++++-- cpp/negentropy_wrapper.h | 11 +++++++ 3 files changed, 85 insertions(+), 9 deletions(-) diff --git a/cpp/example/test.c b/cpp/example/test.c index 0fb7211..f3511db 100644 --- a/cpp/example/test.c +++ b/cpp/example/test.c @@ -102,13 +102,22 @@ int main(){ printf("reconcile returned with output of len %zu \n", outSize); b3.len = outSize; - outSize = reconcile_with_ids(ngn_inst1, &b3, &rec_callback); + //outSize = reconcile_with_ids(ngn_inst1, &b3, &rec_callback); + + result res; + reconcile_with_ids_no_cbk(ngn_inst1, &b3, &res); + printf("needIds count:%llu , haveIds count: %llu \n",res.need_ids_len, res.have_ids_len); + + for (int i=0; i < res.need_ids_len ; i++) { + printf("need ID at %d :", i); + printHexBuffer(res.need_ids[i]); + } + + for (int j=0; j < res.have_ids_len ; j++) { + printf("need ID at %d :", j); + printHexBuffer(res.have_ids[j]); + } free(b3.data); free(b4.data); - //b3.len = len; - //b3.data = (char*)malloc(len); - - //reconcile_with_ids(ngn_inst1, &b3, ) - } \ No newline at end of file diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index f624a7f..5a14c37 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -74,7 +74,6 @@ void negentropy_setinitiator(void* negentropy){ } - bool storage_insert(void* storage, uint64_t createdAt, buffer* id){ negentropy::storage::BTreeMem* lmdbStorage; lmdbStorage = reinterpret_cast(storage); @@ -88,7 +87,6 @@ bool storage_insert(void* storage, uint64_t createdAt, buffer* id){ return lmdbStorage->insert(createdAt, data); } - bool storage_erase(void* storage, uint64_t createdAt, buffer* id){ negentropy::storage::BTreeMem* lmdbStorage; lmdbStorage = reinterpret_cast(storage); @@ -101,7 +99,6 @@ bool storage_erase(void* storage, uint64_t createdAt, buffer* id){ return lmdbStorage->erase(createdAt, data); } - size_t reconcile(void* negentropy, buffer* query, buffer* output){ Negentropy *ngn_inst; ngn_inst = reinterpret_cast*>(negentropy); @@ -171,3 +168,62 @@ int reconcile_with_ids(void* negentropy, buffer* query,reconcile_cbk cbk, char* free(need_ids); return 0; } + +void transform_with_alloc(std::vector &from_ids, buffer* to_ids) +{ + for (int i=0; i < from_ids.size(); i ++){ + to_ids[i].data = (unsigned char*) malloc(from_ids[i].size()*sizeof(unsigned char)); + to_ids[i].len = from_ids[i].size(); + memcpy(to_ids[i].data, from_ids[i].c_str(),to_ids[i].len); + } +} + +void reconcile_with_ids_no_cbk(void* negentropy, buffer* query, result* result){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::optional out; + std::vector haveIds, needIds; + try { + out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len), haveIds, needIds); + + result->have_ids_len = haveIds.size(); + result->need_ids_len = needIds.size(); + result->have_ids = (buffer*)malloc(result->have_ids_len*sizeof(buffer)); + result->need_ids = (buffer*)malloc(result->need_ids_len*sizeof(buffer)); + + std::cout << "have_ids_len:" << result->have_ids_len << "need_ids_len:" << result->need_ids_len << std::endl; + + transform_with_alloc(haveIds, result->have_ids); + transform_with_alloc(needIds, result->need_ids); + + } catch(negentropy::err e){ + std::cout << "caught error "<< e.what() << std::endl; + //TODO:Find a way to return this error and cleanup partially allocated memory if any + return ; + } + buffer output = {0,NULL}; + if (out) { + result->output.len = out.value().size(); + result->output.data = (unsigned char*)malloc(out.value().size()*sizeof(unsigned char)); + result->output.data = (unsigned char*)out.value().c_str(); + std::cout << "reconcile_with_ids output of reconcile is, len:" << out.value().size() << ", output:"; + printHexString(std::string_view(out.value())); + } + return ; +} + +//Note: This function assumes that all relevant heap memory is alloced and just tries to free +void free_result(result* r){ + free((void *) r->output.data); + + for (int i = 0; i < r->have_ids_len; i++) { + free((void *) r->have_ids[i].data); + } + free((void *)r->have_ids); + + for (int i = 0; i < r->need_ids_len; i++) { + free((void *) r->need_ids[i].data); + } + free((void *)r->need_ids); +} \ No newline at end of file diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h index f3d0eac..dd3fd29 100644 --- a/cpp/negentropy_wrapper.h +++ b/cpp/negentropy_wrapper.h @@ -13,6 +13,14 @@ typedef struct _buffer_{ unsigned char* data; }buffer; +typedef struct _result_ { + buffer output; + uint64_t have_ids_len; + uint64_t need_ids_len; + buffer* have_ids; + buffer* need_ids; +} result; + //This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. //TODO: Do error handling by catching exceptions @@ -34,6 +42,9 @@ EXTERNC typedef void (*reconcile_cbk)(buffer* have_ids, uint64_t have_ids_len, b EXTERNC int reconcile_with_ids(void* negentropy, buffer* query, reconcile_cbk cbk, char* outptr); +EXTERNC void reconcile_with_ids_no_cbk(void* negentropy, buffer* query, result* result); + +EXTERNC void free_result(result* result); #endif From 1487908100846577f6b3f34ab866702439075306 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Wed, 6 Mar 2024 16:22:19 +0530 Subject: [PATCH 23/24] chore: added header file to check if it fixes CI build issue --- cpp/negentropy_wrapper.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index 5a14c37..f4ca627 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -1,4 +1,5 @@ #include +#include #include "negentropy.h" #include "negentropy/storage/BTreeMem.h" @@ -226,4 +227,4 @@ void free_result(result* r){ free((void *) r->need_ids[i].data); } free((void *)r->need_ids); -} \ No newline at end of file +} From fa1473792677f5cb66215b2b7017e8ae7996a452 Mon Sep 17 00:00:00 2001 From: Prem Chaitanya Prathi Date: Thu, 7 Mar 2024 09:02:06 +0530 Subject: [PATCH 24/24] fix: leak in initiate --- cpp/negentropy_wrapper.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c index 5a14c37..363282d 100644 --- a/cpp/negentropy_wrapper.c +++ b/cpp/negentropy_wrapper.c @@ -63,7 +63,9 @@ size_t negentropy_initiate(void* negentropy, buffer* out){ return 0; } memcpy( out->data, output->c_str() ,output->size()); - return output->size(); + size_t outlen = output->size(); + delete output; + return outlen; } void negentropy_setinitiator(void* negentropy){