diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..93a4daf --- /dev/null +++ b/.gitignore @@ -0,0 +1,57 @@ +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +# ignore vscode files +.vscode/ + + diff --git a/.gitmodules b/.gitmodules index 2834d9b..8b45d4c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,5 +2,5 @@ path = test/cpp/hoytech-cpp url = https://github.com/hoytech/hoytech-cpp.git [submodule "test/cpp/lmdbxx"] - path = test/cpp/lmdbxx + path = cpp/vendor/lmdbxx url = https://github.com/hoytech/lmdbxx.git diff --git a/cpp/Makefile b/cpp/Makefile new file mode 100644 index 0000000..39f0719 --- /dev/null +++ b/cpp/Makefile @@ -0,0 +1,23 @@ +# Build a shared library of negentropy + +#TODO: Need to add compilation flags based on OS +INCS = -I./ -I/opt/homebrew/include/ -I./vendor/lmdbxx/include/ +TARGET = libnegentropy.so + +.PHONY: all clean install-deps precompiled-header shared-lib + +all: precompiled-header shared-lib + +#TODO: Need to add compilation flags based on OS +install-deps: + brew install lmdb openssl + +# Generate 'negentropy.h.gch' +precompiled-header: + g++ -O0 --std=c++20 -Wall -fexceptions -g negentropy.h $(INCS) + +shared-lib: + g++ -O0 -g -std=c++20 $(INCS) -shared -fPIC -o $(TARGET) negentropy_wrapper.c -lcrypto -lssl -L/opt/homebrew/lib/ + +clean: + rm -f $(TARGET) negentropy.h.gch libnegentropy.so diff --git a/cpp/example/Makefile b/cpp/example/Makefile new file mode 100644 index 0000000..070a18e --- /dev/null +++ b/cpp/example/Makefile @@ -0,0 +1,4 @@ +.PHONY: all + +all: + g++ -g -O0 --std=c++20 test.c -I../ -lnegentropy -lcrypto -L../ -L/opt/homebrew/lib/ -Wc++11-narrowing diff --git a/cpp/example/test.c b/cpp/example/test.c new file mode 100644 index 0000000..f3511db --- /dev/null +++ b/cpp/example/test.c @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "../negentropy_wrapper.h" + +void printHexBuffer(buffer buf){ + for (uint64_t i = 0; i < buf.len; ++i) { + printf("%0hhx", buf.data[i]); + } + printf("\n"); +} + +void rec_callback(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output){ + printf("needIds count:%llu , haveIds count: %llu \n",need_ids_len, have_ids_len); + + for (int i=0; i < need_ids_len ; i++) { + printf("need ID at %d :", i); + printHexBuffer(need_ids[i]); + } + + for (int j=0; j < have_ids_len ; j++) { + printf("need ID at %d :", j); + printHexBuffer(have_ids[j]); + } +} + + +int main(){ + void* st1 = storage_new("",""); + if(st1 == NULL){ + perror("failed to create storage"); + } + void* ngn_inst1 = negentropy_new(st1, 153600); + if(ngn_inst1 == NULL){ + perror("failed to create negentropy instance"); + } + + void* st2 = storage_new("",""); + if(st2 == NULL){ + perror("failed to create storage"); + } + void* ngn_inst2 = negentropy_new(st2, 153600); + if(ngn_inst2 == NULL){ + perror("failed to create negentropy instance"); + } + + unsigned char m1[] = {0x6a, 0xdf, 0xaa, 0xe0, 0x31, 0xeb, 0x61, 0xa8, \ + 0x3c, 0xff, 0x9c, 0xfd, 0xd2, 0xae, 0xf6, 0xed, \ + 0x63, 0xda, 0xcf, 0xaa, 0x96, 0xd0, 0x51, 0x26, \ + 0x7e, 0xf1, 0x0c, 0x8b, 0x61, 0xae, 0x35, 0xe9};//"61dfaae031eb61a83cff9cfdd2aef6ed63dacfaa96d051267ef10c8b61ae35e9"; + buffer b1 ; + b1.len = 32; + b1.data = m1; + + unsigned char m2[] = {0x28 ,0x79 ,0x8d ,0x29 ,0x5c ,0x30 ,0xc7 ,0xe6 \ + ,0xd9 ,0xa4 ,0xa9 ,0x6c ,0xdd ,0xa7 ,0xe0 ,0x20 \ + ,0xf7 ,0xaa ,0x71 ,0x68 ,0xcc ,0xe0 ,0x63 ,0x30 \ + ,0x2e ,0xd1 ,0x9b ,0x85 ,0x63 ,0x32 ,0x95 ,0x9e}; //28798d295c30c7e6d9a4a96cdda7e020f7aa7168cce063302ed19b856332959e + buffer b2 ; + b2.len = 32; + b2.data = m2; + + bool ret = storage_insert(st1,time(NULL),&b1); + if (ret){ + printf("inserted hash successfully in st1\n"); + } + + ret = storage_insert(st2,time(NULL),&b2); + if (ret){ + printf("inserted hash successfully in st2\n"); + } + + ret = storage_insert(st2,time(NULL),&b1); + if (ret){ + printf("inserted hash successfully in st2\n"); + } + + buffer b4 ; + b4.len = 153600; + b4.data = (unsigned char*)malloc(153600); + + size_t outSize = negentropy_initiate(ngn_inst1, &b4); + if(outSize == 0){ + perror("failed to initiate negentropy instance"); + } + printf("initiated negentropy successfully with output of len %zu \n", outSize); + b4.len = outSize; + + buffer b3 ; + b3.len = 153600; + b3.data = (unsigned char*)malloc(153600); + + outSize = reconcile(ngn_inst2, &b4, &b3); + if(outSize == 0){ + perror("nothing to reconcile"); + } + printf("reconcile returned with output of len %zu \n", outSize); + b3.len = outSize; + + //outSize = reconcile_with_ids(ngn_inst1, &b3, &rec_callback); + + result res; + reconcile_with_ids_no_cbk(ngn_inst1, &b3, &res); + printf("needIds count:%llu , haveIds count: %llu \n",res.need_ids_len, res.have_ids_len); + + for (int i=0; i < res.need_ids_len ; i++) { + printf("need ID at %d :", i); + printHexBuffer(res.need_ids[i]); + } + + for (int j=0; j < res.have_ids_len ; j++) { + printf("need ID at %d :", j); + printHexBuffer(res.have_ids[j]); + } + + free(b3.data); + free(b4.data); +} \ No newline at end of file diff --git a/cpp/negentropy.h b/cpp/negentropy.h index d3de53e..9a8e4ee 100644 --- a/cpp/negentropy.h +++ b/cpp/negentropy.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "negentropy/encoding.h" #include "negentropy/types.h" @@ -49,7 +50,7 @@ struct Negentropy { std::string output; output.push_back(PROTOCOL_VERSION); - + std::cout << "storage size" << storage.size() << std::endl; output += splitRange(0, storage.size(), Bound(MAX_U64)); return output; diff --git a/cpp/negentropy_wrapper.c b/cpp/negentropy_wrapper.c new file mode 100644 index 0000000..3d3790b --- /dev/null +++ b/cpp/negentropy_wrapper.c @@ -0,0 +1,232 @@ +#include +#include + +#include "negentropy.h" +#include "negentropy/storage/BTreeMem.h" +#include "negentropy_wrapper.h" + +//This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. +//TODO: Do error handling by catching exceptions + +void printHexString(std::string_view toPrint){ + for (size_t i = 0; i < toPrint.size(); ++i) { + printf("%0hhx", toPrint[i]); + } + printf("\n"); +} + +void* storage_new(const char* db_path, const char* name){ + negentropy::storage::BTreeMem* storage; +/* + auto env = lmdb::env::create(); + env.set_max_dbs(64); + env.open(db_path, 0); + + lmdb::dbi btreeDbi; + + { + auto txn = lmdb::txn::begin(env); + btreeDbi = negentropy::storage::BTreeMem::setupDB(txn, name); + txn.commit(); + } */ + //TODO: Finish constructor + storage = new negentropy::storage::BTreeMem(); + return storage; +} + +void* negentropy_new(void* storage, uint64_t frameSizeLimit){ + //TODO: Make these typecasts into macros?? + negentropy::storage::BTreeMem* lmdbStorage; + //TODO: reinterpret cast is risky, need to use more safe type conversion. + lmdbStorage = reinterpret_cast(storage); + + Negentropy* ne; + try{ + ne = new Negentropy(*lmdbStorage, frameSizeLimit); + }catch(negentropy::err e){ + //TODO:Find a way to return this error + return NULL; + } + return ne; +} + +size_t negentropy_initiate(void* negentropy, buffer* out){ + Negentropy* ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::string* output = new std::string(); + try { + *output = ngn_inst->initiate(); + std::cout << "output of initiate is, len:" << output->size() << ", output:"; + printHexString(std::string_view(*output)); + } catch(negentropy::err e){ + //TODO:Find a way to return this error + return 0; + } + memcpy( out->data, output->c_str() ,output->size()); + size_t outlen = output->size(); + delete output; + return outlen; +} + +void negentropy_setinitiator(void* negentropy){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + ngn_inst->setInitiator(); + +} + +bool storage_insert(void* storage, uint64_t createdAt, buffer* id){ + negentropy::storage::BTreeMem* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); + std::string_view data(reinterpret_cast< char const* >(id->data), id->len); + + std::cout << "inserting entry in storage, createdAt:" << createdAt << ",id:"; + printHexString(data); + + //TODO: Error handling. Is it required? + //How does out of memory get handled? + return lmdbStorage->insert(createdAt, data); +} + +bool storage_erase(void* storage, uint64_t createdAt, buffer* id){ + negentropy::storage::BTreeMem* lmdbStorage; + lmdbStorage = reinterpret_cast(storage); + std::string_view data(reinterpret_cast< char const* >(id->data), id->len); + + std::cout << "erasing entry from storage, createdAt:" << createdAt << ",id:"; + printHexString(data); + + //TODO: Error handling + return lmdbStorage->erase(createdAt, data); +} + +size_t reconcile(void* negentropy, buffer* query, buffer* output){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + std::string* out = new std::string(); + try { + *out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len)); + std::cout << "reconcile output of reconcile is, len:" << out->size() << ", output:"; + printHexString(std::string_view(*out)); + } catch(negentropy::err e){ + //TODO:Find a way to return this error + return 0; + } + memcpy( output->data, out->c_str() ,out->size()); + return out->size(); +} + +void transform(std::vector &from_ids, buffer* to_ids) +{ + for (int i=0; i < from_ids.size(); i ++){ + to_ids[i].len = from_ids[i].size(); + to_ids[i].data = (unsigned char*)from_ids[i].c_str(); + } +} + +int reconcile_with_ids(void* negentropy, buffer* query,reconcile_cbk cbk, char* outptr){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::optional out; + std::vector haveIds, needIds; + uint64_t have_ids_len, need_ids_len; + buffer* have_ids; + buffer* need_ids; + + try { + out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len), haveIds, needIds); + + have_ids_len = haveIds.size(); + need_ids_len = needIds.size(); + have_ids = (buffer*)malloc(have_ids_len*sizeof(buffer)); + need_ids = (buffer*)malloc(need_ids_len*sizeof(buffer)); + + std::cout << "have_ids_len:" << have_ids_len << "need_ids_len:" << need_ids_len << std::endl; + + transform(haveIds, have_ids); + transform(needIds, need_ids); + } catch(negentropy::err e){ + std::cout << "caught error "<< e.what() << std::endl; + //TODO:Find a way to return this error and cleanup partially allocated memory if any + return -1; + } + buffer output = {0,NULL}; + if (out) { + output.len = out.value().size(); + output.data = (unsigned char*)out.value().c_str(); + std::cout << "reconcile_with_ids output of reconcile is, len:" << out.value().size() << ", output:"; + printHexString(std::string_view(out.value())); + } + std::cout << "invoking callback" << std::endl; + std::flush(std::cout); + + cbk(have_ids, have_ids_len, need_ids, need_ids_len, &output, outptr); + std::cout << "invoked callback" << std::endl; + std::flush(std::cout); + + free(have_ids); + free(need_ids); + return 0; +} + +void transform_with_alloc(std::vector &from_ids, buffer* to_ids) +{ + for (int i=0; i < from_ids.size(); i ++){ + to_ids[i].data = (unsigned char*) malloc(from_ids[i].size()*sizeof(unsigned char)); + to_ids[i].len = from_ids[i].size(); + memcpy(to_ids[i].data, from_ids[i].c_str(),to_ids[i].len); + } +} + +void reconcile_with_ids_no_cbk(void* negentropy, buffer* query, result* result){ + Negentropy *ngn_inst; + ngn_inst = reinterpret_cast*>(negentropy); + + std::optional out; + std::vector haveIds, needIds; + try { + out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len), haveIds, needIds); + + result->have_ids_len = haveIds.size(); + result->need_ids_len = needIds.size(); + result->have_ids = (buffer*)malloc(result->have_ids_len*sizeof(buffer)); + result->need_ids = (buffer*)malloc(result->need_ids_len*sizeof(buffer)); + + std::cout << "have_ids_len:" << result->have_ids_len << "need_ids_len:" << result->need_ids_len << std::endl; + + transform_with_alloc(haveIds, result->have_ids); + transform_with_alloc(needIds, result->need_ids); + + } catch(negentropy::err e){ + std::cout << "caught error "<< e.what() << std::endl; + //TODO:Find a way to return this error and cleanup partially allocated memory if any + return ; + } + buffer output = {0,NULL}; + if (out) { + result->output.len = out.value().size(); + result->output.data = (unsigned char*)malloc(out.value().size()*sizeof(unsigned char)); + result->output.data = (unsigned char*)out.value().c_str(); + std::cout << "reconcile_with_ids output of reconcile is, len:" << out.value().size() << ", output:"; + printHexString(std::string_view(out.value())); + } + return ; +} + +//Note: This function assumes that all relevant heap memory is alloced and just tries to free +void free_result(result* r){ + free((void *) r->output.data); + + for (int i = 0; i < r->have_ids_len; i++) { + free((void *) r->have_ids[i].data); + } + free((void *)r->have_ids); + + for (int i = 0; i < r->need_ids_len; i++) { + free((void *) r->need_ids[i].data); + } + free((void *)r->need_ids); +} diff --git a/cpp/negentropy_wrapper.h b/cpp/negentropy_wrapper.h new file mode 100644 index 0000000..dd3fd29 --- /dev/null +++ b/cpp/negentropy_wrapper.h @@ -0,0 +1,50 @@ + +#ifndef _NEGENTROPY_WRAPPER_H +#define _NEGENTROPY_WRAPPER_H + +#ifdef __cplusplus +#define EXTERNC extern "C" +#else +#define EXTERNC +#endif + +typedef struct _buffer_{ + uint64_t len ; + unsigned char* data; +}buffer; + +typedef struct _result_ { + buffer output; + uint64_t have_ids_len; + uint64_t need_ids_len; + buffer* have_ids; + buffer* need_ids; +} result; + +//This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code. +//TODO: Do error handling by catching exceptions + +EXTERNC void* storage_new(const char* db_path, const char* name); + +EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit); + +EXTERNC size_t negentropy_initiate(void* negentropy, buffer* output); + +EXTERNC void negentropy_setinitiator(void* negentropy); + +EXTERNC bool storage_insert(void* storage, uint64_t createdAt, buffer* id); + +EXTERNC bool storage_erase(void* storage, uint64_t createdAt, buffer* id); + +EXTERNC size_t reconcile(void* negentropy, buffer* query, buffer* output); + +EXTERNC typedef void (*reconcile_cbk)(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output, char* outptr ); + +EXTERNC int reconcile_with_ids(void* negentropy, buffer* query, reconcile_cbk cbk, char* outptr); + +EXTERNC void reconcile_with_ids_no_cbk(void* negentropy, buffer* query, result* result); + +EXTERNC void free_result(result* result); + +#endif + diff --git a/cpp/vendor/lmdbxx b/cpp/vendor/lmdbxx new file mode 160000 index 0000000..af64901 --- /dev/null +++ b/cpp/vendor/lmdbxx @@ -0,0 +1 @@ +Subproject commit af649014456719b16100c7da31d46378c91b0e7b diff --git a/test/cpp/Makefile b/test/cpp/Makefile index db2070e..6b0d5c4 100644 --- a/test/cpp/Makefile +++ b/test/cpp/Makefile @@ -2,7 +2,7 @@ W = -Wall OPT = -g -O2 STD = -std=c++20 CXXFLAGS = $(STD) $(OPT) $(W) -fPIC $(XCXXFLAGS) -INCS = -I../../cpp/ -I./hoytech-cpp/ -Ilmdbxx/include/ +INCS = -I../../cpp/ -I./hoytech-cpp/ -I../cpp/vendor/lmdbxx/include/ #-I/opt/homebrew/include/ -L/opt/homebrew/lib/ DEPS = ../../cpp/negentropy.h ../../cpp/negentropy/* ../../cpp/negentropy/storage/* ../../cpp/negentropy/storage/btree/* diff --git a/test/cpp/lmdbxx b/test/cpp/lmdbxx deleted file mode 160000 index d649a58..0000000 --- a/test/cpp/lmdbxx +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d649a581d3cebfe7d8bd4d345bc2c1c4c2cc59a2