Merge pull request #2 from waku-org/feat/c-wrapper

feat: c wrapper

.gitignore

@@ -0,0 +1,57 @@
+# Prerequisites
+*.d
+
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+
+# Linker output
+*.ilk
+*.map
+*.exp
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# Debug files
+*.dSYM/
+*.su
+*.idb
+*.pdb
+
+# Kernel Module Compile Results
+*.mod*
+*.cmd
+.tmp_versions/
+modules.order
+Module.symvers
+Mkfile.old
+dkms.conf
+
+# ignore vscode files
+.vscode/
+
+

.gitmodules

@@ -2,5 +2,5 @@
 	path = test/cpp/hoytech-cpp
 	url = https://github.com/hoytech/hoytech-cpp.git
 [submodule "test/cpp/lmdbxx"]
-	path = test/cpp/lmdbxx
+	path = cpp/vendor/lmdbxx
 	url = https://github.com/hoytech/lmdbxx.git

cpp/Makefile

@@ -0,0 +1,23 @@
+# Build a shared library of negentropy
+
+#TODO: Need to add compilation flags based on OS
+INCS	= -I./ -I/opt/homebrew/include/ -I./vendor/lmdbxx/include/
+TARGET  = libnegentropy.so
+
+.PHONY: all clean install-deps precompiled-header shared-lib
+
+all: precompiled-header shared-lib
+
+#TODO: Need to add compilation flags based on OS
+install-deps:
+	brew install lmdb openssl
+
+# Generate 'negentropy.h.gch'
+precompiled-header:
+	g++ -O0 --std=c++20 -Wall -fexceptions -g negentropy.h $(INCS)
+
+shared-lib:
+	g++ -O0 -g -std=c++20 $(INCS) -shared -fPIC -o $(TARGET) negentropy_wrapper.c -lcrypto -lssl -L/opt/homebrew/lib/
+
+clean:
+	rm -f $(TARGET) negentropy.h.gch libnegentropy.so

cpp/example/Makefile

@@ -0,0 +1,4 @@
+.PHONY: all
+
+all: 
+	g++ -g -O0 --std=c++20 test.c -I../ -lnegentropy -lcrypto -L../ -L/opt/homebrew/lib/ -Wc++11-narrowing

cpp/example/test.c

@@ -0,0 +1,123 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <time.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <iostream>
+#include <string>
+#include "../negentropy_wrapper.h"
+
+void printHexBuffer(buffer buf){
+    for (uint64_t i = 0; i < buf.len; ++i) {
+        printf("%0hhx", buf.data[i]);
+    }
+    printf("\n");
+}
+
+void rec_callback(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output){
+   printf("needIds count:%llu , haveIds count: %llu \n",need_ids_len, have_ids_len);
+
+   for (int i=0; i < need_ids_len ; i++) {
+      printf("need ID at %d :", i);
+      printHexBuffer(need_ids[i]);
+   }
+
+   for (int j=0; j < have_ids_len ; j++) {
+      printf("need ID at %d :", j);
+      printHexBuffer(have_ids[j]);
+   }
+}
+
+
+int main(){
+   void* st1 = storage_new("","");
+   if(st1 == NULL){
+    perror("failed to create storage");
+   }
+   void* ngn_inst1 = negentropy_new(st1, 153600);
+   if(ngn_inst1 == NULL){
+    perror("failed to create negentropy instance");
+   }
+
+      void* st2 = storage_new("","");
+   if(st2 == NULL){
+    perror("failed to create storage");
+   }
+   void* ngn_inst2 = negentropy_new(st2, 153600);
+   if(ngn_inst2 == NULL){
+    perror("failed to create negentropy instance");
+   }
+
+   unsigned char m1[] =  {0x6a, 0xdf, 0xaa, 0xe0, 0x31, 0xeb, 0x61, 0xa8, \
+                          0x3c, 0xff, 0x9c, 0xfd, 0xd2, 0xae, 0xf6, 0xed, \
+                          0x63, 0xda, 0xcf, 0xaa, 0x96, 0xd0, 0x51, 0x26, \
+                          0x7e, 0xf1, 0x0c, 0x8b, 0x61, 0xae, 0x35, 0xe9};//"61dfaae031eb61a83cff9cfdd2aef6ed63dacfaa96d051267ef10c8b61ae35e9";
+   buffer b1 ;
+   b1.len = 32;
+   b1.data = m1;
+
+   unsigned char m2[] = {0x28 ,0x79 ,0x8d ,0x29 ,0x5c ,0x30 ,0xc7 ,0xe6 \
+                  ,0xd9 ,0xa4 ,0xa9 ,0x6c ,0xdd ,0xa7 ,0xe0 ,0x20 \
+                  ,0xf7 ,0xaa ,0x71 ,0x68 ,0xcc ,0xe0 ,0x63 ,0x30 \
+                  ,0x2e ,0xd1 ,0x9b ,0x85 ,0x63 ,0x32 ,0x95 ,0x9e}; //28798d295c30c7e6d9a4a96cdda7e020f7aa7168cce063302ed19b856332959e
+   buffer b2 ;
+   b2.len = 32;
+   b2.data = m2; 
+
+   bool ret = storage_insert(st1,time(NULL),&b1);
+   if (ret){
+      printf("inserted hash successfully in st1\n");
+   }
+ 
+   ret = storage_insert(st2,time(NULL),&b2);
+   if (ret){
+      printf("inserted hash successfully in st2\n");
+   }
+
+   ret = storage_insert(st2,time(NULL),&b1);
+   if (ret){
+      printf("inserted hash successfully in st2\n");
+   }
+ 
+   buffer b4 ;
+   b4.len = 153600;
+   b4.data = (unsigned char*)malloc(153600); 
+
+   size_t outSize = negentropy_initiate(ngn_inst1, &b4);
+   if(outSize == 0){
+    perror("failed to initiate negentropy instance");
+   }
+   printf("initiated negentropy successfully with output of len %zu \n", outSize);
+   b4.len = outSize;
+
+   buffer b3 ;
+   b3.len = 153600;
+   b3.data = (unsigned char*)malloc(153600);
+
+   outSize = reconcile(ngn_inst2, &b4, &b3); 
+   if(outSize == 0){
+    perror("nothing to reconcile");
+   }
+   printf("reconcile returned with output of len %zu \n", outSize);
+   b3.len = outSize;
+
+   //outSize = reconcile_with_ids(ngn_inst1, &b3, &rec_callback);
+
+   result res;
+   reconcile_with_ids_no_cbk(ngn_inst1, &b3, &res);
+   printf("needIds count:%llu , haveIds count: %llu \n",res.need_ids_len, res.have_ids_len);
+
+   for (int i=0; i < res.need_ids_len ; i++) {
+      printf("need ID at %d :", i);
+      printHexBuffer(res.need_ids[i]);
+   }
+
+   for (int j=0; j < res.have_ids_len ; j++) {
+      printf("need ID at %d :", j);
+      printHexBuffer(res.have_ids[j]);
+   }
+
+   free(b3.data);
+   free(b4.data);
+}

cpp/negentropy.h

@@ -14,6 +14,7 @@
 #include <stdexcept>
 #include <optional>
 #include <bit>
+#include <iostream>
 
 #include "negentropy/encoding.h"
 #include "negentropy/types.h"
@@ -49,7 +50,7 @@ struct Negentropy {
 
         std::string output;
         output.push_back(PROTOCOL_VERSION);
-
+        std::cout << "storage size" << storage.size() << std::endl;
         output += splitRange(0, storage.size(), Bound(MAX_U64));
 
         return output;

cpp/negentropy_wrapper.c

@@ -0,0 +1,232 @@
+#include <iostream>
+#include <unordered_map>
+
+#include "negentropy.h"
+#include "negentropy/storage/BTreeMem.h"
+#include "negentropy_wrapper.h"
+
+//This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code.
+//TODO: Do error handling by catching exceptions
+
+void printHexString(std::string_view toPrint){
+    for (size_t i = 0; i < toPrint.size(); ++i) {
+        printf("%0hhx", toPrint[i]);
+    }
+    printf("\n");
+}
+
+void* storage_new(const char* db_path, const char* name){
+    negentropy::storage::BTreeMem* storage;
+/* 
+    auto env = lmdb::env::create();
+    env.set_max_dbs(64);
+    env.open(db_path, 0);
+
+    lmdb::dbi btreeDbi;
+
+    {
+        auto txn = lmdb::txn::begin(env);
+        btreeDbi = negentropy::storage::BTreeMem::setupDB(txn, name);
+        txn.commit();
+    } */
+    //TODO: Finish constructor 
+    storage = new negentropy::storage::BTreeMem();
+    return storage;
+}
+
+void* negentropy_new(void* storage, uint64_t frameSizeLimit){
+    //TODO: Make these typecasts into macros??
+    negentropy::storage::BTreeMem* lmdbStorage;
+    //TODO: reinterpret cast is risky, need to use more safe type conversion.
+    lmdbStorage = reinterpret_cast<negentropy::storage::BTreeMem*>(storage);
+
+    Negentropy<negentropy::storage::BTreeMem>* ne;
+    try{
+    ne = new Negentropy<negentropy::storage::BTreeMem>(*lmdbStorage, frameSizeLimit);
+    }catch(negentropy::err e){
+        //TODO:Find a way to return this error
+        return NULL;
+    }
+    return ne;
+}
+
+size_t negentropy_initiate(void* negentropy, buffer* out){
+    Negentropy<negentropy::storage::BTreeMem>* ngn_inst;
+    ngn_inst = reinterpret_cast<Negentropy<negentropy::storage::BTreeMem>*>(negentropy);
+
+    std::string* output = new std::string();
+    try {
+        *output = ngn_inst->initiate();
+        std::cout << "output of initiate is, len:" << output->size() << ", output:";
+        printHexString(std::string_view(*output));
+    } catch(negentropy::err e){
+        //TODO:Find a way to return this error
+        return 0;
+    }
+    memcpy( out->data, output->c_str() ,output->size());
+    size_t outlen = output->size();
+    delete output;
+    return outlen;
+}
+
+void negentropy_setinitiator(void* negentropy){
+    Negentropy<negentropy::storage::BTreeMem> *ngn_inst;
+    ngn_inst = reinterpret_cast<Negentropy<negentropy::storage::BTreeMem>*>(negentropy);
+
+    ngn_inst->setInitiator();
+
+}
+
+bool storage_insert(void* storage, uint64_t createdAt, buffer* id){
+    negentropy::storage::BTreeMem* lmdbStorage;
+    lmdbStorage = reinterpret_cast<negentropy::storage::BTreeMem*>(storage);
+    std::string_view data(reinterpret_cast< char const* >(id->data), id->len);
+    
+    std::cout << "inserting entry in storage, createdAt:" << createdAt << ",id:"; 
+    printHexString(data);
+    
+    //TODO: Error handling. Is it required?
+    //How does out of memory get handled?
+    return lmdbStorage->insert(createdAt, data);
+}
+
+bool storage_erase(void* storage, uint64_t createdAt, buffer* id){
+    negentropy::storage::BTreeMem* lmdbStorage;
+    lmdbStorage = reinterpret_cast<negentropy::storage::BTreeMem*>(storage);
+    std::string_view data(reinterpret_cast< char const* >(id->data), id->len);
+
+    std::cout << "erasing entry from storage, createdAt:" << createdAt << ",id:"; 
+    printHexString(data);
+    
+    //TODO: Error handling
+    return lmdbStorage->erase(createdAt, data);
+}
+
+size_t reconcile(void* negentropy, buffer* query, buffer* output){
+    Negentropy<negentropy::storage::BTreeMem> *ngn_inst;
+    ngn_inst = reinterpret_cast<Negentropy<negentropy::storage::BTreeMem>*>(negentropy);
+    std::string* out = new std::string();
+    try {
+        *out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len));
+        std::cout << "reconcile output of reconcile is, len:" << out->size() << ", output:";
+        printHexString(std::string_view(*out));
+    } catch(negentropy::err e){
+        //TODO:Find a way to return this error
+        return 0;
+    }
+    memcpy( output->data, out->c_str() ,out->size());
+    return out->size();
+}
+
+void transform(std::vector<std::string> &from_ids, buffer* to_ids)
+{
+   for (int i=0; i < from_ids.size(); i ++){    
+    to_ids[i].len = from_ids[i].size();
+    to_ids[i].data =  (unsigned char*)from_ids[i].c_str();
+   }
+}
+
+int reconcile_with_ids(void* negentropy, buffer*  query,reconcile_cbk cbk, char* outptr){
+    Negentropy<negentropy::storage::BTreeMem> *ngn_inst;
+    ngn_inst = reinterpret_cast<Negentropy<negentropy::storage::BTreeMem>*>(negentropy);
+
+    std::optional<std::string> out;
+    std::vector<std::string> haveIds, needIds;
+    uint64_t have_ids_len, need_ids_len;
+    buffer* have_ids;
+    buffer* need_ids;
+
+    try {
+        out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len), haveIds, needIds);
+
+        have_ids_len = haveIds.size();
+        need_ids_len = needIds.size();
+        have_ids = (buffer*)malloc(have_ids_len*sizeof(buffer));
+        need_ids = (buffer*)malloc(need_ids_len*sizeof(buffer));
+
+        std::cout << "have_ids_len:" << have_ids_len << "need_ids_len:" << need_ids_len << std::endl;
+
+        transform(haveIds, have_ids);
+        transform(needIds, need_ids);
+    } catch(negentropy::err e){
+        std::cout << "caught error "<< e.what() << std::endl;
+        //TODO:Find a way to return this error and cleanup partially allocated memory if any
+        return -1;
+    }
+    buffer output = {0,NULL};
+    if (out) {
+        output.len = out.value().size();
+        output.data = (unsigned char*)out.value().c_str();
+        std::cout << "reconcile_with_ids output of reconcile is, len:" << out.value().size() << ", output:";
+        printHexString(std::string_view(out.value()));
+    }
+    std::cout << "invoking callback" << std::endl;
+    std::flush(std::cout);
+
+    cbk(have_ids, have_ids_len, need_ids, need_ids_len, &output, outptr);
+    std::cout << "invoked callback" << std::endl;
+    std::flush(std::cout);
+
+    free(have_ids);
+    free(need_ids);
+    return 0;
+}   
+
+void transform_with_alloc(std::vector<std::string> &from_ids, buffer* to_ids)
+{
+   for (int i=0; i < from_ids.size(); i ++){
+    to_ids[i].data = (unsigned char*) malloc(from_ids[i].size()*sizeof(unsigned char));  
+    to_ids[i].len = from_ids[i].size();
+    memcpy(to_ids[i].data, from_ids[i].c_str(),to_ids[i].len);
+   }
+}
+
+void reconcile_with_ids_no_cbk(void* negentropy, buffer*  query, result* result){
+    Negentropy<negentropy::storage::BTreeMem> *ngn_inst;
+    ngn_inst = reinterpret_cast<Negentropy<negentropy::storage::BTreeMem>*>(negentropy);
+
+    std::optional<std::string> out;
+    std::vector<std::string> haveIds, needIds;
+    try {
+        out = ngn_inst->reconcile(std::string_view(reinterpret_cast< char const* >(query->data), query->len), haveIds, needIds);
+
+        result->have_ids_len = haveIds.size();
+        result->need_ids_len = needIds.size();
+        result->have_ids = (buffer*)malloc(result->have_ids_len*sizeof(buffer));
+        result->need_ids = (buffer*)malloc(result->need_ids_len*sizeof(buffer));
+
+        std::cout << "have_ids_len:" << result->have_ids_len << "need_ids_len:" << result->need_ids_len << std::endl;
+
+        transform_with_alloc(haveIds, result->have_ids);
+        transform_with_alloc(needIds, result->need_ids);
+
+    } catch(negentropy::err e){
+        std::cout << "caught error "<< e.what() << std::endl;
+        //TODO:Find a way to return this error and cleanup partially allocated memory if any
+        return ;
+    }
+    buffer output = {0,NULL};
+    if (out) {
+        result->output.len = out.value().size();
+        result->output.data = (unsigned char*)malloc(out.value().size()*sizeof(unsigned char));
+        result->output.data = (unsigned char*)out.value().c_str();
+        std::cout << "reconcile_with_ids output of reconcile is, len:" << out.value().size() << ", output:";
+        printHexString(std::string_view(out.value()));
+    }
+    return ;
+}
+
+//Note: This function assumes that all relevant heap memory is alloced and just tries to free
+void free_result(result* r){
+    free((void *) r->output.data);
+    
+    for (int i = 0; i < r->have_ids_len; i++) {
+        free((void *) r->have_ids[i].data);
+    }
+    free((void *)r->have_ids);
+
+    for (int i = 0; i < r->need_ids_len; i++) {
+        free((void *) r->need_ids[i].data);
+    }
+    free((void *)r->need_ids);
+}

cpp/negentropy_wrapper.h

@@ -0,0 +1,50 @@
+
+#ifndef _NEGENTROPY_WRAPPER_H
+#define _NEGENTROPY_WRAPPER_H
+
+#ifdef __cplusplus
+#define EXTERNC extern "C"
+#else
+#define EXTERNC
+#endif
+
+typedef struct _buffer_{
+    uint64_t len ;
+    unsigned char* data;
+}buffer;
+
+typedef struct _result_ {
+  buffer output;
+  uint64_t have_ids_len;
+  uint64_t need_ids_len;
+  buffer* have_ids;
+  buffer* need_ids;
+} result;
+
+//This is a C-wrapper for the C++ library that helps in integrating negentropy with nim code.
+//TODO: Do error handling by catching exceptions
+
+EXTERNC void* storage_new(const char* db_path, const char* name);
+
+EXTERNC void* negentropy_new(void* storage, uint64_t frameSizeLimit);
+
+EXTERNC size_t negentropy_initiate(void* negentropy, buffer* output);
+
+EXTERNC void negentropy_setinitiator(void* negentropy);
+
+EXTERNC bool storage_insert(void* storage, uint64_t createdAt, buffer* id);
+
+EXTERNC bool storage_erase(void* storage, uint64_t createdAt, buffer* id);
+
+EXTERNC size_t reconcile(void* negentropy, buffer* query, buffer* output);
+
+EXTERNC typedef void (*reconcile_cbk)(buffer* have_ids, uint64_t have_ids_len, buffer* need_ids, uint64_t need_ids_len, buffer* output, char* outptr );
+
+EXTERNC int reconcile_with_ids(void* negentropy, buffer*  query, reconcile_cbk cbk, char* outptr);
+
+EXTERNC void reconcile_with_ids_no_cbk(void* negentropy, buffer*  query, result* result);
+
+EXTERNC void free_result(result* result);
+
+#endif
+

cpp/vendor/lmdbxx

@@ -0,0 +1 @@
+Subproject commit af649014456719b16100c7da31d46378c91b0e7b

test/cpp/Makefile

@@ -2,7 +2,7 @@ W        = -Wall
 OPT      = -g -O2
 STD      = -std=c++20
 CXXFLAGS = $(STD) $(OPT) $(W) -fPIC $(XCXXFLAGS)
-INCS     = -I../../cpp/ -I./hoytech-cpp/ -Ilmdbxx/include/
+INCS     = -I../../cpp/ -I./hoytech-cpp/ -I../cpp/vendor/lmdbxx/include/ #-I/opt/homebrew/include/ -L/opt/homebrew/lib/
 
 DEPS = ../../cpp/negentropy.h ../../cpp/negentropy/* ../../cpp/negentropy/storage/* ../../cpp/negentropy/storage/btree/*
 

test/cpp/lmdbxx

@@ -1 +0,0 @@
-Subproject commit d649a581d3cebfe7d8bd4d345bc2c1c4c2cc59a2