waku-sys new approach to allow smaller crate packages

* REAME.md explain how to build libwaku.a
* update nwaku vendor to use the optimized libwaku size changes
* remove vendor/vendor after creating the package
* Cargo.toml to reflect correct files
* persisting waku libraries, build-libwaku.sh script, and libwaku.h
* simplify build.rs accordingly

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "waku-sys/vendor"]
-	path = waku-sys/vendor
-	url = https://github.com/waku-org/nwaku
+[submodule "waku-sys/vendor/nwaku"]
+	path = waku-sys/vendor/nwaku
+	url = https://github.com/waku-org/nwaku.git

examples/Cargo.lock

@@ -4945,7 +4945,7 @@ checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
 
 [[package]]
 name = "waku-bindings"
-version = "0.5.0"
+version = "1.0.0"
 dependencies = [
  "aes-gcm",
  "base64 0.21.7",
@@ -4972,7 +4972,7 @@ dependencies = [
 
 [[package]]
 name = "waku-sys"
-version = "0.5.0"
+version = "1.0.0"
 dependencies = [
  "bindgen",
  "cc",

waku-sys/Cargo.toml

@@ -13,18 +13,16 @@ repository = "https://github.com/waku-org/waku-rust-bindings"
 keywords = ["waku", "peer-to-peer", "libp2p", "networking"]
 categories = ["network-programming"]
 
-exclude = [
-    "vendor/examples/*",
-    "vendor/docs/*",
-    "vendor/coverage/*",
-    "vendor/pkg/*",
-    "vendor/tests/*",
-    "vendor/ci/*",
-    "vendor/cmd/*",
-    "**/*.md",
-    "**/*.lock",
-    "**/*.nix",
-    "**/Dockerfile",
+include = [
+    "build.rs",
+    "src/cmd.c",
+    "src/lib.rs",
+    "libwaku/libwaku.a",
+    "libwaku/libwaku.h",
+    "libwaku/libbacktrace_wrapper.o",
+    "libwaku/libbacktrace.a",
+    "libwaku/libminiupnpc.a",
+    "libwaku/libnatpmp.a",
 ]
 
 [lib]

waku-sys/README.md

@@ -5,7 +5,7 @@
 [<img alt="docs.rs" src="https://img.shields.io/badge/doc/waku-bindings-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs" height="20">](https://docs.rs/waku-sys)
 [<img alt="build status" src="https://img.shields.io/github/actions/workflow/status/waku-org/waku-rust-bindings/main.yml?branch=master" height="20">](https://github.com/waku-org/waku-rust-bindings/actions/workflows/main.yml?query=branch%3Amaster)
 
-Rust layer on top of [`go-waku`](https://github.com/status-im/go-waku) [c ffi bindings](https://github.com/status-im/go-waku/blob/v0.2.2/library/README.md).
+Rust layer on top of [`nwaku`](https://github.com/status-im/go-waku) [c ffi bindings](https://github.com/waku-org/nwaku/blob/master/library/libwaku.h).
 
 ## Usage
 
@@ -18,7 +18,6 @@ Add this to your `Cargo.toml`:
 waku-sys = "0.1.0"
 ```
 
-
 ## About [Waku](https://waku.org/)
 
 Waku is the communication layer for Web3. Decentralized communication that scales.
@@ -46,6 +45,24 @@ The first version of Waku had its origins in the Whisper protocol, with optimiza
 
 This makes it ideal for running a p2p protocol on mobile, or in other similarly resource-restricted environments.
 
+Read the [Waku docs](https://docs.wakuconnect.dev/)
+
+## Compile waku-sys crate
+
+The waku-sys crate is a wrapper on top of [nwaku](https://github.com/waku-org/nwaku).
+This has nwaku a submodule within the vendor folder and, we need to manually compile the
+libwaku.a file. The build.rs file assumes the libwaku.a
+already exists.
+
+### Compile libwaku.a
+
+In Linux, do:
+
+```code
+bash build-libwaku.sh
+```
+and that should create the needed files in waku-sys/libwaku folder. Notice that these files should
+be committed in order to create a waku-sys package.
+
 
 
-Read the [Waku docs](https://docs.wakuconnect.dev/)

waku-sys/build-libwaku.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+## This is needed to manually build libwaku and copy the needed files
+## into the waku-sys/libwaku folder.
+## The waky-sys folder will take the files contained in waku-sys/libwaku/, which are git-versioned.
+
+cd vendor/nwaku/
+
+## Create vendor/nwaku/build/libwaku.a
+make POSTGRES=1 STATIC=true libwaku -j8
+
+echo "Stripping the library"
+## Reduce the libwaku.a size
+strip --strip-debug build/libwaku.a
+
+## Go back to the previous folder
+cd -
+
+echo "Copying generated files to waku-sys/libwaku"
+
+cp vendor/nwaku/build/libwaku.a libwaku/
+cp vendor/nwaku/library/libwaku.h libwaku/
+cp vendor/nwaku/vendor/nim-libbacktrace/libbacktrace_wrapper.o libwaku/
+cp vendor/nwaku/vendor/nim-libbacktrace/install/usr/lib/libbacktrace.a libwaku/
+cp vendor/nwaku/vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build/libminiupnpc.a libwaku/
+cp vendor/nwaku/vendor/nim-nat-traversal/vendor/libnatpmp-upstream/libnatpmp.a libwaku/
+

waku-sys/build.rs

@@ -1,69 +1,29 @@
 use std::env;
-use std::env::set_current_dir;
 use std::path::{Path, PathBuf};
-use std::process::Command;
 
 extern crate cc;
 
-fn build_nwaku_lib(project_dir: &Path) {
-    let vendor_path = project_dir.join("vendor");
-
-    set_current_dir(vendor_path).expect("Moving to vendor dir");
-
-    let mut cmd = Command::new("make");
-    cmd.arg("libwaku").arg("STATIC=true");
-    cmd.status()
-        .map_err(|e| println!("cargo:warning=make build failed due to: {e}"))
-        .unwrap();
-
-    set_current_dir(project_dir).expect("Going back to project dir");
-}
-
 fn generate_bindgen_code(project_dir: &Path) {
-    let vendor_path = project_dir.join("vendor");
-    let header_path = vendor_path.join("library/libwaku.h");
+    let libwaku_path = project_dir.join("libwaku/"); // dir with libs, objects and header
 
-    cc::Build::new()
-        .object(
-            vendor_path
-                .join("vendor/nim-libbacktrace/libbacktrace_wrapper.o")
-                .display()
-                .to_string(),
-        )
-        .compile("libbacktrace_wrapper");
-
-    println!("cargo:rerun-if-changed={}", header_path.display());
+    println!("cargo:rerun-if-changed={}", libwaku_path.display());
     println!(
         "cargo:rustc-link-search={}",
-        vendor_path.join("build").display()
+        libwaku_path.join("build").display()
     );
     println!("cargo:rustc-link-lib=static=waku");
 
-    println!(
-        "cargo:rustc-link-search={}",
-        vendor_path
-            .join("vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build")
-            .display()
-    );
+    println!("cargo:rustc-link-search={}", libwaku_path.display());
     println!("cargo:rustc-link-lib=static=miniupnpc");
 
-    println!(
-        "cargo:rustc-link-search={}",
-        vendor_path
-            .join("vendor/nim-nat-traversal/vendor/libnatpmp-upstream")
-            .display()
-    );
+    println!("cargo:rustc-link-search={}", libwaku_path.display());
+
     println!("cargo:rustc-link-lib=static=natpmp");
 
     println!("cargo:rustc-link-lib=dl");
     println!("cargo:rustc-link-lib=m");
 
-    println!(
-        "cargo:rustc-link-search=native={}",
-        vendor_path
-            .join("vendor/nim-libbacktrace/install/usr/lib")
-            .display()
-    );
+    println!("cargo:rustc-link-search=native={}", libwaku_path.display());
     println!("cargo:rustc-link-lib=static=backtrace");
 
     cc::Build::new()
@@ -75,7 +35,7 @@ fn generate_bindgen_code(project_dir: &Path) {
     let bindings = bindgen::Builder::default()
         // The input header we would like to generate
         // bindings for.
-        .header(format!("{}", header_path.display()))
+        .header(format!("{}", libwaku_path.join("libwaku.h").display()))
         // Tell cargo to invalidate the built crate whenever any of the
         // included header files changed.
         .parse_callbacks(Box::new(bindgen::CargoCallbacks))
@@ -93,8 +53,7 @@ fn generate_bindgen_code(project_dir: &Path) {
 
 #[cfg(not(doc))]
 fn main() {
+    // Notice that CARGO_MANIFEST_DIR is different depending on "cargo build" or "cargo package"
     let project_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
-
-    build_nwaku_lib(&project_dir);
     generate_bindgen_code(&project_dir);
 }

waku-sys/libwaku/libwaku.h

@@ -0,0 +1,225 @@
+
+// Generated manually and inspired by the one generated by the Nim Compiler.
+// In order to see the header file generated by Nim just run `make libwaku`
+// from the root repo folder and the header should be created in
+// nimcache/release/libwaku/libwaku.h
+#ifndef __libwaku__
+#define __libwaku__
+
+#include <stddef.h>
+#include <stdint.h>
+
+// The possible returned values for the functions that return int
+#define RET_OK                0
+#define RET_ERR               1
+#define RET_MISSING_CALLBACK  2
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void (*WakuCallBack) (int callerRet, const char* msg, size_t len, void* userData);
+
+// Creates a new instance of the waku node.
+// Sets up the waku node from the given configuration.
+// Returns a pointer to the Context needed by the rest of the API functions.
+void* waku_new(
+             const char* configJson,
+             WakuCallBack callback,
+             void* userData);
+
+int waku_start(void* ctx,
+               WakuCallBack callback,
+               void* userData);
+
+int waku_stop(void* ctx,
+              WakuCallBack callback,
+              void* userData);
+
+// Destroys an instance of a waku node created with waku_new
+int waku_destroy(void* ctx,
+                 WakuCallBack callback,
+                 void* userData);
+
+int waku_version(void* ctx,
+                 WakuCallBack callback,
+                 void* userData);
+
+void waku_set_event_callback(void* ctx,
+                             WakuCallBack callback,
+                             void* userData);
+
+int waku_content_topic(void* ctx,
+                       const char* appName,
+                       unsigned int appVersion,
+                       const char* contentTopicName,
+                       const char* encoding,
+                       WakuCallBack callback,
+                       void* userData);
+
+int waku_pubsub_topic(void* ctx,
+                      const char* topicName,
+                      WakuCallBack callback,
+                      void* userData);
+
+int waku_default_pubsub_topic(void* ctx,
+                              WakuCallBack callback,
+                              void* userData);
+
+int waku_relay_publish(void* ctx,
+                       const char* pubSubTopic,
+                       const char* jsonWakuMessage,
+                       unsigned int timeoutMs,
+                       WakuCallBack callback,
+                       void* userData);
+
+int waku_lightpush_publish(void* ctx,
+                       const char* pubSubTopic,
+                       const char* jsonWakuMessage,
+                       WakuCallBack callback,
+                       void* userData);
+
+int waku_relay_subscribe(void* ctx,
+                         const char* pubSubTopic,
+                         WakuCallBack callback,
+                         void* userData);
+
+int waku_relay_add_protected_shard(void* ctx,
+                                   int clusterId,
+                                   int shardId,
+                                   char* publicKey,
+                                   WakuCallBack callback,
+                                   void* userData);
+
+int waku_relay_unsubscribe(void* ctx,
+                           const char* pubSubTopic,
+                           WakuCallBack callback,
+                           void* userData);
+
+int waku_filter_subscribe(void* ctx,
+                          const char* pubSubTopic,
+                          const char* contentTopics,
+                          WakuCallBack callback,
+                          void* userData);
+
+int waku_filter_unsubscribe(void* ctx,
+                            const char* pubSubTopic,
+                            const char* contentTopics,
+                            WakuCallBack callback,
+                            void* userData);
+
+int waku_filter_unsubscribe_all(void* ctx,
+                                WakuCallBack callback,
+                                void* userData);
+
+int waku_relay_get_num_connected_peers(void* ctx,
+                           const char* pubSubTopic,
+                           WakuCallBack callback,
+                           void* userData);
+
+int waku_relay_get_num_peers_in_mesh(void* ctx,
+                           const char* pubSubTopic,
+                           WakuCallBack callback,
+                           void* userData);
+
+int waku_store_query(void* ctx,
+                        const char* jsonQuery,
+                        const char* peerAddr,
+                        int timeoutMs,
+                        WakuCallBack callback,
+                        void* userData);
+
+int waku_connect(void* ctx,
+                 const char* peerMultiAddr,
+                 unsigned int timeoutMs,
+                 WakuCallBack callback,
+                 void* userData);
+
+int waku_disconnect_peer_by_id(void* ctx,
+                 const char* peerId,
+                 WakuCallBack callback,
+                 void* userData);
+
+int waku_dial_peer(void* ctx,
+                 const char* peerMultiAddr,
+                 const char* protocol,
+                 int timeoutMs,
+                 WakuCallBack callback,
+                 void* userData);
+
+int waku_dial_peer_by_id(void* ctx,
+                 const char* peerId,
+                 const char* protocol,
+                 int timeoutMs,
+                 WakuCallBack callback,
+                 void* userData);
+
+int waku_get_peerids_from_peerstore(void* ctx,
+                                    WakuCallBack callback,
+                                    void* userData);
+
+int waku_get_peerids_by_protocol(void* ctx,
+                                 const char* protocol,
+                                 WakuCallBack callback,
+                                 void* userData);
+
+int waku_listen_addresses(void* ctx,
+                          WakuCallBack callback,
+                          void* userData);
+
+int waku_get_connected_peers(void* ctx,
+                          WakuCallBack callback,
+                          void* userData);
+
+// Returns a list of multiaddress given a url to a DNS discoverable ENR tree
+// Parameters
+//     char* entTreeUrl: URL containing a discoverable ENR tree
+//     char* nameDnsServer: The nameserver to resolve the ENR tree url.
+//     int timeoutMs: Timeout value in milliseconds to execute the call.
+int waku_dns_discovery(void* ctx,
+                       const char* entTreeUrl,
+                       const char* nameDnsServer,
+                       int timeoutMs,
+                       WakuCallBack callback,
+                       void* userData);
+
+// Updates the bootnode list used for discovering new peers via DiscoveryV5
+// bootnodes - JSON array containing the bootnode ENRs i.e. `["enr:...", "enr:..."]`
+int waku_discv5_update_bootnodes(void* ctx,
+                                 char* bootnodes,
+                                 WakuCallBack callback,
+                                 void* userData);
+
+int waku_start_discv5(void* ctx,
+                      WakuCallBack callback,
+                      void* userData);
+
+int waku_stop_discv5(void* ctx,
+                     WakuCallBack callback,
+                     void* userData);
+
+// Retrieves the ENR information
+int waku_get_my_enr(void* ctx,
+                    WakuCallBack callback,
+                    void* userData);
+
+int waku_get_my_peerid(void* ctx,
+                    WakuCallBack callback,
+                    void* userData);
+
+int waku_peer_exchange_request(void* ctx,
+                               int numPeers,
+                               WakuCallBack callback,
+                               void* userData);
+
+int waku_ping_peer(void* ctx,
+                        const char* peerAddr,
+                        int timeoutMs,
+                        WakuCallBack callback,
+                        void* userData);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __libwaku__ */

waku-sys/vendor

@@ -1 +0,0 @@
-Subproject commit 625c8ee51bc3e065da0e2e8d3a53d3634589f548

waku-sys/vendor/nwaku

@@ -0,0 +1 @@
+Subproject commit 9c0ad85179c35a94096347d17eee7c933f542069