Support for static linking RocksDb. (#40)

* Added rocksdb as a submodule in vendor directory.

* Add support for static linking using nimble.

* Add script to build static library dependencies.

* Disable warnings as error in deps build.

* Set recommended compression options in build and in library.

* Move static linking into wrapper code.

* Conditionally set linker using when defined(macosx).

* Add support for linking windows static libraries.

* Build rocksdb static libs using vcpkg.

* Remove lib prefix from library names for windows.

* Static linking is not supported on windows. Update documentation and CI.

.github/workflows/ci.yml

@@ -14,8 +14,8 @@ jobs:
         target:
           - os: linux
             cpu: amd64
-          - os: linux
+          # - os: linux
-            cpu: i386
+          #   cpu: i386
           - os: macos
             cpu: amd64
           - os: windows
@@ -34,7 +34,7 @@ jobs:
             shell: bash
           - target:
               os: windows
-            builder: windows-2019
+            builder: windows-latest
             shell: msys2 {0}
 
     defaults:
@@ -188,7 +188,13 @@ jobs:
             # https://github.com/status-im/nimbus-eth2/issues/3121
             export NIMFLAGS="-d:nimRawSetjmp"
           fi
+
           nim --version
           nimble --version
           nimble install -y --depsOnly
           nimble test
+
+          # static linking is not supported on windows
+          if [[ "${{ matrix.target.os }}" != "windows" ]]; then
+            nimble test_static
+          fi

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "vendor/rocksdb"]
+	path = vendor/rocksdb
+	url = https://github.com/facebook/rocksdb
+	ignore = dirty
+	branch = master

README.md

@@ -25,10 +25,10 @@ See [simple_example](examples/simple_example.nim)
 To statically link librocksdb, you would do something like:
 
 ```nim
-nim c -d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++ --threads:on your_program.nim
+nim c -d:rocksdb_static_linking --threads:on your_program.nim
 ```
 
-(we need the C++ linker profile because it's a C++ library)
+See the config.nims file which contains the static linking configuration which is switched on with the `rocksdb_static_linking` flag. Note that static linking is currently not supported on windows.
 
 ### Contribution
 

config.nims

@@ -11,3 +11,14 @@
 when fileExists("nimble.paths"):
   include "nimble.paths"
 # end Nimble config
+
+when defined(rocksdb_static_linking):
+  # use the C++ linker profile because it's a C++ library
+  when defined(macosx):
+    switch("clang.linkerexe", "clang++")
+  else:
+    switch("gcc.linkerexe", "g++")
+
+  switch("dynlibOverride", "rocksdb")
+  switch("dynlibOverride", "lz4")
+  switch("dynlibOverride", "zstd")

nim.cfg

@@ -1,2 +1,3 @@
--p:"src"
+--threads:on
-
+--outdir:build
+--hints:off

rocksdb.nim

@@ -8,8 +8,13 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  ./rocksdb/[backup, columnfamily, rocksdb, rocksiterator],
+  ./rocksdb/[backup,
-  ./rocksdb/[sstfilewriter, transactiondb, writebatch]
+  columnfamily,
+  rocksdb,
+  rocksiterator,
+  sstfilewriter,
+  transactiondb,
+  writebatch]
 
 export
   backup,

rocksdb.nimble

@@ -8,19 +8,18 @@ mode          = ScriptMode.Verbose
 
 ### Dependencies
 requires "nim >= 1.6",
-         "stew",
+         "results",
          "tempfile",
          "unittest2"
 
-proc test(args, path: string) =
+task clean, "Remove temporary files":
-  if not dirExists "build":
+  exec "rm -rf build"
-    mkDir "build"
+  exec "make -C vendor/rocksdb clean"
-  exec "nim " & getEnv("TEST_LANG", "c") & " " & getEnv("NIMFLAGS") & " " & args &
-    " --outdir:build -r --hints:off --threads:on --skipParentCfg " & path
 
 task test, "Run tests":
-  test "", "tests/test_all.nim"
+  exec "nim c -r --threads:on tests/test_all.nim"
-  # Too troublesome to install "librocksdb.a" in CI, but this is how we would
-  # test it (we need the C++ linker profile because it's a C++ library):
-  # test "-d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++", "tests/test_all.nim"
 
+task test_static, "Run tests after static linking dependencies":
+  when not defined(windows):
+    exec "scripts/build_static_deps.sh"
+  exec "nim c -d:rocksdb_static_linking -r --threads:on tests/test_all.nim"

rocksdb/columnfamily/cfopts.nim

@@ -34,6 +34,10 @@ proc setCreateMissingColumnFamilies*(cfOpts: ColFamilyOptionsRef, flag: bool) =
 
 proc defaultColFamilyOptions*(): ColFamilyOptionsRef =
   let opts = newColFamilyOptions()
+
+  rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression)
+  # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression)
+
   # Enable creating column families if they do not exist
   opts.setCreateMissingColumnFamilies(true)
   return opts

rocksdb/lib/librocksdb.nim

@@ -47,8 +47,6 @@ proc shouldUseNativeLinking(): bool {.compileTime.} =
   when defined(linux):
     return true
 
-const LibrocksbStaticArgs {.strdefine.}: string = ""
-
 type
   rocksdb_t* = object
   rocksdb_backup_engine_t* = object
@@ -114,11 +112,24 @@ type
 
 ##  DB operations
 
-when LibrocksbStaticArgs != "":
+when defined(rocksdb_static_linking):
   {.pragma: importrocks, importc, cdecl.}
-  {.passL: LibrocksbStaticArgs.}
+
+  import std/[os, strutils]
+  const
+    topLevelPath = currentSourcePath.parentDir().parentDir().parentDir()
+    libsDir = topLevelPath.replace('\\', '/') & "/build/lib"
+
   when defined(windows):
     {.passL: "-lshlwapi -lrpcrt4".}
+    {.passL: libsDir & "/rocksdb.lib".}
+    {.passL: libsDir & "/lz4.lib".}
+    {.passL: libsDir & "/zstd.lib".}
+  else:
+    {.passL: libsDir & "/librocksdb.a".}
+    {.passL: libsDir & "/liblz4.a".}
+    {.passL: libsDir & "/libzstd.a".}
+
 else:
   when shouldUseNativeLinking():
     {.pragma: importrocks, importc, cdecl.}

rocksdb/options/backupopts.nim

@@ -31,8 +31,11 @@ proc cPtr*(engineOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr =
 # TODO: Add setters and getters for backup options properties.
 
 proc defaultBackupEngineOptions*(): BackupEngineOptionsRef {.inline.} =
-  newBackupEngineOptions()
+  let opts = newBackupEngineOptions()
-  # TODO: set prefered defaults
+  rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression)
+  # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression)
+  opts
+
 
 proc close*(engineOpts: BackupEngineOptionsRef) =
   if not engineOpts.isClosed():

rocksdb/options/dbopts.nim

@@ -48,7 +48,11 @@ proc setCreateMissingColumnFamilies*(dbOpts: DbOptionsRef, flag: bool) =
   rocksdb_options_set_create_missing_column_families(dbOpts.cPtr, flag.uint8)
 
 proc defaultDbOptions*(): DbOptionsRef =
-  let opts = newDbOptions()
+  let opts: DbOptionsRef = newDbOptions()
+
+  rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression)
+  # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression)
+
   # Optimize RocksDB. This is the easiest way to get RocksDB to perform well:
   opts.setIncreaseParallelism(countProcessors())
   # This requires snappy - disabled because rocksdb is not always compiled with

scripts/build_static_deps.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Nim-RocksDB
+# Copyright 2018-2024 Status Research & Development GmbH
+# Licensed under either of
+#
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
+#
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")"/..
+
+REPO_DIR="${PWD}"
+ROCKSDB_LIB_DIR="${REPO_DIR}/vendor/rocksdb"
+BUILD_DEST="${REPO_DIR}/build/lib"
+
+
+
+[[ -z "$NPROC" ]] && NPROC=2 # number of CPU cores available
+
+git submodule update --init
+
+export DISABLE_WARNING_AS_ERROR=1
+
+export ROCKSDB_DISABLE_SNAPPY=1
+export ROCKSDB_DISABLE_ZLIB=1
+export ROCKSDB_DISABLE_BZIP=1
+
+export PORTABLE=1
+export DEBUG_LEVEL=0
+
+make -C "${ROCKSDB_LIB_DIR}" -j${NPROC} liblz4.a libzstd.a --no-print-directory > /dev/null
+
+export EXTRA_CFLAGS="-fpermissive -Wno-error -w -I${ROCKSDB_LIB_DIR}/lz4-1.9.4/lib -I${ROCKSDB_LIB_DIR}/zstd-1.5.5/lib -DLZ4 -DZSTD"
+export EXTRA_CXXFLAGS="-fpermissive -Wno-error -w -I${ROCKSDB_LIB_DIR}/lz4-1.9.4/lib -I${ROCKSDB_LIB_DIR}/zstd-1.5.5/lib -DLZ4 -DZSTD"
+
+make -C "${ROCKSDB_LIB_DIR}" -j${NPROC} static_lib --no-print-directory > /dev/null
+
+#cat "${REPO_DIR}/vendor/rocksdb/make_config.mk"
+
+mkdir -p "${BUILD_DEST}"
+
+cp "${ROCKSDB_LIB_DIR}/liblz4.a" "${BUILD_DEST}/"
+cp "${ROCKSDB_LIB_DIR}/libzstd.a" "${BUILD_DEST}/"
+cp "${ROCKSDB_LIB_DIR}/librocksdb.a" "${BUILD_DEST}/"

vendor/rocksdb

@@ -0,0 +1 @@
+Subproject commit a66daec5410683c0f66e7759b425631ffbfd8677