From 90027f187515d760ae4119032ecdcc82368827a5 Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:18:21 +0800 Subject: [PATCH] Support for static linking RocksDb. (#40) * Added rocksdb as a submodule in vendor directory. * Add support for static linking using nimble. * Add script to build static library dependencies. * Disable warnings as error in deps build. * Set recommended compression options in build and in library. * Move static linking into wrapper code. * Conditionally set linker using when defined(macosx). * Add support for linking windows static libraries. * Build rocksdb static libs using vcpkg. * Remove lib prefix from library names for windows. * Static linking is not supported on windows. Update documentation and CI. --- .github/workflows/ci.yml | 12 ++++++--- .gitmodules | 5 ++++ README.md | 4 +-- config.nims | 11 ++++++++ nim.cfg | 5 ++-- rocksdb.nim | 9 +++++-- rocksdb.nimble | 19 +++++++------ rocksdb/columnfamily/cfopts.nim | 4 +++ rocksdb/lib/librocksdb.nim | 19 ++++++++++--- rocksdb/options/backupopts.nim | 7 +++-- rocksdb/options/dbopts.nim | 6 ++++- scripts/build_static_deps.sh | 48 +++++++++++++++++++++++++++++++++ vendor/rocksdb | 1 + 13 files changed, 124 insertions(+), 26 deletions(-) create mode 100644 .gitmodules create mode 100755 scripts/build_static_deps.sh create mode 160000 vendor/rocksdb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2de3f9b..a79970d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,8 @@ jobs: target: - os: linux cpu: amd64 - - os: linux - cpu: i386 + # - os: linux + # cpu: i386 - os: macos cpu: amd64 - os: windows @@ -34,7 +34,7 @@ jobs: shell: bash - target: os: windows - builder: windows-2019 + builder: windows-latest shell: msys2 {0} defaults: @@ -188,7 +188,13 @@ jobs: # https://github.com/status-im/nimbus-eth2/issues/3121 export NIMFLAGS="-d:nimRawSetjmp" fi + nim --version nimble --version nimble install -y --depsOnly nimble test + + # static linking is not supported on windows + if [[ "${{ matrix.target.os }}" != "windows" ]]; then + nimble test_static + fi diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8231cca --- /dev/null +++ b/.gitmodules @@ -0,0 +1,5 @@ +[submodule "vendor/rocksdb"] + path = vendor/rocksdb + url = https://github.com/facebook/rocksdb + ignore = dirty + branch = master diff --git a/README.md b/README.md index cb10454..368d6c8 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ See [simple_example](examples/simple_example.nim) To statically link librocksdb, you would do something like: ```nim -nim c -d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++ --threads:on your_program.nim +nim c -d:rocksdb_static_linking --threads:on your_program.nim ``` -(we need the C++ linker profile because it's a C++ library) +See the config.nims file which contains the static linking configuration which is switched on with the `rocksdb_static_linking` flag. Note that static linking is currently not supported on windows. ### Contribution diff --git a/config.nims b/config.nims index 1f5d13e..4850a6e 100644 --- a/config.nims +++ b/config.nims @@ -11,3 +11,14 @@ when fileExists("nimble.paths"): include "nimble.paths" # end Nimble config + +when defined(rocksdb_static_linking): + # use the C++ linker profile because it's a C++ library + when defined(macosx): + switch("clang.linkerexe", "clang++") + else: + switch("gcc.linkerexe", "g++") + + switch("dynlibOverride", "rocksdb") + switch("dynlibOverride", "lz4") + switch("dynlibOverride", "zstd") diff --git a/nim.cfg b/nim.cfg index 4e7b49d..df3b973 100644 --- a/nim.cfg +++ b/nim.cfg @@ -1,2 +1,3 @@ --p:"src" - +--threads:on +--outdir:build +--hints:off diff --git a/rocksdb.nim b/rocksdb.nim index 774ae8b..c857a1d 100644 --- a/rocksdb.nim +++ b/rocksdb.nim @@ -8,8 +8,13 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import - ./rocksdb/[backup, columnfamily, rocksdb, rocksiterator], - ./rocksdb/[sstfilewriter, transactiondb, writebatch] + ./rocksdb/[backup, + columnfamily, + rocksdb, + rocksiterator, + sstfilewriter, + transactiondb, + writebatch] export backup, diff --git a/rocksdb.nimble b/rocksdb.nimble index b2e82ea..92ca2bc 100644 --- a/rocksdb.nimble +++ b/rocksdb.nimble @@ -8,19 +8,18 @@ mode = ScriptMode.Verbose ### Dependencies requires "nim >= 1.6", - "stew", + "results", "tempfile", "unittest2" -proc test(args, path: string) = - if not dirExists "build": - mkDir "build" - exec "nim " & getEnv("TEST_LANG", "c") & " " & getEnv("NIMFLAGS") & " " & args & - " --outdir:build -r --hints:off --threads:on --skipParentCfg " & path +task clean, "Remove temporary files": + exec "rm -rf build" + exec "make -C vendor/rocksdb clean" task test, "Run tests": - test "", "tests/test_all.nim" - # Too troublesome to install "librocksdb.a" in CI, but this is how we would - # test it (we need the C++ linker profile because it's a C++ library): - # test "-d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++", "tests/test_all.nim" + exec "nim c -r --threads:on tests/test_all.nim" +task test_static, "Run tests after static linking dependencies": + when not defined(windows): + exec "scripts/build_static_deps.sh" + exec "nim c -d:rocksdb_static_linking -r --threads:on tests/test_all.nim" diff --git a/rocksdb/columnfamily/cfopts.nim b/rocksdb/columnfamily/cfopts.nim index 174365c..baef797 100644 --- a/rocksdb/columnfamily/cfopts.nim +++ b/rocksdb/columnfamily/cfopts.nim @@ -34,6 +34,10 @@ proc setCreateMissingColumnFamilies*(cfOpts: ColFamilyOptionsRef, flag: bool) = proc defaultColFamilyOptions*(): ColFamilyOptionsRef = let opts = newColFamilyOptions() + + rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression) + # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression) + # Enable creating column families if they do not exist opts.setCreateMissingColumnFamilies(true) return opts diff --git a/rocksdb/lib/librocksdb.nim b/rocksdb/lib/librocksdb.nim index b64faef..4d83c33 100644 --- a/rocksdb/lib/librocksdb.nim +++ b/rocksdb/lib/librocksdb.nim @@ -47,8 +47,6 @@ proc shouldUseNativeLinking(): bool {.compileTime.} = when defined(linux): return true -const LibrocksbStaticArgs {.strdefine.}: string = "" - type rocksdb_t* = object rocksdb_backup_engine_t* = object @@ -114,11 +112,24 @@ type ## DB operations -when LibrocksbStaticArgs != "": +when defined(rocksdb_static_linking): {.pragma: importrocks, importc, cdecl.} - {.passL: LibrocksbStaticArgs.} + + import std/[os, strutils] + const + topLevelPath = currentSourcePath.parentDir().parentDir().parentDir() + libsDir = topLevelPath.replace('\\', '/') & "/build/lib" + when defined(windows): {.passL: "-lshlwapi -lrpcrt4".} + {.passL: libsDir & "/rocksdb.lib".} + {.passL: libsDir & "/lz4.lib".} + {.passL: libsDir & "/zstd.lib".} + else: + {.passL: libsDir & "/librocksdb.a".} + {.passL: libsDir & "/liblz4.a".} + {.passL: libsDir & "/libzstd.a".} + else: when shouldUseNativeLinking(): {.pragma: importrocks, importc, cdecl.} diff --git a/rocksdb/options/backupopts.nim b/rocksdb/options/backupopts.nim index 444ded5..4fdc0ee 100644 --- a/rocksdb/options/backupopts.nim +++ b/rocksdb/options/backupopts.nim @@ -31,8 +31,11 @@ proc cPtr*(engineOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr = # TODO: Add setters and getters for backup options properties. proc defaultBackupEngineOptions*(): BackupEngineOptionsRef {.inline.} = - newBackupEngineOptions() - # TODO: set prefered defaults + let opts = newBackupEngineOptions() + rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression) + # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression) + opts + proc close*(engineOpts: BackupEngineOptionsRef) = if not engineOpts.isClosed(): diff --git a/rocksdb/options/dbopts.nim b/rocksdb/options/dbopts.nim index 738b2a3..5be51b3 100644 --- a/rocksdb/options/dbopts.nim +++ b/rocksdb/options/dbopts.nim @@ -48,7 +48,11 @@ proc setCreateMissingColumnFamilies*(dbOpts: DbOptionsRef, flag: bool) = rocksdb_options_set_create_missing_column_families(dbOpts.cPtr, flag.uint8) proc defaultDbOptions*(): DbOptionsRef = - let opts = newDbOptions() + let opts: DbOptionsRef = newDbOptions() + + rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression) + # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression) + # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: opts.setIncreaseParallelism(countProcessors()) # This requires snappy - disabled because rocksdb is not always compiled with diff --git a/scripts/build_static_deps.sh b/scripts/build_static_deps.sh new file mode 100755 index 0000000..8d617ff --- /dev/null +++ b/scripts/build_static_deps.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Nim-RocksDB +# Copyright 2018-2024 Status Research & Development GmbH +# Licensed under either of +# +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +# +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")"/.. + +REPO_DIR="${PWD}" +ROCKSDB_LIB_DIR="${REPO_DIR}/vendor/rocksdb" +BUILD_DEST="${REPO_DIR}/build/lib" + + + +[[ -z "$NPROC" ]] && NPROC=2 # number of CPU cores available + +git submodule update --init + +export DISABLE_WARNING_AS_ERROR=1 + +export ROCKSDB_DISABLE_SNAPPY=1 +export ROCKSDB_DISABLE_ZLIB=1 +export ROCKSDB_DISABLE_BZIP=1 + +export PORTABLE=1 +export DEBUG_LEVEL=0 + +make -C "${ROCKSDB_LIB_DIR}" -j${NPROC} liblz4.a libzstd.a --no-print-directory > /dev/null + +export EXTRA_CFLAGS="-fpermissive -Wno-error -w -I${ROCKSDB_LIB_DIR}/lz4-1.9.4/lib -I${ROCKSDB_LIB_DIR}/zstd-1.5.5/lib -DLZ4 -DZSTD" +export EXTRA_CXXFLAGS="-fpermissive -Wno-error -w -I${ROCKSDB_LIB_DIR}/lz4-1.9.4/lib -I${ROCKSDB_LIB_DIR}/zstd-1.5.5/lib -DLZ4 -DZSTD" + +make -C "${ROCKSDB_LIB_DIR}" -j${NPROC} static_lib --no-print-directory > /dev/null + +#cat "${REPO_DIR}/vendor/rocksdb/make_config.mk" + +mkdir -p "${BUILD_DEST}" + +cp "${ROCKSDB_LIB_DIR}/liblz4.a" "${BUILD_DEST}/" +cp "${ROCKSDB_LIB_DIR}/libzstd.a" "${BUILD_DEST}/" +cp "${ROCKSDB_LIB_DIR}/librocksdb.a" "${BUILD_DEST}/" diff --git a/vendor/rocksdb b/vendor/rocksdb new file mode 160000 index 0000000..a66daec --- /dev/null +++ b/vendor/rocksdb @@ -0,0 +1 @@ +Subproject commit a66daec5410683c0f66e7759b425631ffbfd8677