Build dynamic libraries for all platforms using vcpkg. (#61)

* Build dynamic libraries for all platforms using vcpkg.

* Cleanup CI.

.github/workflows/ci.yml

@@ -66,59 +66,6 @@ jobs:
         with:
           submodules: true
 
-      - name: Install build dependencies (Linux i386)
-        if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
-        run: |
-          sudo dpkg --add-architecture i386
-          sudo apt-fast update -qq
-          sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
-            --no-install-recommends -yq gcc-multilib g++-multilib \
-            libz-dev:i386 libbz2-dev:i386 libssl-dev:i386 liblz4-dev:i386
-          mkdir -p external/bin
-          cat << EOF > external/bin/gcc
-          #!/bin/bash
-          exec $(which gcc) -m32 "\$@"
-          EOF
-          cat << EOF > external/bin/g++
-          #!/bin/bash
-          exec $(which g++) -m32 "\$@"
-          EOF
-          chmod 755 external/bin/gcc external/bin/g++
-          echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
-
-      - name: Restore rocksdb from cache (Linux + macOS)
-        if: runner.os != 'Windows'
-        id: rocksdb-cache
-        uses: actions/cache@v4
-        with:
-          path: rocks-db-cache-${{ matrix.target.cpu }}
-          key: 'rocksdb-v1-${{ matrix.target.os }}-${{ matrix.target.cpu }}'
-
-      - name: Build and install rocksdb (Linux i386 + macOS)
-        # no librocksdb-dev:i386
-        if: (runner.os == 'Linux' && matrix.target.cpu == 'i386') || runner.os == 'Macos'
-        run: |
-          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_rocksdb.sh
-          bash build_rocksdb.sh rocks-db-cache-${{ matrix.target.cpu }}
-
-      - name: Install rocksdb (Linux amd64)
-        # mysterious illegal instruction error if we build our own librocksdb
-        if: runner.os == 'Linux' && matrix.target.cpu == 'amd64'
-        run: |
-         sudo apt-get -q update
-         sudo apt-get install -y librocksdb-dev
-
-      - name: MSYS2 (Windows i386)
-        if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
-        uses: msys2/setup-msys2@v2
-        with:
-          path-type: inherit
-          msystem: MINGW32
-          install: >-
-            base-devel
-            git
-            mingw-w64-i686-toolchain
-
       - name: MSYS2 (Windows amd64)
         if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
         uses: msys2/setup-msys2@v2
@@ -129,28 +76,6 @@ jobs:
             git
             mingw-w64-x86_64-toolchain
 
-      - name: Install DLL dependencies (Windows)
-        if: >
-          runner.os == 'Windows'
-        run: |
-          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
-            ROCKSDBSUB=x64
-          else
-            ROCKSDBSUB=x86
-          fi
-          DLLPATH="external/dlls-${{ matrix.target.cpu }}"
-          mkdir -p ${DLLPATH}
-
-          # ROCKSDB
-          ./scripts/build_dlls_windows.sh
-          cp ./build/librocksdb.dll "${DLLPATH}/librocksdb.dll"
-
-      - name: Path to cached dependencies (Windows)
-        if: >
-          runner.os == 'Windows'
-        run: |
-          echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH
-
       - name: Derive environment variables
         run: |
           if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
@@ -196,7 +121,7 @@ jobs:
 
           nim --version
           nimble --version
-          nimble install -y --depsOnly
+          nimble install -y
           nimble test
 
           # static linking is not supported on windows

rocksdb.nimble

@@ -22,9 +22,26 @@ task clean, "Remove temporary files":
   exec "make -C vendor/rocksdb clean"
 
 task test, "Run tests":
-  exec "nim c -r --threads:on tests/test_all.nim"
+  let runTests = "nim c -r --threads:on tests/test_all.nim"
+  when defined(linux):
+    exec "export LD_LIBRARY_PATH=build; " & runTests
+  when defined(macosx):
+    exec "export DYLD_LIBRARY_PATH=build; " & runTests
+  when defined(windows):
+    exec runTests
 
 task test_static, "Run tests after static linking dependencies":
-  when not defined(windows):
-    exec "scripts/build_static_deps.sh"
+  when defined(windows):
+    echo "Static linking is not supported on windows"
+    quit(1)
+
+  exec "scripts/build_static_deps.sh"
   exec "nim c -d:rocksdb_static_linking -r --threads:on tests/test_all.nim"
+
+before install:
+  when defined(linux):
+    exec "scripts/build_shared_deps_linux.sh"
+  when defined(macosx):
+    exec "scripts/build_shared_deps_osx.sh"
+  when defined(windows):
+    exec ".\\scripts\\build_dlls_windows.bat"

rocksdb/lib/librocksdb.nim

@@ -24,10 +24,6 @@
 
 {.push raises: [].}
 
-proc shouldUseNativeLinking(): bool {.compileTime.} =
-  when defined(linux):
-    return true
-
 type
   rocksdb_t* = object
   rocksdb_backup_engine_t* = object
@@ -108,10 +104,6 @@ when defined(rocksdb_static_linking):
   when defined(windows):
     {.passl: "-lshlwapi -lrpcrt4".}
 else:
-  when shouldUseNativeLinking():
-    {.pragma: importrocks, importc, cdecl.}
-    {.passl: "-lrocksdb".}
-  else:
-    {.pragma: importrocks, importc, cdecl, dynlib: librocksdb.}
+  {.pragma: importrocks, importc, cdecl, dynlib: librocksdb.}
 
 include ./rocksdb_gen.nim

rocksdb/options/tableopts.nim

@@ -26,11 +26,11 @@ type
     binarySearchAndHash =
       rocksdb_block_based_table_data_block_index_type_binary_search_and_hash
 
-proc createRibbon*(bitsPerKey: float, autoClose = false): FilterPolicyRef =
+proc createRibbon*(bitsPerKey: float): FilterPolicyRef =
   FilterPolicyRef(cPtr: rocksdb_filterpolicy_create_ribbon(bitsPerKey))
 
 proc createRibbonHybrid*(
-    bitsPerKey: float, bloomBeforeLevel: int = 0, autoClose = false
+    bitsPerKey: float, bloomBeforeLevel: int = 0
 ): FilterPolicyRef =
   FilterPolicyRef(
     cPtr: rocksdb_filterpolicy_create_ribbon_hybrid(bitsPerKey, bloomBeforeLevel.cint)

scripts/build_shared_deps_linux.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Nim-RocksDB
+# Copyright 2024 Status Research & Development GmbH
+# Licensed under either of
+#
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
+#
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")"/..
+
+REPO_DIR="${PWD}"
+BUILD_DEST="${REPO_DIR}/build/"
+
+
+git submodule update --init
+
+${REPO_DIR}/vendor/vcpkg/bootstrap-vcpkg.sh -disableMetrics
+
+${REPO_DIR}/vendor/vcpkg/vcpkg install rocksdb[lz4,zstd]:x64-linux-rocksdb --recurse --overlay-triplets=${REPO_DIR}/triplets
+
+mkdir -p "${BUILD_DEST}"
+
+cp "${REPO_DIR}/vendor/vcpkg/installed/x64-linux-rocksdb/lib/liblz4.so" "${BUILD_DEST}/"
+cp "${REPO_DIR}/vendor/vcpkg/installed/x64-linux-rocksdb/lib/libzstd.so" "${BUILD_DEST}/"
+cp "${REPO_DIR}/vendor/vcpkg/installed/x64-linux-rocksdb/lib/librocksdb.so" "${BUILD_DEST}/"

scripts/build_shared_deps_osx.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Nim-RocksDB
+# Copyright 2024 Status Research & Development GmbH
+# Licensed under either of
+#
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
+#
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")"/..
+
+REPO_DIR="${PWD}"
+BUILD_DEST="${REPO_DIR}/build/"
+
+
+git submodule update --init
+
+${REPO_DIR}/vendor/vcpkg/bootstrap-vcpkg.sh -disableMetrics
+
+${REPO_DIR}/vendor/vcpkg/vcpkg install rocksdb[lz4,zstd]:x64-osx-rocksdb --recurse --overlay-triplets=${REPO_DIR}/triplets
+
+mkdir -p "${BUILD_DEST}"
+
+cp "${REPO_DIR}/vendor/vcpkg/installed/x64-osx-rocksdb/lib/liblz4.dylib" "${BUILD_DEST}/"
+cp "${REPO_DIR}/vendor/vcpkg/installed/x64-osx-rocksdb/lib/libzstd.dylib" "${BUILD_DEST}/"
+cp "${REPO_DIR}/vendor/vcpkg/installed/x64-osx-rocksdb/lib/librocksdb.dylib" "${BUILD_DEST}/"

scripts/generate_wrapper.sh

@@ -11,6 +11,8 @@
 
 set -e
 
+nimble install c2nim
+
 cd "$(dirname "${BASH_SOURCE[0]}")"/..
 
 VENDOR_HEADER_FILE="vendor/rocksdb/include/rocksdb/c.h"
@@ -71,3 +73,5 @@ sed -i ':a;N;$!ba;s/#ifdef _WIN32\
 
 # generate nim wrapper
 c2nim ${OUTPUT_HEADER_FILE} --out:"${C2NIM_GENERATED_WRAPPER}"
+
+nimble format

tests/options/test_dbopts.nim

@@ -20,10 +20,9 @@ suite "DbOptionsRef Tests":
     dbOpts.maxOpenFiles = 10
     dbOpts.createMissingColumnFamilies = false
 
-    # TODO rocksdb 5.17.2 used in some tests has no getters for settings exposed!
-    # check:
-    #   dbOpts.maxOpenFiles == 10
-    #   not dbOpts.createMissingColumnFamilies
+    check:
+      dbOpts.maxOpenFiles == 10
+      not dbOpts.createMissingColumnFamilies
 
     dbOpts.close()
 
@@ -36,20 +35,19 @@ suite "DbOptionsRef Tests":
     dbOpts.close()
     check dbOpts.isClosed()
 
-  # This is currently failing in MacOS CI due to older version of RocksDb
-  # test "Test auto close enabled":
-  #   let
-  #     dbOpts = defaultDbOptions()
-  #     cache = cacheCreateLRU(1000, autoClose = true)
+  test "Test auto close enabled":
+    let
+      dbOpts = defaultDbOptions()
+      cache = cacheCreateLRU(1000, autoClose = true)
 
-  #   dbOpts.rowCache = cache
+    dbOpts.rowCache = cache
 
-  #   check:
-  #     dbOpts.isClosed() == false
-  #     cache.isClosed() == false
+    check:
+      dbOpts.isClosed() == false
+      cache.isClosed() == false
 
-  #   dbOpts.close()
+    dbOpts.close()
 
-  #   check:
-  #     dbOpts.isClosed() == true
-  #     cache.isClosed() == true
+    check:
+      dbOpts.isClosed() == true
+      cache.isClosed() == true

tests/options/test_tableopts.nim

@@ -36,45 +36,43 @@ suite "TableOptionsRef Tests":
     check tableOpts.isClosed()
 
   test "Test auto close enabled":
-    # TODO: enable filter policy once creating updated DLL build
     let
       tableOpts = defaultTableOptions()
       cache = cacheCreateLRU(1000, autoClose = true)
-      # filter = createRibbon(9.9, autoClose = true)
+      filter = createRibbon(9.9)
 
     tableOpts.blockCache = cache
-    # tableOpts.filterPolicy = filter
+    tableOpts.filterPolicy = filter
 
     check:
       tableOpts.isClosed() == false
       cache.isClosed() == false
-      # filter.isClosed() == true # closed because tableopts takes ownership
+      filter.isClosed() == true # closed because tableopts takes ownership
 
     tableOpts.close()
 
     check:
       tableOpts.isClosed() == true
       cache.isClosed() == true
-      # filter.isClosed() == true
+      filter.isClosed() == true
 
   test "Test auto close disabled":
-    # TODO: enable filter policy once creating updated DLL build
     let
       tableOpts = defaultTableOptions()
       cache = cacheCreateLRU(1000, autoClose = false)
-      # filter = createRibbon(9.9, autoClose = true)
+      filter = createRibbon(9.9)
 
     tableOpts.blockCache = cache
-    # tableOpts.filterPolicy = filter
+    tableOpts.filterPolicy = filter
 
     check:
       tableOpts.isClosed() == false
       cache.isClosed() == false
-      # filter.isClosed() == true # closed because tableopts takes ownership
+      filter.isClosed() == true # closed because tableopts takes ownership
 
     tableOpts.close()
 
     check:
       tableOpts.isClosed() == true
       cache.isClosed() == false
-      # filter.isClosed() == true
+      filter.isClosed() == true

tests/test_all.nim

@@ -25,4 +25,5 @@ import
   ./test_rocksdb,
   ./test_rocksiterator,
   ./test_sstfilewriter,
-  ./test_writebatch
+  ./test_writebatch,
+  ./test_transactiondb

triplets/x64-linux-rocksdb.cmake

@@ -0,0 +1,7 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+set(VCPKG_BUILD_TYPE release)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE dynamic)
+
+set(VCPKG_CMAKE_SYSTEM_NAME Linux)
+set(VCPKG_FIXUP_ELF_RPATH ON)

triplets/x64-osx-rocksdb.cmake

@@ -0,0 +1,7 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+set(VCPKG_BUILD_TYPE release)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE dynamic)
+
+set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
+set(VCPKG_OSX_ARCHITECTURES x86_64)

triplets/x86-windows-rocksdb.cmake

@@ -1,10 +0,0 @@
-set(VCPKG_TARGET_ARCHITECTURE x86)
-set(VCPKG_BUILD_TYPE release)
-
-if(${PORT} MATCHES "snappy|zlib|lz4|zstd")
-  set(VCPKG_CRT_LINKAGE static)
-  set(VCPKG_LIBRARY_LINKAGE static)
-else()
-  set(VCPKG_CRT_LINKAGE static)
-  set(VCPKG_LIBRARY_LINKAGE dynamic)
-endif()