From 96c9d37940e29e460b3a138839474ca77ddcc4b2 Mon Sep 17 00:00:00 2001 From: "Michael Bradley, Jr" Date: Fri, 23 Oct 2020 16:39:06 -0500 Subject: [PATCH] refactor: adjust Makefile, etc. so that static/shared lib linking works as expected Clean up logic, variables, etc. in the Makefile and related files. Bump nimterop and sqlcipher in `vendor/` to latest versions. In addition to tests on GitHub Actions, manually test all static/shared combinations on Linux, macOS, and Windows. Also, for each combination check locally that a separately compiled `sqlite3` cli *cannot* read e.g. `test/build/my.db` even with `PRAGMA key = '[password]';`. Likewise, check that a separately compiled `sqlcipher` cli *can* read a database if and only if the correct password is used, e.g. `PRAGMA key = 'qwerty';` for `test/build/my.db`. --- .github/workflows/test.yml | 115 +++++++------------------------------ Makefile | 97 ++++++++++++++++--------------- generator/generate.nim | 2 +- sqlcipher.nimble | 1 + test/db_smoke.nim | 4 +- vendor/nimterop | 2 +- vendor/sqlcipher | 2 +- 7 files changed, 77 insertions(+), 146 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f9af7aa..3643515 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,33 +56,16 @@ jobs: "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 deps - - name: Generate the sqlite.nim wrapper for SQLCipher (static) + - name: Build the sqlite.nim wrapper and run tests (static) shell: bash run: | [[ ${{ matrix.platform }} = macos* ]] && \ - SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \ - SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib + export SSL_INCLUDE_DIR=/usr/local/opt/openssl/include && \ + export SSL_LIB_DIR=/usr/local/opt/openssl/lib [[ ${{ matrix.platform }} = windows* ]] && \ export PATH="${PATH}:${HOME}/scoop/shims" && \ - SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ - SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" - export M="$(which mingw32-make || echo make)" - "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \ - SSL_INCLUDE_DIR="${SSL_INCLUDE_DIR}" \ - SSL_LIB_DIR="${SSL_LIB_DIR}" \ - V=1 \ - sqlite.nim - - - name: Run tests (static) - shell: bash - run: | - [[ ${{ matrix.platform }} = macos* ]] && \ - SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \ - SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib - [[ ${{ matrix.platform }} = windows* ]] && \ - export PATH="${PATH}:${HOME}/scoop/shims" && \ - SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ - SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" + export SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ + export SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" export M="$(which mingw32-make || echo make)" "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \ SSL_INCLUDE_DIR="${SSL_INCLUDE_DIR}" \ @@ -90,36 +73,16 @@ jobs: V=1 \ test - - name: Generate the sqlite.nim wrapper for SQLCipher (shared) - shell: bash - run: | - rm -rf nimcache sqlcipher sqlite test/build - [[ ${{ matrix.platform }} = macos* ]] && \ - SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \ - SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib - [[ ${{ matrix.platform }} = windows* ]] && \ - export PATH="${PATH}:${HOME}/scoop/shims" && \ - SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ - SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" - export M="$(which mingw32-make || echo make)" - "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \ - SQLITE_STATIC=false \ - SSL_INCLUDE_DIR="${SSL_INCLUDE_DIR}" \ - SSL_LIB_DIR="${SSL_LIB_DIR}" \ - SSL_STATIC=false \ - V=1 \ - sqlite.nim - - - name: Run tests (shared) + - name: Build the sqlite.nim wrapper and run tests (shared) shell: bash run: | [[ ${{ matrix.platform }} = macos* ]] && \ - SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \ - SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib + export SSL_INCLUDE_DIR=/usr/local/opt/openssl/include && \ + export SSL_LIB_DIR=/usr/local/opt/openssl/lib [[ ${{ matrix.platform }} = windows* ]] && \ export PATH="${PATH}:${HOME}/scoop/shims" && \ - SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ - SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" + export SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ + export SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" export M="$(which mingw32-make || echo make)" "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \ SQLITE_STATIC=false \ @@ -129,35 +92,16 @@ jobs: V=1 \ test - - name: "Generate the sqlite.nim wrapper for SQLCipher (mixed: shared SQLITE)" - shell: bash - run: | - rm -rf nimcache sqlcipher sqlite test/build - [[ ${{ matrix.platform }} = macos* ]] && \ - SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \ - SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib - [[ ${{ matrix.platform }} = windows* ]] && \ - export PATH="${PATH}:${HOME}/scoop/shims" && \ - SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ - SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" - export M="$(which mingw32-make || echo make)" - "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \ - SQLITE_STATIC=false \ - SSL_INCLUDE_DIR="${SSL_INCLUDE_DIR}" \ - SSL_LIB_DIR="${SSL_LIB_DIR}" \ - V=1 \ - sqlite.nim - - - name: "Run tests (mixed: shared SQLITE)" + - name: "Build the sqlite.nim wrapper and run tests (mixed: shared SQLITE)" shell: bash run: | [[ ${{ matrix.platform }} = macos* ]] && \ - SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \ - SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib + export SSL_INCLUDE_DIR=/usr/local/opt/openssl/include && \ + export SSL_LIB_DIR=/usr/local/opt/openssl/lib [[ ${{ matrix.platform }} = windows* ]] && \ export PATH="${PATH}:${HOME}/scoop/shims" && \ - SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ - SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" + export SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ + export SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" export M="$(which mingw32-make || echo make)" "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \ SQLITE_STATIC=false \ @@ -166,35 +110,16 @@ jobs: V=1 \ test - - name: "Generate the sqlite.nim wrapper for SQLCipher (mixed: shared SSL)" - shell: bash - run: | - rm -rf nimcache sqlcipher sqlite test/build - [[ ${{ matrix.platform }} = macos* ]] && \ - SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \ - SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib - [[ ${{ matrix.platform }} = windows* ]] && \ - export PATH="${PATH}:${HOME}/scoop/shims" && \ - SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ - SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" - export M="$(which mingw32-make || echo make)" - "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \ - SSL_INCLUDE_DIR="${SSL_INCLUDE_DIR}" \ - SSL_LIB_DIR="${SSL_LIB_DIR}" \ - SSL_STATIC=false \ - V=1 \ - sqlite.nim - - - name: "Run tests (mixed: shared SSL)" + - name: "Build the sqlite.nim wrapper and run tests (mixed: shared SSL)" shell: bash run: | [[ ${{ matrix.platform }} = macos* ]] && \ - SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \ - SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib + export SSL_INCLUDE_DIR=/usr/local/opt/openssl/include && \ + export SSL_LIB_DIR=/usr/local/opt/openssl/lib [[ ${{ matrix.platform }} = windows* ]] && \ export PATH="${PATH}:${HOME}/scoop/shims" && \ - SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ - SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" + export SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \ + export SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib" export M="$(which mingw32-make || echo make)" "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \ SSL_INCLUDE_DIR="${SSL_INCLUDE_DIR}" \ diff --git a/Makefile b/Makefile index 03a0cb2..a129540 100644 --- a/Makefile +++ b/Makefile @@ -67,17 +67,7 @@ deps: | deps-common update: | update-common -SQLITE_CDEFS ?= -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 -SQLITE_CFLAGS ?= -pthread -ifndef SQLITE_LDFLAGS - ifeq ($(detected_OS),Windows) - SQLITE_LDFLAGS := -lwinpthread - else - SQLITE_LDFLAGS := -lpthread - endif -endif -SQLITE_STATIC ?= true - +SSL_STATIC ?= true SSL_INCLUDE_DIR ?= /usr/include ifeq ($(SSL_INCLUDE_DIR),) override SSL_INCLUDE_DIR = /usr/include @@ -86,21 +76,32 @@ SSL_LIB_DIR ?= /usr/lib/x86_64-linux-gnu ifeq ($(SSL_LIB_DIR),) override SSL_LIB_DIR = /usr/lib/x86_64-linux-gnu endif - -SSL_CFLAGS ?= -I$(SSL_INCLUDE_DIR) -SSL_STATIC ?= true ifndef SSL_LDFLAGS ifeq ($(SSL_STATIC),false) SSL_LDFLAGS := -L$(SSL_LIB_DIR) -lcrypto + SSL_LDFLAGS_SQLITE3_C ?= $(SSL_LDFLAGS) else - SSL_LDFLAGS := -L$(SSL_LIB_DIR) $(SSL_LIB_DIR)/libcrypto.a + SSL_LDFLAGS := $(SSL_LIB_DIR)/libcrypto.a + # SQLCipher's configure script fails if SSL_LIB_DIR isn't supplied with -L in LDFLAGS + SSL_LDFLAGS_SQLITE3_C ?= -L$(SSL_LIB_DIR) $(SSL_LDFLAGS) endif ifeq ($(detected_OS),Windows) SSL_LDFLAGS += -lws2_32 endif endif -SQLITE3_C ?= sqlite/sqlite3.c +SQLITE_STATIC ?= true +SQLITE_CDEFS ?= -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3 +SQLITE_CFLAGS ?= -I$(SSL_INCLUDE_DIR) -pthread +ifndef SQLITE_LDFLAGS + ifeq ($(detected_OS),Windows) + SQLITE_LDFLAGS := -lwinpthread + else + SQLITE_LDFLAGS := -lpthread + endif +endif + +SQLITE3_C ?= $(shell pwd)/sqlite/sqlite3.c SQLITE3_H ?= $(shell pwd)/sqlite/sqlite3.h $(SQLITE3_C): | deps @@ -108,18 +109,21 @@ ifeq ($(detected_OS),Windows) sed -i "s/tr -d '\\\\\\n'/tr -d '\\\\\\r\\\\\\n'/" vendor/sqlcipher/configure endif echo -e $(BUILD_MSG) "SQLCipher's SQLite C amalgamation" - + cd vendor/sqlcipher && \ + + mkdir -p sqlite + cd vendor/sqlcipher && \ ./configure \ - CFLAGS="$(SQLITE_CDEFS) $(SQLITE_CFLAGS) $(SSL_CFLAGS)" \ - LDFLAGS="$(SQLITE_LDFLAGS) $(SSL_LDFLAGS)" \ + CFLAGS="$(SQLITE_CDEFS) $(SQLITE_CFLAGS)" \ + LDFLAGS="$(SQLITE_LDFLAGS) $(SSL_LDFLAGS_SQLITE3_C)" \ $(HANDLE_OUTPUT) ifeq ($(detected_OS),Windows) - sed -E -i "s/TOP = \\/([A-Za-z])/TOP = \\u\\1:/" vendor/sqlcipher/Makefile + sed -i "/TOP =/c\\\\\\TOP := \$$(shell cygpath -m \$$(shell pwd))" vendor/sqlcipher/Makefile + sed -i "s/\$$(TCLSH_CMD) \$$(TOP)\\/tool\\/mkshellc.tcl/\$$(TCLSH_CMD) \$$(shell pwd)\\/tool\\/mkshellc.tcl/" vendor/sqlcipher/Makefile endif cd vendor/sqlcipher && $(MAKE) sqlite3.c $(HANDLE_OUTPUT) - mkdir -p sqlite - cp vendor/sqlcipher/sqlite3.c sqlite/ - cp vendor/sqlcipher/sqlite3.h sqlite/ + cp \ + vendor/sqlcipher/sqlite3.c \ + vendor/sqlcipher/sqlite3.h \ + sqlite/ cd vendor/sqlcipher && \ git clean -dfx $(HANDLE_OUTPUT) && \ (git stash $(HANDLE_OUTPUT) || true) && \ @@ -127,18 +131,19 @@ endif sqlite3.c: $(SQLITE3_C) -SQLITE_STATIC_LIB ?= $(shell pwd)/sqlite/sqlite3.a +SQLITE_STATIC_LIB ?= $(shell pwd)/sqlcipher/sqlcipher.a +SQLITE_STATIC_OBJ ?= sqlcipher/sqlcipher.o $(SQLITE_STATIC_LIB): $(SQLITE3_C) echo -e $(BUILD_MSG) "SQLCipher static library" - + $(ENV_SCRIPT) $(CC) \ - -c \ - sqlite/sqlite3.c \ + + mkdir -p sqlcipher + $(ENV_SCRIPT) $(CC) \ $(SQLITE_CDEFS) \ $(SQLITE_CFLAGS) \ - $(SSL_CFLAGS) \ - -o sqlite/sqlite3.o $(HANDLE_OUTPUT) - $(ENV_SCRIPT) ar rcs $(SQLITE_STATIC_LIB) sqlite/sqlite3.o $(HANDLE_OUTPUT) + $(SQLITE3_C) \ + -c \ + -o $(SQLITE_STATIC_OBJ) $(HANDLE_OUTPUT) + $(ENV_SCRIPT) ar rcs $(SQLITE_STATIC_LIB) $(SQLITE_STATIC_OBJ) $(HANDLE_OUTPUT) ifndef SHARED_LIB_EXT ifeq ($(detected_OS),macOS) @@ -150,31 +155,26 @@ ifndef SHARED_LIB_EXT endif endif -SQLITE_SHARED_LIB ?= $(shell pwd)/sqlite/libsqlite3.$(SHARED_LIB_EXT) +SQLITE_SHARED_LIB ?= $(shell pwd)/sqlcipher/libsqlcipher.$(SHARED_LIB_EXT) -ifndef PLATFORM_FLAGS +ifndef PLATFORM_FLAGS_SHARED_LIB ifeq ($(detected_OS),macOS) - ifeq ($(SSL_STATIC),false) - PLATFORM_FLAGS := -shared -dylib -undefined dynamic_lookup - else - PLATFORM_FLAGS := -shared -dylib -undefined dynamic_lookup $(SSL_LDFLAGS) - endif - else ifeq ($(detected_OS),Windows) - PLATFORM_FLAGS := -shared $(SSL_LDFLAGS) + PLATFORM_FLAGS_SHARED_LIB := -shared -dylib else - PLATFORM_FLAGS := -shared -fPIC + PLATFORM_FLAGS_SHARED_LIB := -shared -fPIC endif endif $(SQLITE_SHARED_LIB): $(SQLITE3_C) echo -e $(BUILD_MSG) "SQLCipher shared library" - + $(ENV_SCRIPT) $(CC) \ + + mkdir -p sqlcipher + $(ENV_SCRIPT) $(CC) \ $(SQLITE_CDEFS) \ $(SQLITE_CFLAGS) \ - $(SSL_CFLAGS) \ - sqlite/sqlite3.c \ + $(SQLITE3_C) \ $(SQLITE_LDFLAGS) \ - $(PLATFORM_FLAGS) \ + $(SSL_LDFLAGS) \ + $(PLATFORM_FLAGS_SHARED_LIB) \ -o $(SQLITE_SHARED_LIB) $(HANDLE_OUTPUT) ifndef SQLITE_LIB @@ -219,7 +219,7 @@ $(SQLITE_NIM): $(NIMTEROP_TOAST) $(SQLITE_LIB) $(ENV_SCRIPT) nim c $(NIM_PARAMS) \ --nimcache:nimcache/sqlcipher \ --verbosity:0 \ - generator/generate.nim > sqlcipher/sqlite.nim 2> /dev/null + generator/generate.nim > $(SQLITE_NIM) 2> /dev/null rm -rf generator/generate generator/generate.exe generator/generate.dSYM sqlite.nim: $(SQLITE_NIM) @@ -227,14 +227,17 @@ sqlite.nim: $(SQLITE_NIM) test: $(SQLITE_NIM) ifeq ($(detected_OS),macOS) SSL_LDFLAGS="$(SSL_LDFLAGS)" \ + SSL_STATIC="$(SSL_STATIC)" \ $(ENV_SCRIPT) nimble tests else ifeq ($(detected_OS),Windows) - PATH="$(shell dirname $(SQLITE_SHARED_LIB)):$${PATH}" \ + PATH="$(shell dirname $(SQLITE_SHARED_LIB)):$(shell dirname $(SSL_LIB_DIR)):$(SSL_LIB_DIR):$${PATH}" \ SSL_LDFLAGS="$(SSL_LDFLAGS)" \ + SSL_STATIC="$(SSL_STATIC)" \ $(ENV_SCRIPT) nimble tests else - LD_LIBRARY_PATH="$(shell pwd)/sqlite$${LD_LIBRARY_PATH:+:$${LD_LIBRARY_PATH}}" \ + LD_LIBRARY_PATH="$(shell dirname $(SQLITE_SHARED_LIB)):$(SSL_LIB_DIR)$${LD_LIBRARY_PATH:+:$${LD_LIBRARY_PATH}}" \ SSL_LDFLAGS="$(SSL_LDFLAGS)" \ + SSL_STATIC="$(SSL_STATIC)" \ $(ENV_SCRIPT) nimble tests endif diff --git a/generator/generate.nim b/generator/generate.nim index 1c90b8c..1ea3b6f 100644 --- a/generator/generate.nim +++ b/generator/generate.nim @@ -24,7 +24,7 @@ static: dynamicCdefine() when getEnv("SQLITE_STATIC") == "false": - cPassL("-L" & splitPath($getEnv("SQLITE_LIB")).head & " " & "-lsqlite3") + cPassL("-L" & splitPath($getEnv("SQLITE_LIB")).head & " " & "-lsqlcipher") when getEnv("SQLITE_STATIC") != "false": cPassL($getEnv("SQLITE_LIB")) diff --git a/sqlcipher.nimble b/sqlcipher.nimble index 2195368..db674d4 100644 --- a/sqlcipher.nimble +++ b/sqlcipher.nimble @@ -28,6 +28,7 @@ proc buildAndRunTest(name: string, " --debugger:native" & " --define:debug" & " --define:ssl" & + (if getEnv("SSL_STATIC").strip != "false": " --dynlibOverride:ssl" else: "") & " --nimcache:nimcache/test/" & name & " --out:" & outDir & name & (if getEnv("SSL_LDFLAGS").strip != "": " --passL:\"" & getEnv("SSL_LDFLAGS") & "\"" else: "") & diff --git a/test/db_smoke.nim b/test/db_smoke.nim index b400ec1..cc66565 100644 --- a/test/db_smoke.nim +++ b/test/db_smoke.nim @@ -4,7 +4,7 @@ from os import parentDir import strformat import times -let db: DbConn = openDatabase(currentSourcePath.parentDir() & "/build/myDatabase") +let db: DbConn = openDatabase(currentSourcePath.parentDir() & "/build/my.db") let passwd = "qwerty" @@ -18,3 +18,5 @@ let time = getClockStr(now()) execScript(db, &"""insert into Log values("{date}:{time}")""") echo rows(db, "select * from Log") + +close(db) diff --git a/vendor/nimterop b/vendor/nimterop index 0c2ca16..f7cee5c 160000 --- a/vendor/nimterop +++ b/vendor/nimterop @@ -1 +1 @@ -Subproject commit 0c2ca16f7ad9b1798f1c28ca0a3268d98e845a8d +Subproject commit f7cee5c983650336f93fde5d4fea087863ac0e5e diff --git a/vendor/sqlcipher b/vendor/sqlcipher index 4a81bea..87b4a1e 160000 --- a/vendor/sqlcipher +++ b/vendor/sqlcipher @@ -1 +1 @@ -Subproject commit 4a81bea61e1da6fec222d713852830f1fd01aed2 +Subproject commit 87b4a1ea57827bbf1177bc6a472590ea2af4b8c3