refactor: adapt repo to use the nimbus-build-system

An important aspect of this refactor is that a user of this library needs to
supply openssl with respect to building the `sqlite3.c` amalgamation and the
`sqlite.nim` wrapper.

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/workflows/test.yml

@@ -0,0 +1,91 @@
+name: Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  tests:
+    strategy:
+      matrix:
+        env:
+          - { NPROC: 2 }
+        platform: [macos-latest, ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.platform }}
+    env: ${{ matrix.env }}
+
+    name: ${{ matrix.platform }} - ${{ matrix.env.NPROC }} processes
+
+    steps:
+      - name: Install Scoop
+        if: startsWith(matrix.platform, 'windows')
+        shell: powershell
+        run: |
+            iwr -useb get.scoop.sh | iex
+
+      - name: Install external dependencies with Scoop
+        if: startsWith(matrix.platform, 'windows')
+        shell: bash
+        run: |
+          export PATH="${PATH}:${HOME}/scoop/shims"
+          scoop install openssl-mingw
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      # We need to do this because of how github cache works
+      - name: Initialize submodules
+        shell: bash
+        run: |
+          git submodule update --init --recursive
+
+      - name: Cache Nim compiler
+        uses: actions/cache@v2
+        with:
+          path: |
+            vendor/nimbus-build-system/vendor/Nim/bin
+          key: ${{ runner.os }}-${{ matrix.env.NPROC }}-nim-${{ hashFiles('.gitmodules') }}
+
+      - name: Install and build dependencies
+        shell: bash
+        run: |
+          [[ ${{ matrix.platform }} = windows* ]] && export PATH="${PATH}:${HOME}/scoop/shims"
+          export M="$(which mingw32-make || echo make)"
+          "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update
+          "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 deps
+
+      - name: Generate the sqlite.nim wrapper for SQLCipher
+        shell: bash
+        run: |
+          [[ ${{ matrix.platform }} = macos* ]] && \
+            SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \
+            SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib
+          [[ ${{ matrix.platform }} = windows* ]] && \
+            export PATH="${PATH}:${HOME}/scoop/shims" && \
+            SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \
+            SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib"
+          export M="$(which mingw32-make || echo make)"
+          "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \
+                            SSL_INCLUDE_DIR="${SSL_INCLUDE_DIR}" \
+                            SSL_LIB_DIR="${SSL_LIB_DIR}" \
+                            V=1 \
+                            sqlite.nim
+
+      - name: Run tests
+        shell: bash
+        run: |
+          [[ ${{ matrix.platform }} = macos* ]] && \
+            SSL_INCLUDE_DIR=/usr/local/opt/openssl@1.1/include && \
+            SSL_LIB_DIR=/usr/local/opt/openssl@1.1/lib
+          [[ ${{ matrix.platform }} = windows* ]] && \
+            export PATH="${PATH}:${HOME}/scoop/shims" && \
+            SSL_INCLUDE_DIR="${HOME}/scoop/apps/openssl-mingw/current/include" && \
+            SSL_LIB_DIR="${HOME}/scoop/apps/openssl-mingw/current/lib"
+          export M="$(which mingw32-make || echo make)"
+          "${M}" -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" \
+                            SSL_INCLUDE_DIR="${SSL_INCLUDE_DIR}" \
+                            SSL_LIB_DIR="${SSL_LIB_DIR}" \
+                            V=1 \
+                            test

.gitignore

@@ -1,5 +1,13 @@
-nimcache/
-nimblecache/
-htmldocs/
-main
-myDatabase
+.DS_Store
+.idea
+.vscode
+/generator/*
+!/generator/generate.nim
+/nimcache
+/sqlcipher
+/sqlite
+/test/build
+/update
+/update.exe
+/vendor/.nimble
+TODO

.gitmodules

@@ -0,0 +1,24 @@
+[submodule "vendor/nimbus-build-system"]
+	path = vendor/nimbus-build-system
+	url = https://github.com/status-im/nimbus-build-system.git
+[submodule "vendor/sqlcipher"]
+	path = vendor/sqlcipher
+	url = https://github.com/sqlcipher/sqlcipher.git
+[submodule "vendor/nimterop"]
+	path = vendor/nimterop
+	url = https://github.com/nimterop/nimterop.git
+[submodule "vendor/nim-regex"]
+	path = vendor/nim-regex
+	url = https://github.com/nitely/nim-regex.git
+[submodule "vendor/nim-unicodedb"]
+	path = vendor/nim-unicodedb
+	url = https://github.com/nitely/nim-unicodedb.git
+[submodule "vendor/nim-unicodeplus"]
+	path = vendor/nim-unicodeplus
+	url = https://github.com/nitely/nim-unicodeplus.git
+[submodule "vendor/nim-segmentation"]
+	path = vendor/nim-segmentation
+	url = https://github.com/nitely/nim-segmentation.git
+[submodule "vendor/cligen"]
+	path = vendor/cligen
+	url = https://github.com/c-blake/cligen.git

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Status
+Copyright (c) 2020 Status Research & Development GmbH
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,225 @@
+# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+SHELL := bash # the shell used internally by Make
+
+# used inside the included makefiles
+BUILD_SYSTEM_DIR := vendor/nimbus-build-system
+
+# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
+-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
+
+.PHONY: \
+	all \
+	clean \
+	deps \
+	sqlite \
+	sqlite.nim \
+	sqlite3.c \
+	test \
+	toast \
+	update
+
+ifeq ($(NIM_PARAMS),)
+# "variables.mk" was not included, so we update the submodules.
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+.DEFAULT:
+	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+		$(GIT_SUBMODULE_UPDATE); \
+		echo
+# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
+# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+#
+# After restarting, it will execute its original goal, so we don't have to start a child Make here
+# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
+
+else # "variables.mk" was included. Business as usual until the end of this file.
+
+all: sqlite.nim
+
+# must be included after the default target
+-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
+
+ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
+ detected_OS := Windows
+else ifeq ($(strip $(shell uname)),Darwin)
+ detected_OS := macOS
+else
+ detected_OS := $(strip $(shell uname)) # e.g. Linux
+endif
+
+clean: | clean-common
+	rm -rf \
+		$(NIMTEROP_TOAST) \
+		$(NIMTEROP_TOAST).dSYM \
+		generator/generate \
+		generator/generate.exe \
+		generator/generate.dSYM \
+		sqlcipher \
+		sqlite \
+		test/build
+
+deps: | deps-common
+
+update: | update-common
+
+SQLITE_CDEFS ?= -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=3
+SQLITE_CFLAGS ?= -pthread
+ifndef SQLITE_LDFLAGS
+ ifneq ($(detected_OS),macOS)
+  SQLITE_LDFLAGS := -pthread
+ endif
+endif
+SQLITE_STATIC ?= true
+
+SSL_INCLUDE_DIR ?= /usr/include
+ifeq ($(SSL_INCLUDE_DIR),)
+ override SSL_INCLUDE_DIR = /usr/include
+endif
+SSL_LIB_DIR ?= /usr/lib/x86_64-linux-gnu
+ifeq ($(SSL_LIB_DIR),)
+ override SSL_LIB_DIR = /usr/lib/x86_64-linux-gnu
+endif
+
+SSL_CFLAGS ?= -I$(SSL_INCLUDE_DIR)
+SSL_STATIC ?= true
+ifndef SSL_LDFLAGS
+ ifeq ($(SSL_STATIC),false)
+  SSL_LDFLAGS := -L$(SSL_LIB_DIR) -lcrypto
+ else
+  SSL_LDFLAGS := -L$(SSL_LIB_DIR) $(SSL_LIB_DIR)/libcrypto.a
+ endif
+ ifeq ($(detected_OS),Windows)
+  SSL_LDFLAGS += -lws2_32
+ endif
+endif
+
+SQLITE3_C ?= sqlite/sqlite3.c
+SQLITE3_H ?= $(CURDIR)/sqlite/sqlite3.h
+
+$(SQLITE3_C): | deps
+ifeq ($(detected_OS),Windows)
+	sed -i "s/tr -d '\\\\\\n'/tr -d '\\\\\\r\\\\\\n'/" vendor/sqlcipher/configure
+endif
+	echo -e $(BUILD_MSG) "SQLCipher's SQLite C amalgamation"
+	+ cd vendor/sqlcipher && \
+		./configure \
+			CFLAGS="$(SQLITE_CDEFS) $(SQLITE_CFLAGS) $(SSL_CFLAGS)" \
+			LDFLAGS="$(SQLITE_LDFLAGS) $(SSL_LDFLAGS)" \
+			$(HANDLE_OUTPUT)
+ifeq ($(detected_OS),Windows)
+	sed -E -i "s/TOP = \\/([A-Za-z])/TOP = \\u\\1:/" vendor/sqlcipher/Makefile
+endif
+	cd vendor/sqlcipher && $(MAKE) sqlite3.c $(HANDLE_OUTPUT)
+	mkdir -p sqlite
+	cp vendor/sqlcipher/sqlite3.c sqlite/
+	cp vendor/sqlcipher/sqlite3.h sqlite/
+	cd vendor/sqlcipher && \
+		git clean -dfx $(HANDLE_OUTPUT) && \
+		(git stash $(HANDLE_OUTPUT) || true) && \
+		(git stash drop $(HANDLE_OUTPUT) || true)
+
+sqlite3.c: $(SQLITE3_C)
+
+SQLITE_STATIC_LIB ?= $(CURDIR)/sqlite/sqlite3.a
+
+$(SQLITE_STATIC_LIB): $(SQLITE3_C)
+	echo -e $(BUILD_MSG) "SQLCipher static library"
+	+ $(ENV_SCRIPT) $(CC) \
+		-c \
+		sqlite/sqlite3.c \
+		$(SQLITE_CDEFS) \
+		$(SQLITE_CFLAGS) \
+		$(SSL_CFLAGS) \
+		-o sqlite/sqlite3.o $(HANDLE_OUTPUT)
+	$(ENV_SCRIPT) ar rcs $(SQLITE_STATIC_LIB) sqlite/sqlite3.o $(HANDLE_OUTPUT)
+
+ifndef SHARED_LIB_EXT
+ ifeq ($(detected_OS),macOS)
+  SHARED_LIB_EXT := dylib
+ else ifeq ($(detected_OS),Windows)
+  SHARED_LIB_EXT := dll
+ else
+  SHARED_LIB_EXT := so
+ endif
+endif
+
+SQLITE_SHARED_LIB ?= $(CURDIR)/sqlite/libsqlite3.$(SHARED_LIB_EXT)
+
+ifndef PLATFORM_LINKER_FLAGS
+ ifeq ($(detected_OS),macOS)
+  PLATFORM_LINKER_FLAGS := -dylib
+ endif
+endif
+
+$(SQLITE_SHARED_LIB): $(SQLITE3_C)
+	echo -e $(BUILD_MSG) "SQLCipher shared library"
+	+ $(ENV_SCRIPT) $(CC) \
+		-c -fPIC \
+		sqlite/sqlite3.c \
+		$(SQLITE_CDEFS) \
+		$(SQLITE_CFLAGS) \
+		$(SSL_CFLAGS) \
+		-o sqlite/sqlite3.o $(HANDLE_OUTPUT)
+	$(ENV_SCRIPT) ld \
+		$(PLATFORM_LINKER_FLAGS) \
+		-undefined dynamic_lookup \
+		sqlite/sqlite3.o \
+		-o $(SQLITE_SHARED_LIB) $(HANDLE_OUTPUT)
+
+ifndef SQLITE_LIB
+ ifneq ($(SQLITE_STATIC),false)
+  SQLITE_LIB := $(SQLITE_STATIC_LIB)
+ else
+  SQLITE_LIB := $(SQLITE_SHARED_LIB)
+ endif
+endif
+
+sqlite: $(SQLITE_LIB)
+
+ifndef NIMTEROP_TOAST
+ ifeq ($(detected_OS),Windows)
+  NIMTEROP_TOAST := vendor/nimterop/nimterop/toast.exe
+ else
+  NIMTEROP_TOAST := vendor/nimterop/nimterop/toast
+ endif
+endif
+
+$(NIMTEROP_TOAST): | deps
+	echo -e $(BUILD_MSG) "Nimterop toast"
+	+ cd vendor/nimterop && \
+		$(ENV_SCRIPT) nim c $(NIM_PARAMS) \
+			--define:danger \
+			--hints:off \
+			--nimcache:../../nimcache/nimterop \
+			nimterop/toast.nim
+	rm -rf $(NIMTEROP_TOAST).dSYM
+
+toast: $(NIMTEROP_TOAST)
+
+SQLITE_NIM ?= sqlcipher/sqlite.nim
+
+$(SQLITE_NIM): $(NIMTEROP_TOAST) $(SQLITE_LIB)
+	echo -e $(BUILD_MSG) "Nim wrapper for SQLCipher"
+	+ mkdir -p sqlcipher
+	SQLITE_CDEFS="$(SQLITE_CDEFS)"\
+	SQLITE_STATIC="$(SQLITE_STATIC)" \
+	SQLITE3_H="$(SQLITE3_H)" \
+	SQLITE_LIB="$(SQLITE_LIB)" \
+	$(ENV_SCRIPT) nim c $(NIM_PARAMS) \
+		--nimcache:nimcache/sqlcipher \
+		--verbosity:0 \
+		generator/generate.nim > sqlcipher/sqlite.nim 2> /dev/null
+	rm -rf generator/generate generator/generate.exe generator/generate.dSYM
+
+sqlite.nim: $(SQLITE_NIM)
+
+test: $(SQLITE_NIM)
+	SSL_LDFLAGS="$(SSL_LDFLAGS)" \
+	$(ENV_SCRIPT) nimble tests
+
+endif # "variables.mk" was not included

README.md

@@ -1,6 +1,7 @@
 # nim-sqlcipher
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
-![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)
+![Stability: experimental](https://img.shields.io/badge/Stability-experimental-orange.svg)
+[![Tests (GitHub Actions)](https://github.com/status-im/nim-sqlcipher/workflows/Tests/badge.svg?branch=master)](https://github.com/status-im/nim-sqlcipher/actions?query=workflow%3ATests+branch%3Amaster)
 
 A nim wrapper for the [SQLCipher](https://github.com/sqlcipher/sqlcipher).
 It builds SQLCipher and provides a simple API based on the [Tiny_SQLite](https://github.com/GULPF/tiny_sqlite) nimble package

env.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
+# and we fall back to a Zsh-specific special var to also support Zsh.
+REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
+ABS_PATH="$(cd ${REL_PATH}; pwd)"
+source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh

generator/generate.nim

@@ -0,0 +1,49 @@
+import macros
+import nimterop/cimport
+import os
+import strutils
+
+macro dynamicCdefine(): untyped =
+  var cdefs: seq[string]
+  for cdef in split(getEnv("SQLITE_CDEFS"), "-D"):
+    let stripped = strip(cdef)
+    if stripped != "":
+      cdefs.add(stripped)
+  result = newStmtList()
+  for cdef in cdefs:
+    result.add(newCall("cDefine", newStrLitNode(cdef)))
+
+static:
+  cDebug()
+
+  cSkipSymbol(@[
+    "sqlite3_version",
+    "sqlite3_destructor_type"
+  ])
+
+  dynamicCdefine()
+
+  when getEnv("SQLITE_STATIC") == "false":
+    cPassL("-L" & splitPath($getEnv("SQLITE_LIB")).head & " " & "-lsqlite3")
+  when getEnv("SQLITE_STATIC") != "false":
+    cPassL($getEnv("SQLITE_LIB"))
+
+cPlugin:
+  import strutils
+
+  var i = 0;
+
+  proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} =
+    # Remove prefixes or suffixes from procs
+    if sym.kind == nskProc and sym.name.contains("sqlite3_"):
+      sym.name = sym.name.replace("sqlite3_", "")
+    # Workaround for duplicate iColumn symbol in generated Nim code
+    # (but generated code for sqlite3_index_info is likely not usable anyway)
+    if sym.name.contains("iColumn"):
+      if i == 0:
+        sym.name = sym.name.replace("iColumn", "iColumn_index_constraint")
+      else:
+        sym.name = sym.name.replace("iColumn", "iColumn_index_orderby")
+      i += 1
+
+cImport($getEnv("SQLITE3_H"), flags = "-f:ast2")

sqlcipher.nim

@@ -1,4 +1,8 @@
 import std / [options, macros, typetraits]
+
+# sqlcipher/sqlite.nim must be generated before this module can be used.
+# To generate it use the `sqlite.nim` target of the Makefile in the same
+# directory as this file.
 from sqlcipher/sqlite as sqlite import nil
 
 # Adapted from https://github.com/GULPF/tiny_sqlite

sqlcipher.nimble

@@ -1,13 +1,43 @@
-# Package
+mode = ScriptMode.Verbose
 
-packageName   = "sqlcipher"
 version     = "0.1.0"
 author      = "Status Research & Development GmbH"
 description = "A wrapper for SQLCipher"
 license     = "MIT"
 skipDirs    = @["test"]
-srcDir        = "src"
 
-# Dependencies
-requires "nim >= 1.0.0"
-requires "nimterop >= 0.5.2"
+requires "nim >= 1.2.0",
+  "nimterop"
+
+proc buildAndRunTest(name: string,
+                     srcDir = "test/",
+                     outDir = "test/build/",
+                     params = "",
+                     cmdParams = "",
+                     lang = "c") =
+  rmdir outDir
+  mkDir outDir
+  # allow something like "nim test --verbosity:0 --hints:off beacon_chain.nims"
+  var extra_params = params
+  for i in 2..<paramCount():
+    extra_params &= " " & paramStr(i)
+  exec "nim " &
+    lang &
+    " --debugger:native" &
+    " --define:debug" &
+    " --define:ssl" &
+    " --nimcache:nimcache/test/" & name &
+    " --out:" & outDir & name &
+    " --passL:\"" & getEnv("SSL_LDFLAGS") & "\"" &
+    " --threads:on" &
+    " --tlsEmulation:off" &
+    " " &
+    extra_params &
+    " " &
+    srcDir & name & ".nim" &
+    " " &
+    cmdParams
+  exec outDir & name
+
+task tests, "Run all tests":
+  buildAndRunTest "db_smoke"

test/db_smoke.nim

@@ -0,0 +1,20 @@
+import ../sqlcipher
+
+from os import parentDir
+import strformat
+import times
+
+let db: DbConn = openDatabase(currentSourcePath.parentDir() & "/build/myDatabase")
+
+let passwd = "qwerty"
+
+key(db, passwd)
+
+execScript(db, "create table if not exists Log (theTime text primary key)")
+
+let date = getDateStr(now())
+let time = getClockStr(now())
+
+execScript(db, &"""insert into Log values("{date}:{time}")""")
+
+echo rows(db, "select * from Log")

test/main.nim

@@ -1,19 +0,0 @@
-import sqlcipher
-import times
-import strformat
-
-when isMainModule:
-  let db: DbConn = openDatabase("./myDatabase")
-
-  let passwd = "qwerty"
-
-  key(db, passwd)
-
-  execScript(db, "create table if not exists Log (theTime text primary key)")
-
-  let date = getDateStr(now())
-  let time = getClockStr(now())
-
-  execScript(db, &"""insert into Log values("{date}:{time}")""")
-
-  echo rows(db, "select * from Log")

vendor/cligen

@@ -0,0 +1 @@
+Subproject commit 992fcc078475bebba259ed09340f2eb30504fba4

vendor/nim-regex

@@ -0,0 +1 @@
+Subproject commit 37799c609105d8aaa5b7a1806d13fbceec5123de

vendor/nim-segmentation

@@ -0,0 +1 @@
+Subproject commit 47bae531c657e01a92734e57aed552957981ad1c

vendor/nim-unicodedb

@@ -0,0 +1 @@
+Subproject commit 7c6ee4bfc184d7121896a098d68b639a96df7af1

vendor/nim-unicodeplus

@@ -0,0 +1 @@
+Subproject commit fd553314df9d9a45aa0d14218e20e7c029f0baa1

vendor/nimbus-build-system

@@ -0,0 +1 @@
+Subproject commit 92e5042667b747d22106f085eaa9b5e9766ba474

vendor/nimterop

@@ -0,0 +1 @@
+Subproject commit 0c2ca16f7ad9b1798f1c28ca0a3268d98e845a8d

vendor/sqlcipher

@@ -0,0 +1 @@
+Subproject commit 4a81bea61e1da6fec222d713852830f1fd01aed2