diff --git a/.gitmodules b/.gitmodules index 1b7228393..b8d2ec3cb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -141,3 +141,6 @@ url = https://github.com/status-im/nim-libbacktrace.git ignore = dirty branch = master +[submodule "vendor/nim-sqlite3-abi"] + path = vendor/nim-sqlite3-abi + url = https://github.com/arnetheduck/nim-sqlite3-abi.git diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 824b41ed4..0c51475ad 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -6,7 +6,7 @@ import stew/[objects, bitseqs, byteutils], chronos, chronicles, confutils, metrics, json_serialization/std/[options, sets], serialization/errors, - kvstore, kvstore_lmdb, + kvstore, kvstore_sqlite3, eth/p2p/enode, eth/[keys, async_utils], eth/p2p/discoveryv5/enr, # Local modules @@ -131,7 +131,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async ourPubKey = netKeys.pubkey.skkey nickname = if conf.nodeName == "auto": shortForm(netKeys) else: conf.nodeName - db = BeaconChainDB.init(kvStore LmdbStoreRef.init(conf.databaseDir)) + db = BeaconChainDB.init(kvStore SqliteStoreRef.init(conf.databaseDir)) var mainchainMonitor: MainchainMonitor diff --git a/beacon_chain/kvstore_lmdb.nim b/beacon_chain/kvstore_lmdb.nim index 615254047..84858699f 100644 --- a/beacon_chain/kvstore_lmdb.nim +++ b/beacon_chain/kvstore_lmdb.nim @@ -1,5 +1,8 @@ ## Implementation of KVStore based on LMDB -## TODO: crashes on win32, investigate +# TODO This implementation has several issues on restricted platforms, possibly +# due to mmap restrictions - see: +# https://github.com/status-im/nim-beacon-chain/issues/732 +# https://github.com/status-im/nim-beacon-chain/issues/688 import os diff --git a/beacon_chain/kvstore_sqlite3.nim b/beacon_chain/kvstore_sqlite3.nim new file mode 100644 index 000000000..b92dfebbc --- /dev/null +++ b/beacon_chain/kvstore_sqlite3.nim @@ -0,0 +1,151 @@ +## Implementation of KVStore based on Sqlite3 + +import + os, + sqlite3_abi, + ./kvstore + +type + SqliteStoreRef* = ref object of RootObj + env: ptr sqlite3 + selectStmt, insertStmt, deleteStmt: ptr sqlite3_stmt + + SqliteError* = object of CatchableError + +func raiseError(err: cint) {.noreturn.} = + let tmp = sqlite3_errstr(err) + raise (ref SqliteError)(msg: $tmp) + +template checkErr(op, cleanup: untyped) = + if (let v = (op); v != SQLITE_OK): + cleanup + raiseError(v) + +template checkErr(op) = + checkErr(op): discard + +proc bindBlob(s: ptr sqlite3_stmt, n: int, blob: openarray[byte]): cint = + sqlite3_bind_blob(s, n.cint, unsafeAddr blob[0], blob.len.cint, nil) + +proc get*(db: SqliteStoreRef, key: openarray[byte], onData: DataProc): bool = + checkErr sqlite3_reset(db.selectStmt) + checkErr sqlite3_clear_bindings(db.selectStmt) + checkErr bindBlob(db.selectStmt, 1, key) + + let v = sqlite3_step(db.selectStmt) + case v + of SQLITE_ROW: + let + p = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(db.selectStmt, 0)) + l = sqlite3_column_bytes(db.selectStmt, 0) + onData(toOpenArray(p, 0, l-1)) + true + of SQLITE_DONE: + false + else: + raiseError(v) + +proc put*(db: SqliteStoreRef, key, value: openarray[byte]) = + checkErr sqlite3_reset(db.insertStmt) + checkErr sqlite3_clear_bindings(db.insertStmt) + + checkErr bindBlob(db.insertStmt, 1, key) + checkErr bindBlob(db.insertStmt, 2, value) + + if (let v = sqlite3_step(db.insertStmt); v != SQLITE_DONE): + raiseError(v) + +proc contains*(db: SqliteStoreRef, key: openarray[byte]): bool = + checkErr sqlite3_reset(db.selectStmt) + checkErr sqlite3_clear_bindings(db.selectStmt) + + checkErr bindBlob(db.selectStmt, 1, key) + + let v = sqlite3_step(db.selectStmt) + case v + of SQLITE_ROW: result = true + of SQLITE_DONE: result = false + else: raiseError(v) + +proc del*(db: SqliteStoreRef, key: openarray[byte]) = + checkErr sqlite3_reset(db.deleteStmt) + checkErr sqlite3_clear_bindings(db.deleteStmt) + + checkErr bindBlob(db.deleteStmt, 1, key) + + if (let v = sqlite3_step(db.deleteStmt); v != SQLITE_DONE): + raiseError(v) + +proc close*(db: SqliteStoreRef) = + discard sqlite3_finalize(db.insertStmt) + discard sqlite3_finalize(db.selectStmt) + discard sqlite3_finalize(db.deleteStmt) + discard sqlite3_close(db.env) + + db[] = SqliteStoreRef()[] + +proc init*( + T: type SqliteStoreRef, + basePath: string, + readOnly = false, + inMemory = false): T = + var + env: ptr sqlite3 + + let + name = + if inMemory: ":memory:" + else: basepath / "nimbus.sqlite3" + flags = + if readOnly: SQLITE_OPEN_READONLY + else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE + + if not inMemory: + createDir(basePath) + + checkErr sqlite3_open_v2(name, addr env, flags.cint, nil) + + template prepare(q: string, cleanup: untyped): ptr sqlite3_stmt = + var s: ptr sqlite3_stmt + checkErr sqlite3_prepare_v2(env, q, q.len.cint, addr s, nil): + cleanup + discard sqlite3_close(env) + s + + template checkExec(q: string) = + let s = prepare(q): discard + + if (let x = sqlite3_step(s); x != SQLITE_DONE): + discard sqlite3_finalize(s) + discard sqlite3_close(env) + raiseError(x) + + if (let x = sqlite3_finalize(s); x != SQLITE_OK): + discard sqlite3_close(env) + raiseError(x) + + # TODO: check current version and implement schema versioning + checkExec "PRAGMA user_version = 1;" + + checkExec """ + CREATE TABLE IF NOT EXISTS kvstore( + key BLOB PRIMARY KEY, + value BLOB + ) WITHOUT ROWID; + """ + + let + selectStmt = prepare "SELECT value FROM kvstore WHERE key = ?;": + discard + insertStmt = prepare "INSERT OR REPLACE INTO kvstore(key, value) VALUES (?, ?);": + discard sqlite3_finalize(selectStmt) + deleteStmt = prepare "DELETE FROM kvstore WHERE key = ?;": + discard sqlite3_finalize(selectStmt) + discard sqlite3_finalize(insertStmt) + + T( + env: env, + selectStmt: selectStmt, + insertStmt: insertStmt, + deleteStmt: deleteStmt + ) diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 0766f6936..bc0833bfa 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -21,6 +21,7 @@ import # Unit test ./test_interop, ./test_kvstore, ./test_kvstore_lmdb, + ./test_kvstore_sqlite3, ./test_ssz, ./test_state_transition, ./test_sync_protocol, diff --git a/tests/test_kvstore_sqlite3.nim b/tests/test_kvstore_sqlite3.nim new file mode 100644 index 000000000..6eec98196 --- /dev/null +++ b/tests/test_kvstore_sqlite3.nim @@ -0,0 +1,14 @@ +{.used.} + +import + os, + unittest, + ../beacon_chain/[kvstore, kvstore_sqlite3], + ./test_kvstore + +suite "Sqlite": + test "KVStore interface": + let db = SqliteStoreRef.init("", inMemory = true) + defer: db.close() + + testKVStore(kvStore db) diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi new file mode 160000 index 000000000..fc2028746 --- /dev/null +++ b/vendor/nim-sqlite3-abi @@ -0,0 +1 @@ +Subproject commit fc2028746d367e259d55ba41fdf5587a0683ee59