Implement a --datadir command-line options

By default, the database files will be written in the
platform-specific application data folder:

$HOME/AppData/Roaming/Nimbus/DB
$HOME/Library/Application Support/Nimbus/DB
$HOME/.cache/nimbus/db

README.md

@@ -25,7 +25,7 @@ Join the Status community chats:
 * A recent version of Facebook's [RocksDB](https://github.com/facebook/rocksdb/)
   * Compile [from source](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) or use the package manager of your OS; for example, [Debian](https://packages.debian.org/search?keywords=librocksdb-dev&searchon=names&exact=1&suite=all&section=all), [Ubuntu](https://packages.ubuntu.com/search?keywords=librocksdb-dev&searchon=names&exact=1&suite=all&section=all), and [Fedora](https://apps.fedoraproject.org/packages/rocksdb) have working RocksDB packages
 
-### Obtaining the prerequisites through the Nix package manager
+#### Obtaining the prerequisites through the Nix package manager
 
 Users of the [Nix package manager](https://nixos.org/nix/download.html) can install all prerequisites simply by running:
 
@@ -37,24 +37,26 @@ nix-shell nimbus.nix
 
 We use [Nimble](https://github.com/nim-lang/nimble) to manage dependencies and run tests.
 
-To build and run test suite:
+To build and install Nimbus in your home folder, just execute:
+
+```bash
+nimble install
+```
+
+After a succesful installation, running `nimbus --help` will provide you with a list of
+the available command-line options. To start syncing with mainnet, just execute `nimbus`
+without any parameters.
+
+To execute all tests:
 ```bash
 nimble test
 ```
 
-Based on our config, Nimble will write binaries to `build/` - you can do this manually also, as in the following examples:
+Our Wiki provides additional helpful information for [debugging individual test cases][1]
+and for [pairing Nimbus with a locally running copy of Geth][2].
 
-Run example:
+[1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests
-```bash
+[2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction
-mkdir -p build
-nim c -o:build/decompile_smart_contract -r examples/decompile_smart_contract.nim
-```
-
-Run Ethereum [JSON-based general state tests](https://github.com/ethereum/tests/):
-```
-mkdir -p build
-nim c -o:build/test_generalstate_json -r --experimental:forLoopMacros tests/test_generalstate_json.nim
-```
 
 #### Troubleshooting
 

nimbus.nimble

@@ -21,12 +21,17 @@ requires "nim >= 0.18.1",
          "eth_keyfile",
          "eth_keys"
 
-proc test(name: string, lang = "c") =
+proc buildBinary(name: string, srcDir = ".", lang = "c") =
-  if not dirExists "build":
+  if not dirExists "build": mkDir "build"
-    mkDir "build"
-  --run
   switch("out", ("./build/" & name))
-  setCommand lang, "tests/" & name & ".nim"
+  setCommand lang, srcDir & name & ".nim"
+
+proc test(name: string, lang = "c") =
+  buildBinary name, "tests/"
+  --run
 
 task test, "Run tests":
   test "all_tests"
+
+task nimbus, "Build Nimbus":
+  buildBinary "nimbus", "nimbus/"

nimbus/config.nim

@@ -7,7 +7,7 @@
 # This file may not be copied, modified, or distributed except according to
 # those terms.
 
-import parseopt, strutils, macros
+import parseopt, strutils, macros, os
 import asyncdispatch2, eth_keys, eth_p2p, eth_common, chronicles
 
 const
@@ -148,6 +148,8 @@ type
 
   NimbusConfiguration* = ref object
     ## Main Nimbus configuration object
+    dataDir*: string
+    keyFile*: string
     rpc*: RpcConfiguration        ## JSON-RPC configuration
     net*: NetConfiguration        ## Network configuration
     debug*: DebugConfiguration    ## Debug configuration
@@ -291,6 +293,20 @@ proc processPrivateKey(v: string, o: var PrivateKey): ConfigStatus =
 #   except:
 #     result = ErrorParseOption
 
+proc processEthArguments(key, value: string): ConfigStatus =
+  result = Success
+  let config = getConfiguration()
+  case key.toLowerAscii()
+  of "keyfile":
+    if fileExists(value):
+      config.keyFile = value
+    else:
+      result = ErrorIncorrectOption
+  of "dataDir":
+    config.dataDir = value
+  else:
+    result = EmptyOption
+
 proc processRpcArguments(key, value: string): ConfigStatus =
   ## Processes only `RPC` related command line options
   result = Success
@@ -462,6 +478,15 @@ proc initConfiguration(): NimbusConfiguration =
   result.net.discPort = 30303'u16
   result.net.ident = NimbusIdent
 
+  const dataDir = when defined(windows):
+                    "AppData" / "Roaming" / "Nimbus" / "DB"
+                  elif defined(macosx):
+                    "Library" / "Application Support" / "Nimbus" / "DB"
+                  else:
+                    ".cache" / "nimbus" / "db"
+
+  result.dataDir = getHomeDir() / dataDir
+
   ## Debug defaults
   result.debug.flags = {}
 
@@ -479,11 +504,12 @@ USAGE:
 
 ETHEREUM OPTIONS:
   --keyfile:<value>       Use keyfile storage file
+  --datadir:<value>       Base directory for all blockchain-related data
 
 NETWORKING OPTIONS:
   --bootnodes:<value>     Comma separated enode URLs for P2P discovery bootstrap (set v4+v5 instead for light servers)
   --bootnodesv4:<value>   Comma separated enode URLs for P2P v4 discovery bootstrap (light server, full nodes)
-  --botnoodesv5:<value>   Comma separated enode URLs for P2P v5 discovery bootstrap (light server, light nodes)
+  --bootnodesv5:<value>   Comma separated enode URLs for P2P v5 discovery bootstrap (light server, light nodes)
   --port:<value>          Network listening TCP port (default: 30303)
   --discport:<value>      Network listening UDP port (default: 30303)
   --maxpeers:<value>      Maximum number of network peers (default: 25)
@@ -531,6 +557,7 @@ proc processArguments*(msg: var string): ConfigStatus =
           result = Success
           break
         else:
+          checkArgument processEthArguments, key, value, msg
           checkArgument processRpcArguments, key, value, msg
           checkArgument processNetArguments, key, value, msg
           checkArgument processDebugArguments, key, value, msg

nimbus/db/backends/sqlite_backend.nim

@@ -1,5 +1,5 @@
 import
-  sqlite3, ranges, ranges/ptr_arith, eth_trie/db_tracing,
+  os, sqlite3, ranges, ranges/ptr_arith, eth_trie/db_tracing,
   ../storage_types
 
 type
@@ -9,8 +9,9 @@ type
 
   ChainDB* = SqliteChainDB
 
-proc newChainDB*(dbPath: string): ChainDB =
+proc newChainDB*(basePath: string, inMemory = false): ChainDB =
   result.new()
+  let dbPath = basePath / (if inMemory: ":memory:" else: "nimbus.db")
   var s = sqlite3.open(dbPath, result.store)
   if s != SQLITE_OK:
     raiseStorageInitError()

nimbus/nimbus.nim

@@ -10,7 +10,7 @@
 import
   os, strutils, net, eth_common, db/[storage_types, db_chain],
   asyncdispatch2, json_rpc/rpcserver, eth_keys,
-  eth_p2p, eth_p2p/rlpx_protocols/[eth, les],
+  eth_p2p, eth_p2p/rlpx_protocols/[eth],
   config, genesis, rpc/[common, p2p], p2p/chain,
   eth_trie
 
@@ -41,10 +41,6 @@ type
     ethNode*: EthereumNode
     state*: NimbusState
 
-proc newTrieDb(): TrieDatabaseRef =
-  # XXX: Setup db storage location according to config
-  result = trieDB(newChainDb("nimbus.db"))
-
 proc initializeEmptyDb(db: BaseChainDB) =
   echo "Writing genesis to DB"
   let networkId = getConfiguration().net.networkId.toPublicNetwork()
@@ -75,7 +71,8 @@ proc start(): NimbusObject =
   address.tcpPort = Port(conf.net.bindPort)
   address.udpPort = Port(conf.net.discPort)
 
-  let trieDB = newTrieDb()
+  createDir(conf.dataDir)
+  let trieDB = trieDB newChainDb(conf.dataDir)
   let chainDB = newBaseChainDB(trieDB)
 
   if canonicalHeadHashKey().toOpenArray notin trieDB:

tests/test_storage_backends.nim

@@ -5,7 +5,7 @@ import
   ../nimbus/db/backends/[sqlite_backend, rocksdb_backend]
 
 template dummyInstance(T: type SqliteChainDB): auto =
-  sqlite_backend.newChainDB ":memory:"
+  sqlite_backend.newChainDB(getTempDir(), inMemory = true)
 
 template dummyInstance(T: type RocksChainDB): auto =
   let tmp = getTempDir() / "nimbus-test-db"