fix: allow having multiple databases each one with their own migration definition

.gitignore

@@ -14,4 +14,5 @@
 /vendor/.nimble
 TODO
 sql_generate
-sql_scripts.nim
+sql_scripts_accounts.nim
+sql_scripts_app.nim

Makefile

@@ -212,16 +212,16 @@ ifneq ($(NIMSTATUS_CFLAGS),)
  NIM_PARAMS += --passC:"$(NIMSTATUS_CFLAGS)"
 endif
 
-MIGRATIONS ?= nim_status/lib/migrations/sql_scripts.nim
+MIGRATIONS ?= nim_status/lib/migrations/sql_scripts_app.nim
 
 $(MIGRATIONS): | deps
-	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) \
-		--verbosity:0 \
-		nim_status/lib/migrations/sql_generate.nim > \
-		nim_status/lib/migrations/sql_scripts.nim 2> /dev/null
+	$(ENV_SCRIPT) nim c $(NIM_PARAMS) --verbosity:0 nim_status/lib/migrations/sql_generate.nim
+	nim_status/lib/migrations/sql_generate nim_status/lib/migrations/accounts > nim_status/lib/migrations/sql_scripts_accounts.nim 
+	nim_status/lib/migrations/sql_generate nim_status/lib/migrations/app > nim_status/lib/migrations/sql_scripts_app.nim 
 
 clean-migration-file:
-	rm -f nim_status/lib/migrations/sql_scripts.nim
+	rm -f nim_status/lib/migrations/sql_scripts_accounts.nim
+	rm -f nim_status/lib/migrations/sql_scripts_app.nim
 
 migrations: clean-migration-file $(MIGRATIONS)
 

nim_status/lib/accounts.nim

@@ -4,7 +4,7 @@ import # vendor libs
   web3, chronos, web3/conversions as web3_conversions, web3/ethtypes,
   sqlcipher, json_serialization, json_serialization/[reader, writer, lexer],
   stew/byteutils
-
+import migrations/sql_scripts_app
 import conversions, settings/types, settings, database, conversions, callrpc
 
 var db_conn*: DbConn
@@ -18,7 +18,7 @@ proc login*(accountData, password: string) =
   # TODO: determine where the web3 conn will live
 
   let path =  getCurrentDir() / accountData & ".db"
-  db_conn = initializeDB(path, password)
+  db_conn = initializeDB(path, password, newMigrationDefinition())
 
   # TODO: these settings should have been set when calling saveAccountAndLogin
   let settingsStr = """{

nim_status/lib/database.nim

@@ -2,8 +2,14 @@ import sqlcipher
 import migration
 import results
 
-proc initializeDB*(path, password: string):DbConn =
+proc initializeDB*(path:string, definition: MigrationDefinition):DbConn =
+  result = openDatabase(path)
+  if not result.migrate(definition).isOk:
+    raise newException(SqliteError, "Failure executing migrations")
+
+
+proc initializeDB*(path, password: string, definition: MigrationDefinition):DbConn =
   result = openDatabase(path)
   result.key(password)
-  if not result.migrate().isOk:
+  if not result.migrate(definition).isOk:
     raise newException(SqliteError, "Failure executing migrations")

nim_status/lib/migration.nim

@@ -1,9 +1,11 @@
 import sqlcipher, results
 import sequtils, tables, algorithm, strformat
 import stew/byteutils
-import migrations/sql_scripts
 import nimcrypto
 import chronicles
+import migrations/types
+
+export MigrationDefinition
 
 type Migration* {.dbTableName("migrations").} = object
   name* {.dbColumnName("name").}: string
@@ -36,9 +38,9 @@ proc getAllMigrationsExecuted*(db: DbConn): seq[Migration] =
   const query = fmt"SELECT {migration.name.columnName}, {migration.hash.columnName} FROM {migration.tableName} ORDER BY rowid ASC;"
   db.all(Migration, query)
 
-proc checkMigrations*(db: DbConn): bool =
+proc checkMigrations*(db: DbConn, definition: MigrationDefinition): bool =
   let allMigrationsExecuted = db.getAllMigrationsExecuted()
-  let migrations = toSeq(migrationUp.keys)
+  let migrations = toSeq(definition.migrationUp.keys)
   
   debug "Verifying migration data"
 
@@ -53,37 +55,37 @@ proc checkMigrations*(db: DbConn): bool =
       warn "Migration order mismatch", migration=migration.name
       return false
 
-    if keccak_256.digest(migrationUp[migration.name]).data.toHex() != migration.hash:
+    if keccak_256.digest(definition.migrationUp[migration.name]).data.toHex() != migration.hash:
       warn "Migration hash mismatch", migration=migration.name
       return false
 
   return true
 
 
-proc isUpToDate*(db: DbConn):bool =
+proc isUpToDate*(db: DbConn, definition: MigrationDefinition):bool =
   let lastMigrationExecuted = db.getLastMigrationExecuted()
   if lastMigrationExecuted.isOk:
     # Check what's the latest migration
     let currentMigration = lastMigrationExecuted.get()
     
     var index = 0
-    for name in migrationUp.keys:
-      if name == currentMigration.name and index == migrationUp.len - 1:
+    for name in definition.migrationUp.keys:
+      if name == currentMigration.name and index == definition.migrationUp.len - 1:
         return true
       index += 1
   
   result = false
 
 
-proc migrate*(db: DbConn): MigrationResult =
+proc migrate*(db: DbConn, definition: MigrationDefinition): MigrationResult =
   db.createMigrationTableIfNotExists()
-  if not db.checkMigrations(): return MigrationResult.err "db/migration mismatch"
+  if not db.checkMigrations(definition): return MigrationResult.err "db/migration mismatch"
   var migration: Migration
   let lastMigrationExecuted = db.getLastMigrationExecuted()
   if not lastMigrationExecuted.isOk:
     try:
       db.transaction:
-        for name, query in migrationUp.pairs:
+        for name, query in definition.migrationUp.pairs:
           debug "Executing migration", name
           db.execScript(string.fromBytes(query))
           db.exec(fmt"INSERT INTO {migration.tableName}({migration.name.columnName}, {migration.hash.columnName}) VALUES(?, ?)", name, keccak_256.digest(query).data.toHex())
@@ -91,13 +93,13 @@ proc migrate*(db: DbConn): MigrationResult =
       warn "Could not execute migration"
       return MigrationResult.err "Could not execute migration"
   else:
-    if db.isUpToDate(): return lastMigrationExecuted
+    if db.isUpToDate(definition): return lastMigrationExecuted
     
     let allMigrationsExecuted = db.getAllMigrationsExecuted()
     var index = -1
     try:
       db.transaction:
-        for name, query in migrationUp.pairs:
+        for name, query in definition.migrationUp.pairs:
           index += 1
           if index <= (allMigrationsExecuted.len - 1): continue
           debug "Executing migration", name
@@ -110,15 +112,15 @@ proc migrate*(db: DbConn): MigrationResult =
   return db.getLastMigrationExecuted()
 
 
-proc tearDown*(db: DbConn):bool =
+proc tearDown*(db: DbConn, definition: MigrationDefinition):bool =
   var migration: Migration
   var allMigrationsExecuted = db.getAllMigrationsExecuted().reversed()
   try:
     db.transaction:
       for m in allMigrationsExecuted:
         debug "Rolling back migration", name=m.name
-        if migrationDown.hasKey(m.name):
-          let script = string.fromBytes(migrationDown[m.name])
+        if definition.migrationDown.hasKey(m.name):
+          let script = string.fromBytes(definition.migrationDown[m.name])
           if script != "": db.execScript(script)
         db.exec(fmt"DELETE FROM {migration.tableName} WHERE {migration.name.columnName} = ?", m.name)
   except SqliteError:

nim_status/lib/migrations/accounts/0001_accounts.down.sql

@@ -0,0 +1 @@
+DROP TABLE accounts;

nim_status/lib/migrations/accounts/0001_accounts.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS accounts (
+keyUid VARCHAR PRIMARY KEY,
+name TEXT NOT NULL,
+loginTimestamp BIG INT,
+photoPath TEXT,
+keycardPairing TEXT
+) WITHOUT ROWID;

nim_status/lib/migrations/accounts/1605007189_identity_images.down.sql

@@ -0,0 +1 @@
+DROP TABLE identity_images;

nim_status/lib/migrations/accounts/1605007189_identity_images.up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS identity_images(
+    key_uid VARCHAR,
+    name VARCHAR,
+    image_payload BLOB NOT NULL,
+    width int,
+    height int,
+    file_size int,
+    resize_target int,
+    PRIMARY KEY (key_uid, name) ON CONFLICT REPLACE
+) WITHOUT ROWID;

nim_status/lib/migrations/accounts/1606224181_drop_photo_path_from_accounts.down.sql

@@ -0,0 +1,23 @@
+/* Copy the accounts table into a temp table, EXCLUDE the `identicon` column and INCLUDE the `photoPath` column */
+CREATE TEMPORARY TABLE accounts_backup(
+    keyUid VARCHAR PRIMARY KEY,
+    name TEXT NOT NULL,
+    loginTimestamp BIG INT,
+    photoPath TEXT,
+    keycardPairing TEXT
+) WITHOUT ROWID;
+INSERT INTO accounts_backup SELECT keyUid, name, loginTimestamp, identicon, keycardPairing FROM accounts;
+
+/* Drop the old accounts table and recreate with all columns EXCLUDING `identicon` and INCLUDING `photoPath` */
+DROP TABLE accounts;
+CREATE TABLE IF NOT EXISTS accounts (
+    keyUid VARCHAR PRIMARY KEY,
+    name TEXT NOT NULL,
+    loginTimestamp BIG INT,
+    photoPath TEXT,
+    keycardPairing TEXT
+) WITHOUT ROWID;
+INSERT INTO accounts SELECT keyUid, name, loginTimestamp, photoPath, keycardPairing FROM accounts_backup;
+
+/* Tidy up, drop the temp table */
+DROP TABLE accounts_backup;

nim_status/lib/migrations/accounts/1606224181_drop_photo_path_from_accounts.up.sql

@@ -0,0 +1,23 @@
+/* Copy the accounts table into a temp table, EXCLUDE the `photoPath` and INCLUDE `identicon` column */
+CREATE TEMPORARY TABLE accounts_backup(
+    keyUid VARCHAR PRIMARY KEY,
+    name TEXT NOT NULL,
+    loginTimestamp BIG INT,
+    identicon TEXT,
+    keycardPairing TEXT
+) WITHOUT ROWID;
+INSERT INTO accounts_backup SELECT keyUid, name, loginTimestamp, photoPath, keycardPairing FROM accounts;
+
+/* Drop the old accounts table and recreate with all columns EXCLUDING `photoPath` and INCLUDING `identicon`*/
+DROP TABLE accounts;
+CREATE TABLE accounts(
+    keyUid VARCHAR PRIMARY KEY,
+    name TEXT NOT NULL,
+    loginTimestamp BIG INT,
+    identicon TEXT,
+    keycardPairing TEXT
+ ) WITHOUT ROWID;
+INSERT INTO accounts SELECT keyUid, name, loginTimestamp, identicon, keycardPairing FROM accounts_backup;
+
+/* Tidy up, drop the temp table */
+DROP TABLE accounts_backup;

nim_status/lib/migrations/sql_generate.nim

@@ -4,8 +4,8 @@ import stew/byteutils
 var upScripts = initOrderedTable[string, string]()
 var downScripts = initOrderedTable[string, string]()
 
-for kind, path in walkDir(currentSourcePath.parentDir):
-  let (dir, name, ext) = splitFile(path)
+for kind, path in walkDir(paramStr(1)):
+  let (_, name, ext) = splitFile(path)
   if ext != ".sql": continue
   
   let parts = name.split(".")
@@ -33,12 +33,13 @@ upScripts.sort(system.cmp)
 downScripts.sort(system.cmp)
 
 echo "# THIS FILE IS AUTOGENERATED - DO NOT MODIFY MANUALY - USE make migrations"
-echo "import tables\n"
-echo "var migrationUp*:OrderedTable[string, seq[byte]] = {"
+echo "import tables"
+echo "import types\n"
+echo "proc newMigrationDefinition*(): MigrationDefinition ="
+echo "  result = MigrationDefinition()"
+echo "  result.migrationUp = initOrderedTable[string, seq[byte]]()"
+echo "  result.migrationDown = initOrderedTable[string, seq[byte]]()"
 for name, query in upScripts.pairs():
-  echo "  \"" & name & "\": " & query.toBytes().print & ","
-echo "}.toOrderedTable\n"
-echo "var migrationDown*:OrderedTable[string, seq[byte]] = {"
+  echo "  result.migrationUp[\"" & name & "\"] = " & query.toBytes().print
 for name, query in downScripts.pairs():
-  echo "  \"" & name & "\": " & query.toBytes().print & ","
-echo "}.toOrderedTable\n"
+  echo "  result.migrationDown[\"" & name & "\"] = " & query.toBytes().print

nim_status/lib/migrations/types.nim

@@ -0,0 +1,6 @@
+import tables
+
+type
+  MigrationDefinition* = ref object of RootObj
+    migrationUp*:OrderedTable[string, seq[byte]]
+    migrationDown*:OrderedTable[string, seq[byte]]

test/nim/accounts.nim

@@ -6,12 +6,13 @@ import # vendor libs
 
 import # nim-status libs
  ../../nim_status/lib/[accounts, database, conversions]
+ ../../nim_status/lib/migrations/sql_accounts_app
 
 import times
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, newMigrationDefinition())
 
 var account:Account = Account(
   name: "Test",

test/nim/callrpc.nim

@@ -4,11 +4,13 @@ import options
 import ../../nim_status/lib/settings
 import ../../nim_status/lib/database
 import ../../nim_status/lib/callrpc
+import ../../nim_status/lib/migrations/sql_scripts_app
+
 import web3/conversions
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 let settingsStr = """{
     "address": "0x1122334455667788990011223344556677889900",
@@ -26,7 +28,7 @@ let settingsStr = """{
   }"""
 
 let settingsObj = JSON.decode(settingsStr, Settings, allowUnknownFields = true)
-#[
+
 let web3Obj = newWeb3(settingsObj)
 
 let rGasPrice = callRPC(web3Obj, eth_gasPrice, %[])
@@ -42,6 +44,6 @@ let rSendTransaction = callRPC(web3Obj, "eth_sendTransaction", %* [%*{"from": "0
 assert rSendTransaction.error == true
 assert rSendTransaction.result["code"].getInt == -32601
 assert rSendTransaction.result["message"].getStr == "the method eth_sendTransaction does not exist/is not available"
-]#
+
 db.close()
 removeFile(path)

test/nim/chats.nim

@@ -5,11 +5,12 @@ import # vendor libs
   sqlcipher, json_serialization, web3/conversions as web3_conversions
 
 import # nim-status libs
-  ../../nim_status/lib/[chats, database, conversions, contacts, messages]
+  ../../nim_status/lib/[chats, database, conversions, contacts, messages],
+  ../../nim_status/lib/migrations/sql_scripts_app
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 var chat = Chat(
   id: "ContactId",

test/nim/contacts.nim

@@ -6,11 +6,12 @@ import # vendor libs
   web3/ethtypes
 
 import # nim-status libs
-  ../../nim_status/lib/[contacts, database, conversions]
+  ../../nim_status/lib/[contacts, database, conversions],
+  ../../nim_status/lib/migrations/sql_scripts_app
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 let contact1 = Contact(
   id: "Contact1",

test/nim/mailservers.nim

@@ -3,10 +3,11 @@ import os, json, json_serialization
 import options
 import ../../nim_status/lib/mailservers
 import ../../nim_status/lib/database
+import ../../nim_status/lib/migrations/sql_scripts_app
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/my.db"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 let mailserver1 = Mailserver(
   id: "mailserver-1",

test/nim/messages.nim

@@ -5,11 +5,12 @@ import # vendor libs
   sqlcipher, json_serialization, web3/conversions as web3_conversions
 
 import # nim-status libs
-  ../../nim_status/lib/[messages, database, conversions, chats]
+  ../../nim_status/lib/[messages, database, conversions, chats],
+  ../../nim_status/lib/migrations/sql_scripts_app
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 var msg = Message(
   id: "msg1",

test/nim/migrations.nim

@@ -1,7 +1,7 @@
 import os, tables
 import sqlcipher, results
 import ../../nim_status/lib/migration
-import ../../nim_status/lib/migrations/sql_scripts
+import ../../nim_status/lib/migrations/sql_scripts_accounts as migration_accounts
 import stew/byteutils
 
 let passwd = "qwerty"
@@ -10,26 +10,27 @@ var dbConn = openDatabase(path)
 
 dbConn.key(passwd)
 
+var migrationDefinition = migration_accounts.newMigrationDefinition()
 
 assert dbConn.getLastMigrationExecuted().error() == "No migrations were executed"
-assert dbConn.migrate().isOk
-assert dbConn.isUpToDate()
+assert dbConn.migrate(migrationDefinition).isOk
+assert dbConn.isUpToDate(migrationDefinition)
 
 # Creating dinamically new migrations just to check if isUpToDate and migrate work as expected
-migrationUp["002_abc"] = "CREATE TABLE anotherTable (address VARCHAR NOT NULL PRIMARY KEY) WITHOUT ROWID;".toBytes
-migrationDown["002_abc"] = "DROP TABLE anotherTable;".toBytes
+migrationDefinition.migrationUp["002_abc"] = "CREATE TABLE anotherTable (address VARCHAR NOT NULL PRIMARY KEY) WITHOUT ROWID;".toBytes
+migrationDefinition.migrationDown["002_abc"] = "DROP TABLE anotherTable;".toBytes
 
-assert not dbConn.isUpToDate()
-assert dbConn.migrate().isOk
+assert not dbConn.isUpToDate(migrationDefinition)
+assert dbConn.migrate(migrationDefinition).isOk
 
-assert dbConn.isUpToDate()
+assert dbConn.isUpToDate(migrationDefinition)
 
-assert dbConn.migrate().isOk
+assert dbConn.migrate(migrationDefinition).isOk
 
-assert dbConn.tearDown()
-assert dbConn.tearDown()
+assert dbConn.tearDown(migrationDefinition)
+assert dbConn.tearDown(migrationDefinition)
 
-assert not dbConn.isUpToDate()
+assert not dbConn.isUpToDate(migrationDefinition)
 assert dbConn.getLastMigrationExecuted().error() == "No migrations were executed"
 
 

test/nim/pendingtxs.nim

@@ -5,11 +5,12 @@ import # vendor libs
   sqlcipher, json_serialization, web3/conversions as web3_conversions
 
 import # nim-status libs
-  ../../nim_status/lib/[pendingtxs, database, conversions]
+  ../../nim_status/lib/[pendingtxs, database, conversions],
+  ../../nim_status/lib/migrations/sql_scripts_app
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 let tx = PendingTx(
   networkId: 1,

test/nim/permissions.nim

@@ -5,11 +5,12 @@ import # vendor libs
   json_serialization
 
 import # nim-status libs
-  ../../nim_status/lib/[database, permissions]
+  ../../nim_status/lib/[database, permissions],
+  ../../nim_status/lib/migrations/sql_scripts_app
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 let dappPerm1: DappPermissions = DappPermissions(
   name: "Dapp1",

test/nim/settings.nim

@@ -7,10 +7,11 @@ import # vendor libs
 
 import # nim-status libs
   ../../nim_status/lib/[settings, database, conversions]
+import ../../nim_status/lib/migrations/sql_scripts_app
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 let settingsStr = """{
     "address": "0x1122334455667788990011223344556677889900",

test/nim/tokens.nim

@@ -4,10 +4,12 @@ import options
 import ../../nim_status/lib/tokens
 import ../../nim_status/lib/database
 import web3/conversions
+import ../../nim_status/lib/migrations/sql_scripts_app
+
 
 let passwd = "qwerty"
 let path = currentSourcePath.parentDir() & "/build/myDatabase"
-let db = initializeDB(path, passwd)
+let db = initializeDB(path, passwd, newMigrationDefinition())
 
 let tokensStr = """{
     "address": "0x1122334455667788990011223344556677889900",