deploy: ddf93814fe865f66576e80fa056a27ffe864666d

CHANGELOG.md

@@ -1,4 +1,22 @@
 # Changelog
+## Next
+
+This release contains the following:
+
+### Features
+
+### Changes
+- Enables db migration for the message store.
+#### General refactoring
+#### Docs
+#### Schema
+- Updates the `Message` table of the persistent message store:
+  - Adds `senderTimestamp` column.
+  - Renames the `timestamp` column to `receiverTimestamp` and changes its type to `REAL`.
+#### API
+
+### Fixes
+
 
 ## 2021-06-03 v0.4
 

tests/all_tests_v2.nim

@@ -14,7 +14,8 @@ import
   ./v2/test_web3, # TODO  remove it when rln-relay tests get finalized
   ./v2/test_waku_bridge,
   ./v2/test_peer_storage,
-  ./v2/test_waku_keepalive
+  ./v2/test_waku_keepalive,
+  ./v2/test_migration_utils
 
 when defined(rln):
   import ./v2/test_waku_rln_relay

tests/v2/00002_test.up.sql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS Message_backup (
+        id BLOB PRIMARY KEY,
+        timestamp INTEGER NOT NULL,
+        contentTopic BLOB NOT NULL,
+        pubsubTopic BLOB NOT NULL,
+        payload BLOB,
+        version INTEGER NOT NULL
+    ) WITHOUT ROWID;

tests/v2/test_message_store.nim

@@ -1,9 +1,10 @@
 {.used.}
 
 import
-  std/[unittest, options, tables, sets],
-  chronos, chronicles,
+  std/[unittest, options, tables, sets, times, os, strutils],
+  chronos,
   ../../waku/v2/node/storage/message/waku_message_store,
+  ../../waku/v2/node/storage/sqlite,
   ../../waku/v2/protocol/waku_store/waku_store,
   ./utils
 
@@ -15,38 +16,99 @@ suite "Message Store":
       topic = ContentTopic("/waku/2/default-content/proto")
       pubsubTopic =  "/waku/2/default-waku/proto"
 
+      t1 = epochTime()
+      t2 = epochTime()
+      t3 = high(float64)
     var msgs = @[
-      WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic, version: uint32(0)),
-      WakuMessage(payload: @[byte 1, 2, 3, 4], contentTopic: topic, version: uint32(1)),
-      WakuMessage(payload: @[byte 1, 2, 3, 4, 5], contentTopic: topic, version: high(uint32)),
+      WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic, version: uint32(0), timestamp: t1),
+      WakuMessage(payload: @[byte 1, 2, 3, 4], contentTopic: topic, version: uint32(1), timestamp: t2),
+      WakuMessage(payload: @[byte 1, 2, 3, 4, 5], contentTopic: topic, version: high(uint32), timestamp: t3),
     ]
 
     defer: store.close()
 
+    var indexes: seq[Index] = @[]
     for msg in msgs:
-      let output = store.put(computeIndex(msg), msg, pubsubTopic)
+      var index = computeIndex(msg)
+      let output = store.put(index, msg, pubsubTopic)
       check output.isOk
+      indexes.add(index)
 
+
+    # flags for version
     var v0Flag, v1Flag, vMaxFlag: bool = false
+    # flags for sender timestamp
+    var t1Flag, t2Flag, t3Flag: bool = false
+    # flags for receiver timestamp
+    var rt1Flag, rt2Flag, rt3Flag: bool = false
+
     var responseCount = 0
-    proc data(timestamp: uint64, msg: WakuMessage, psTopic: string) =
+    proc data(receiverTimestamp: float64, msg: WakuMessage, psTopic: string) =
       responseCount += 1
       check msg in msgs
       check psTopic == pubsubTopic
+
       # check the correct retrieval of versions
       if msg.version == uint32(0): v0Flag = true
       if msg.version == uint32(1): v1Flag = true
       # high(uint32) is the largest value that fits in uint32, this is to make sure there is no overflow in the storage
       if msg.version == high(uint32): vMaxFlag = true
 
+      # check correct retrieval of sender timestamps
+      if msg.timestamp == t1: t1Flag = true
+      if msg.timestamp == t2: t2Flag = true
+      if msg.timestamp == t3: t3Flag = true
+
+      # check correct retrieval of receiver timestamps
+      if receiverTimestamp == indexes[0].receivedTime: rt1Flag = true
+      if receiverTimestamp == indexes[1].receivedTime: rt2Flag = true
+      if receiverTimestamp == indexes[2].receivedTime: rt3Flag = true
+
 
     let res = store.getAll(data)
     
     check:
       res.isErr == false
       responseCount == 3
+      # check version
       v0Flag == true
       v1Flag == true
       vMaxFlag == true
+      # check sender timestamp
+      t1Flag == true
+      t2Flag == true
+      t3Flag == true
+      # check receiver timestamp
+      rt1Flag == true
+      rt2Flag == true
+      rt3Flag == true
+  test "set and get user version":
+    let 
+      database = SqliteDatabase.init("", inMemory = true)[]
+      store = WakuMessageStore.init(database)[]
+    defer: store.close()
 
+    let res = database.setUserVersion(5)
+    check res.isErr == false
 
+    let ver = database.getUserVersion()
+    check:
+      ver.isErr == false
+      ver.value == 5
+  test "migration":
+    let 
+      database = SqliteDatabase.init("", inMemory = true)[]
+      store = WakuMessageStore.init(database)[]
+    defer: store.close()
+
+    template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
+    let migrationPath = sourceDir
+
+    let res = database.migrate(migrationPath, 10)
+    check:
+      res.isErr == false
+
+    let ver = database.getUserVersion()
+    check:
+      ver.isErr == false
+      ver.value == 10

tests/v2/test_migration_utils.nim

@@ -0,0 +1,63 @@
+{.used.}
+
+import
+  std/[unittest, tables, strutils, os, sequtils],
+  chronicles,
+  stew/results,
+  ../../waku/v2/node/storage/migration/[migration_types, migration_utils]
+
+template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
+const MIGRATION_PATH = sourceDir / "../../waku/v2/node/storage/migration/migrations_scripts/message"
+
+suite "Migration utils":
+  test "read migration scripts":
+    let migrationScriptsRes = getScripts(MIGRATION_PATH)
+    check:
+      migrationScriptsRes.isErr == false
+
+  test "filter migration scripts":
+    let migrationUp = [("0001_init", "script1"), ("0001_add", "script1"), ("0002_init", "script2"), ("0003_init", "script3")].toOrderedTable()
+    let migrationScripts = MigrationScripts(migrationUp: migrationUp)
+    let scriptsRes = filterScripts(migrationScripts, 1, 3)
+    check:
+      scriptsRes.isErr == false
+      scriptsRes.value.len == 2
+      scriptsRes.value[0] == "script2"
+      scriptsRes.value[1] == "script3"
+
+  test "filter migration scripts with varying zero-prefixed user versions":
+    let migrationUp = [("0001_init", "script1"), ("1_add", "script1"), ("000002_init", "script2"), ("003_init", "script3")].toOrderedTable()
+    let migrationScripts = MigrationScripts(migrationUp: migrationUp)
+    let scriptsRes = filterScripts(migrationScripts, 1, 3)
+    check:
+      scriptsRes.isErr == false
+      scriptsRes.value.len == 2
+      scriptsRes.value[0] == "script2"
+      scriptsRes.value[1] == "script3"
+
+  test "split scripts with no queries":
+    let script = "; ;"
+    let queries = splitScript(script)
+    check queries.len == 0
+
+  test "split scripts with multiple queries":
+    let q1 = """CREATE TABLE contacts2 (
+                contact_id INTEGER PRIMARY KEY,
+                first_name TEXT NOT NULL,
+                last_name TEXT NOT NULL,
+                email TEXT NOT NULL UNIQUE,
+                phone TEXT NOT NULL UNIQUE
+                );"""
+    let q2 = """CREATE TABLE contacts2 (
+                contact_id INTEGER PRIMARY KEY,
+                first_name TEXT NOT NULL,
+                last_name TEXT NOT NULL,
+                email TEXT NOT NULL UNIQUE,
+                phone TEXT NOT NULL UNIQUE
+                );"""
+    let script = q1 & q2
+    let queries = splitScript(script)
+    check:
+      queries.len == 2
+      queries[0] == q1
+      queries[1] == q2

vendor/nim-libbacktrace/vendor/libbacktrace-upstream/libtool

@@ -2,7 +2,7 @@
 
 # libtool - Provide generalized library-building support services.
 # Generated automatically by config.status (libbacktrace) version-unused
-# Libtool was configured on host fv-az132-116:
+# Libtool was configured on host fv-az272-819:
 # NOTE: Changes made to this file will be lost: look at ltmain.sh.
 #
 #   Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005,