Create different index for filtering

The index for message was fairly inefficient as it was only using the
cursor, as it was referring to the old `chat_id` field.

This meant that newer messages would be fetched much faster then older
messages.

The index has been changed so that now it includes `local_chat_id`
(which is currently used for filtering), and not using `hide`.

The reason being is that `hide` is a low cardinality index, so there's
no performance benefit to have it in, also it's mostly ignored by the
query planner.

This commit also adds the missing migrations, we generated the file, but
the source was missing, probably I forgot to add them in a rebase. They
have been generated from the migration file, using `RestoreAsset`.

VERSION

@@ -1 +1 @@
-0.54.0
+0.54.1

protocol/message_persistence.go

@@ -372,7 +372,7 @@ func (db sqlitePersistence) MessagesByIDs(ids []string) ([]*Message, error) {
 			ON
 
 			m1.source = c.id
-			WHERE m1.hide != 1 AND m1.id IN (%s)`, allFields, inVector), idsArgs...)
+			WHERE NOT(m1.hide) AND m1.id IN (%s)`, allFields, inVector), idsArgs...)
 	if err != nil {
 		return nil, err
 	}
@@ -423,7 +423,7 @@ func (db sqlitePersistence) MessageByChatID(chatID string, currCursor string, li
 
 			m1.source = c.id
 			WHERE
-				m1.hide != 1 AND m1.local_chat_id = ? %s
+				NOT(m1.hide) AND m1.local_chat_id = ? %s
 			ORDER BY cursor DESC
 			LIMIT ?
 		`, allFields, cursorWhere),

protocol/migrations/migrations.go

@@ -10,6 +10,8 @@
 // 1588665364_add_image_data.up.sql (186B)
 // 1589365189_add_pow_target.down.sql (0)
 // 1589365189_add_pow_target.up.sql (66B)
+// 1591277220_add_index_messages.down.sql (237B)
+// 1591277220_add_index_messages.up.sql (240B)
 // doc.go (850B)
 
 package migrations
@@ -214,7 +216,7 @@ func _1588665364_add_image_dataDownSql() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "1588665364_add_image_data.down.sql", size: 0, mode: os.FileMode(0644), modTime: time.Unix(1589363796, 0)}
+	info := bindataFileInfo{name: "1588665364_add_image_data.down.sql", size: 0, mode: os.FileMode(0644), modTime: time.Unix(1591361643, 0)}
 	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}
 	return a, nil
 }
@@ -234,7 +236,7 @@ func _1588665364_add_image_dataUpSql() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "1588665364_add_image_data.up.sql", size: 186, mode: os.FileMode(0644), modTime: time.Unix(1589363796, 0)}
+	info := bindataFileInfo{name: "1588665364_add_image_data.up.sql", size: 186, mode: os.FileMode(0644), modTime: time.Unix(1591361643, 0)}
 	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd6, 0xc6, 0x35, 0xb4, 0x4c, 0x39, 0x96, 0x29, 0x30, 0xda, 0xf4, 0x8f, 0xcb, 0xf1, 0x9f, 0x84, 0xdc, 0x88, 0xd4, 0xd5, 0xbc, 0xb6, 0x5b, 0x46, 0x78, 0x67, 0x76, 0x1a, 0x5, 0x36, 0xdc, 0xe5}}
 	return a, nil
 }
@@ -254,7 +256,7 @@ func _1589365189_add_pow_targetDownSql() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "1589365189_add_pow_target.down.sql", size: 0, mode: os.FileMode(0644), modTime: time.Unix(1589365242, 0)}
+	info := bindataFileInfo{name: "1589365189_add_pow_target.down.sql", size: 0, mode: os.FileMode(0644), modTime: time.Unix(1591361643, 0)}
 	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}
 	return a, nil
 }
@@ -274,11 +276,51 @@ func _1589365189_add_pow_targetUpSql() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "1589365189_add_pow_target.up.sql", size: 66, mode: os.FileMode(0644), modTime: time.Unix(1589365245, 0)}
+	info := bindataFileInfo{name: "1589365189_add_pow_target.up.sql", size: 66, mode: os.FileMode(0644), modTime: time.Unix(1591361643, 0)}
 	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x4e, 0x3a, 0xe2, 0x2e, 0x7d, 0xaf, 0xbb, 0xcc, 0x21, 0xa1, 0x7a, 0x41, 0x9a, 0xd0, 0xbb, 0xa9, 0xc8, 0x35, 0xf9, 0x32, 0x34, 0x46, 0x44, 0x9a, 0x86, 0x40, 0x7c, 0xb9, 0x23, 0xc7, 0x3, 0x3f}}
 	return a, nil
 }
 
+var __1591277220_add_index_messagesDownSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\xce\xb1\xaa\xc2\x30\x18\xc5\xf1\x3d\x4f\x71\xb6\xb6\x90\x0b\x77\x28\x5d\x3a\x89\xed\xe0\xd2\x4a\x71\x70\xfb\x48\x93\x60\x83\xd1\x40\xbe\x44\x14\xfa\xf0\x22\x74\x74\xf3\xac\x87\x1f\xfc\xbb\x69\x3c\xe2\x30\x74\xfd\x19\xce\x3c\x89\xad\x8a\x7a\xa1\xf9\x45\x3e\x68\xe5\x49\x2f\x2a\x91\x33\xc4\x21\x26\x0a\x77\xd2\x39\x72\x88\xad\x10\xfb\xa9\xdf\x9d\xfa\xaf\x72\x33\x18\x07\x20\xb3\x8d\x74\xb3\xcc\xea\x62\xb9\x14\x00\xc0\x79\xe6\x14\xcb\xe2\xff\xc7\x15\x58\x57\x68\x1f\xf4\x95\x1e\xca\x67\x2b\xf1\xd7\xd4\x12\x4d\x5d\x7d\x0e\x67\x24\xb6\x10\x89\xc5\x19\x2b\xaa\x56\xbc\x03\x00\x00\xff\xff\xaf\xf2\xdc\xc0\xed\x00\x00\x00")
+
+func _1591277220_add_index_messagesDownSqlBytes() ([]byte, error) {
+	return bindataRead(
+		__1591277220_add_index_messagesDownSql,
+		"1591277220_add_index_messages.down.sql",
+	)
+}
+
+func _1591277220_add_index_messagesDownSql() (*asset, error) {
+	bytes, err := _1591277220_add_index_messagesDownSqlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "1591277220_add_index_messages.down.sql", size: 237, mode: os.FileMode(0644), modTime: time.Unix(1591361643, 0)}
+	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x79, 0xe5, 0x42, 0x56, 0x64, 0x1d, 0xb7, 0x8a, 0x1b, 0x0, 0x99, 0xf0, 0x18, 0x8c, 0x69, 0xe3, 0x14, 0x3a, 0x7f, 0x78, 0xfe, 0xe3, 0x2e, 0xcb, 0x6e, 0x5c, 0x8c, 0x1f, 0x7b, 0xfc, 0x21, 0xc7}}
+	return a, nil
+}
+
+var __1591277220_add_index_messagesUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\xca\xb1\xaa\x83\x30\x14\x87\xf1\xdd\xa7\xf8\x6f\x2a\x78\xe1\x0e\xe2\xe2\x24\x9a\xa1\x8b\x16\xed\xd0\xed\x10\x63\xa8\xd2\xb4\x81\x73\x4c\x69\xc1\x87\x2f\x1d\x3a\x14\xba\xf5\x5b\xbf\x5f\xd3\x77\x7b\xec\xda\x46\x1d\xb1\x4c\x77\x12\xab\xd9\xcc\x34\x3e\xc8\xcc\x7a\xa5\x65\x2a\xa3\xa8\xee\x55\x75\x50\x5f\x91\xf3\x46\xbb\x37\x25\xf1\xbc\x92\xbf\x92\x09\x2c\x9e\xd1\xb5\x08\x62\x99\x2e\x56\x44\x9f\xac\x20\xf9\xe0\xa8\x86\x3a\x83\x84\x51\x56\x4e\xe2\xff\x1f\x8b\xb1\x6d\x30\xce\x9b\x33\xdd\xb4\x0b\x36\xc3\x5f\x91\x67\x28\xf2\xf4\x35\x96\x09\x8d\x1a\xea\xb4\x8c\x9e\x01\x00\x00\xff\xff\xdb\x1d\x3d\x0b\xf0\x00\x00\x00")
+
+func _1591277220_add_index_messagesUpSqlBytes() ([]byte, error) {
+	return bindataRead(
+		__1591277220_add_index_messagesUpSql,
+		"1591277220_add_index_messages.up.sql",
+	)
+}
+
+func _1591277220_add_index_messagesUpSql() (*asset, error) {
+	bytes, err := _1591277220_add_index_messagesUpSqlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "1591277220_add_index_messages.up.sql", size: 240, mode: os.FileMode(0644), modTime: time.Unix(1591361844, 0)}
+	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x9c, 0xfe, 0xbe, 0xd5, 0xb8, 0x8f, 0xdd, 0xef, 0xbb, 0xa8, 0xad, 0x7f, 0xed, 0x5b, 0x5b, 0x2f, 0xe6, 0x82, 0x27, 0x78, 0x1f, 0xb9, 0x57, 0xdc, 0x8, 0xc2, 0xb2, 0xa9, 0x9a, 0x4, 0xe1, 0x7a}}
+	return a, nil
+}
+
 var _docGo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x52\x3f\x8f\xdb\x3e\x0c\xdd\xf3\x29\x1e\x6e\xb9\xe5\x22\x07\xf8\xfd\xa6\xdb\x3a\x74\xe8\xd2\x2e\xd9\x0b\x46\xa6\x6d\x22\x32\xe5\x8a\xf4\x39\xf9\xf6\x85\x74\x17\x9c\x51\x14\xe8\x4a\x89\x8f\xef\x5f\xd7\xe1\x3c\x89\x61\x90\xc4\x10\x83\x72\x64\x33\x2a\x77\x5c\x38\xd2\x6a\x8c\xa7\x51\x7c\x5a\x2f\x21\xe6\xb9\x33\x27\x5f\xed\x28\x73\x37\xcb\x58\xc8\xb9\x7b\xfb\xff\xe9\xd0\x75\x88\xa4\xcf\x8e\x89\xb4\x4f\xdc\xb0\x0c\xe6\x54\x5c\x74\xc4\x26\x3e\x81\xb0\x14\x1e\xe4\x16\xf0\xc5\x91\x98\xcc\xe1\x13\xf9\xb3\xc1\x27\x46\x24\xe3\x0a\x33\xe4\x82\x31\x1f\x2f\xa2\x3d\x39\x85\x3a\xfa\x36\xec\x26\x95\x61\xa4\x94\xb8\xc7\x50\xf2\xdc\x76\x8d\x66\x46\x2f\x85\xa3\xe7\x72\x7f\x01\x99\xb1\x43\x69\x66\xab\xfb\x13\xbd\x31\x34\x7f\x9c\x07\x69\xff\x6f\x45\xd8\x72\xb9\x1a\xc8\xc0\xb7\x85\xa3\x73\x1f\x0e\x15\xeb\xfb\x8f\xf3\xd7\x57\x9c\x27\xae\xf0\x55\x5a\x1e\x1a\x85\x66\x9e\x32\xf7\x06\xcf\x18\x72\x4a\x79\x6b\x0f\xab\xca\x0d\x2e\x33\x9b\xd3\xbc\x20\x66\x7d\x63\x75\xc9\x5a\xd1\x56\x4d\x72\xe5\xf6\xcf\xb7\x0c\x51\x71\xa1\xf4\xee\x5e\x93\x7e\x7e\x37\xe8\x11\x44\x5c\x4b\x61\xf5\x74\x6f\x2b\xac\xb1\xdc\x97\x8a\x85\x77\xe6\x92\xd5\x9a\xbc\xa5\x64\xcf\x31\xa7\xdd\xbc\xa2\xd9\x44\x85\x3f\x1d\x73\xba\x24\x7e\xc1\x36\x49\x9c\x30\x33\xa9\xb5\x40\xda\x87\x44\xce\xe6\x9f\xfb\x10\x85\x73\x99\xad\x0a\xae\xfc\xaa\xbb\x15\xb3\x16\xe7\x91\xc3\x8e\x50\x33\x7f\xa1\xf8\x51\x85\xc7\x95\xd5\xd8\x40\x7f\x98\xf2\x08\x79\x63\x50\xdf\xe3\x74\x3a\x9d\xfe\xfb\x19\x42\x68\x5d\xe0\x1b\xcd\x4b\xa5\xe9\xb5\xa3\x9b\xa4\x84\x0b\x43\x46\xcd\x85\xfb\xca\x8a\x6f\x62\xad\x64\x31\x09\xab\xd7\xcc\x2a\x5e\x4e\x3d\x97\xaa\x47\xf7\x7a\xfe\x66\x59\x38\x1c\x16\x8a\x57\x1a\x19\xf6\x2b\x89\x73\x0d\x7a\xcc\xaf\x23\x2b\xd7\x3a\xec\xcb\x77\x5c\xae\xe3\xde\xec\x63\x46\x08\xdd\xe7\x20\x8c\x19\xe1\xf0\x3b\x00\x00\xff\xff\x12\xcd\x7f\xc4\x52\x03\x00\x00")
 
 func docGoBytes() ([]byte, error) {
@@ -410,6 +452,10 @@ var _bindata = map[string]func() (*asset, error){
 
 	"1589365189_add_pow_target.up.sql": _1589365189_add_pow_targetUpSql,
 
+	"1591277220_add_index_messages.down.sql": _1591277220_add_index_messagesDownSql,
+
+	"1591277220_add_index_messages.up.sql": _1591277220_add_index_messagesUpSql,
+
 	"doc.go": docGo,
 }
 
@@ -464,6 +510,8 @@ var _bintree = &bintree{nil, map[string]*bintree{
 	"1588665364_add_image_data.up.sql":         &bintree{_1588665364_add_image_dataUpSql, map[string]*bintree{}},
 	"1589365189_add_pow_target.down.sql":       &bintree{_1589365189_add_pow_targetDownSql, map[string]*bintree{}},
 	"1589365189_add_pow_target.up.sql":         &bintree{_1589365189_add_pow_targetUpSql, map[string]*bintree{}},
+	"1591277220_add_index_messages.down.sql":   &bintree{_1591277220_add_index_messagesDownSql, map[string]*bintree{}},
+	"1591277220_add_index_messages.up.sql":     &bintree{_1591277220_add_index_messagesUpSql, map[string]*bintree{}},
 	"doc.go":                                   &bintree{docGo, map[string]*bintree{}},
 }}
 

protocol/migrations/sqlite/1588665364_add_image_data.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE user_messages ADD COLUMN image_payload BLOB;
+ALTER TABLE user_messages ADD COLUMN image_type INT;
+ALTER TABLE user_messages ADD COLUMN image_base64 TEXT NOT NULL DEFAULT "";

protocol/migrations/sqlite/1589365189_add_pow_target.up.sql

@@ -0,0 +1 @@
+ALTER TABLE raw_messages ADD COLUMN pow_target REAL default 0.02;

protocol/migrations/sqlite/1591277220_add_index_messages.down.sql

@@ -0,0 +1,5 @@
+DROP INDEX idx_search_by_local_chat_id_sort_on_cursor;
+
+CREATE INDEX idx_search_by_chat_id ON  user_messages(
+    substr('0000000000000000000000000000000000000000000000000000000000000000' || clock_value, -64, 64) || id, chat_id, hide
+);

protocol/migrations/sqlite/1591277220_add_index_messages.up.sql

@@ -0,0 +1,3 @@
+DROP INDEX idx_search_by_chat_id;
+
+CREATE INDEX idx_search_by_local_chat_id_sort_on_cursor ON user_messages (local_chat_id ASC, substr('0000000000000000000000000000000000000000000000000000000000000000' || clock_value, -64, 64) || id DESC);