feat: db migrations (#246)

Makefile

@@ -67,6 +67,7 @@ test-ci: _before-cc test _after-cc
 
 generate:
 	go generate ./waku/v2/protocol/pb/generate.go
+	go generate ./waku/persistence/migrations/sql
 
 coverage:
 	go test  -count 1 -coverprofile=coverage.out ./...

go.mod

@@ -10,11 +10,15 @@ require (
 	contrib.go.opencensus.io/exporter/prometheus v0.4.1
 	github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d
 	github.com/ethereum/go-ethereum v1.10.17
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/gogo/protobuf v1.3.2
+	github.com/golang-migrate/migrate/v4 v4.15.2
 	github.com/golang/protobuf v1.5.2
 	github.com/gorilla/rpc v1.2.0
 	github.com/ipfs/go-ds-sql v0.3.0
 	github.com/ipfs/go-log v1.0.5
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/lib/pq v1.10.3 // indirect
 	github.com/libp2p/go-libp2p v0.18.0
 	github.com/libp2p/go-libp2p-connmgr v0.3.1
 	github.com/libp2p/go-libp2p-core v0.14.0
@@ -24,8 +28,11 @@ require (
 	github.com/libp2p/go-msgio v0.1.0
 	github.com/libp2p/go-tcp-transport v0.5.1
 	github.com/libp2p/go-ws-transport v0.6.1-0.20220221074654-eeaddb3c061d
-	github.com/mattn/go-sqlite3 v1.14.12
+	github.com/mattn/go-sqlite3 v2.0.2+incompatible
 	github.com/multiformats/go-multiaddr v0.5.0
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.17.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.0 // indirect
 	github.com/status-im/go-discover v0.0.0-20220406135310-85a2ce36f63e
 	github.com/status-im/go-waku-rendezvous v0.0.0-20211018070416-a93f3b70c432
 	github.com/stretchr/testify v1.7.1
@@ -34,5 +41,6 @@ require (
 	go.opencensus.io v0.23.0
 	go.uber.org/zap v1.21.0
 	golang.org/x/sync v0.0.0-20220513210516-0976fa681c29
+	golang.org/x/tools v0.1.7 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 )

waku/persistence/migrations/bindata.go

@@ -0,0 +1,320 @@
+// Code generated by go-bindata. DO NOT EDIT.
+// sources:
+// 1_messages.down.sql (0)
+// 1_messages.up.sql (464B)
+// doc.go (74B)
+
+package migrations
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func bindataRead(data []byte, name string) ([]byte, error) {
+	gz, err := gzip.NewReader(bytes.NewBuffer(data))
+	if err != nil {
+		return nil, fmt.Errorf("read %q: %w", name, err)
+	}
+
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, gz)
+	clErr := gz.Close()
+
+	if err != nil {
+		return nil, fmt.Errorf("read %q: %w", name, err)
+	}
+	if clErr != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+type asset struct {
+	bytes  []byte
+	info   os.FileInfo
+	digest [sha256.Size]byte
+}
+
+type bindataFileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+}
+
+func (fi bindataFileInfo) Name() string {
+	return fi.name
+}
+func (fi bindataFileInfo) Size() int64 {
+	return fi.size
+}
+func (fi bindataFileInfo) Mode() os.FileMode {
+	return fi.mode
+}
+func (fi bindataFileInfo) ModTime() time.Time {
+	return fi.modTime
+}
+func (fi bindataFileInfo) IsDir() bool {
+	return false
+}
+func (fi bindataFileInfo) Sys() interface{} {
+	return nil
+}
+
+var __1_messagesDownSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x01\x00\x00\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00")
+
+func _1_messagesDownSqlBytes() ([]byte, error) {
+	return bindataRead(
+		__1_messagesDownSql,
+		"1_messages.down.sql",
+	)
+}
+
+func _1_messagesDownSql() (*asset, error) {
+	bytes, err := _1_messagesDownSqlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "1_messages.down.sql", size: 0, mode: os.FileMode(0664), modTime: time.Unix(1653588136, 0)}
+	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}
+	return a, nil
+}
+
+var __1_messagesUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x90\x41\x4f\x83\x40\x10\x85\xcf\xec\xaf\x98\x23\x24\x1c\xbc\x73\x82\xb2\xd5\x89\xeb\xae\x59\x86\xb4\x3d\x19\x0a\x13\xb3\x89\x2c\x84\xa5\x8d\xfe\x7b\x93\xc6\x1a\x52\x34\x7a\xfe\x76\xdf\x9b\xf7\x6d\xac\xcc\x49\x02\xe5\x85\x92\x80\x5b\xd0\x86\x40\xee\xb1\xa2\x0a\x7a\x0e\xa1\x79\x65\x88\x45\xe4\x3a\x28\x94\x29\x52\x11\x4d\xdc\xb2\x3b\xf3\x44\xae\xe7\x30\x37\xfd\x08\xa8\x49\xde\x4b\x7b\xf9\xa9\x6b\xa5\x52\x11\x05\xf6\xdd\x1f\x4f\xda\xc1\xcf\xec\x67\x1a\x46\xd7\x5e\xb2\x97\x70\x3c\x1d\xc3\xe9\xf8\x0b\x6b\x3e\xde\x86\xe6\xfb\x9e\x33\x4f\xc1\x0d\x7e\x55\x01\xa5\xdc\xe6\xb5\x22\xb8\x4b\x45\xb4\x31\xba\x22\x9b\xa3\xa6\xeb\x28\xf4\x1d\xbf\xc3\xb3\xc5\xa7\xdc\x1e\xe0\x51\x1e\x20\x76\x5d\x0a\x8b\xe2\x44\x24\xb0\x43\x7a\x30\x35\x81\x35\x3b\x2c\x33\x21\xbe\x64\xa1\x2e\xe5\xfe\x67\x59\x2f\xb7\xd3\x8d\xbe\xa2\xf8\x06\x25\xd9\x7f\xf2\xd6\xbe\x17\x89\x2b\x98\x64\x9f\x01\x00\x00\xff\xff\x59\xcd\x67\xb6\xd0\x01\x00\x00")
+
+func _1_messagesUpSqlBytes() ([]byte, error) {
+	return bindataRead(
+		__1_messagesUpSql,
+		"1_messages.up.sql",
+	)
+}
+
+func _1_messagesUpSql() (*asset, error) {
+	bytes, err := _1_messagesUpSqlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "1_messages.up.sql", size: 464, mode: os.FileMode(0664), modTime: time.Unix(1653581124, 0)}
+	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x4, 0xd8, 0x47, 0x7b, 0xe, 0x47, 0x2a, 0x4b, 0x48, 0x36, 0x23, 0x93, 0x28, 0xb3, 0x1e, 0x5, 0x76, 0x64, 0x73, 0xb, 0x2b, 0x5b, 0x10, 0x62, 0x36, 0x21, 0x6f, 0xa3, 0x3c, 0xdd, 0xe2, 0xcf}}
+	return a, nil
+}
+
+var _docGo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x2c\xc9\xb1\x0d\xc4\x20\x0c\x05\xd0\x9e\x29\xfe\x02\xd8\xfd\x6d\xe3\x4b\xac\x2f\x44\x82\x09\x78\x7f\xa5\x49\xfd\xa6\x1d\xdd\xe8\xd8\xcf\x55\x8a\x2a\xe3\x47\x1f\xbe\x2c\x1d\x8c\xfa\x6f\xe3\xb4\x34\xd4\xd9\x89\xbb\x71\x59\xb6\x18\x1b\x35\x20\xa2\x9f\x0a\x03\xa2\xe5\x0d\x00\x00\xff\xff\x60\xcd\x06\xbe\x4a\x00\x00\x00")
+
+func docGoBytes() ([]byte, error) {
+	return bindataRead(
+		_docGo,
+		"doc.go",
+	)
+}
+
+func docGo() (*asset, error) {
+	bytes, err := docGoBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "doc.go", size: 74, mode: os.FileMode(0664), modTime: time.Unix(1653581182, 0)}
+	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xde, 0x7c, 0x28, 0xcd, 0x47, 0xf2, 0xfa, 0x7c, 0x51, 0x2d, 0xd8, 0x38, 0xb, 0xb0, 0x34, 0x9d, 0x4c, 0x62, 0xa, 0x9e, 0x28, 0xc3, 0x31, 0x23, 0xd9, 0xbb, 0x89, 0x9f, 0xa0, 0x89, 0x1f, 0xe8}}
+	return a, nil
+}
+
+// Asset loads and returns the asset for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+func Asset(name string) ([]byte, error) {
+	canonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[canonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
+		}
+		return a.bytes, nil
+	}
+	return nil, fmt.Errorf("Asset %s not found", name)
+}
+
+// AssetString returns the asset contents as a string (instead of a []byte).
+func AssetString(name string) (string, error) {
+	data, err := Asset(name)
+	return string(data), err
+}
+
+// MustAsset is like Asset but panics when Asset would return an error.
+// It simplifies safe initialization of global variables.
+func MustAsset(name string) []byte {
+	a, err := Asset(name)
+	if err != nil {
+		panic("asset: Asset(" + name + "): " + err.Error())
+	}
+
+	return a
+}
+
+// MustAssetString is like AssetString but panics when Asset would return an
+// error. It simplifies safe initialization of global variables.
+func MustAssetString(name string) string {
+	return string(MustAsset(name))
+}
+
+// AssetInfo loads and returns the asset info for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+func AssetInfo(name string) (os.FileInfo, error) {
+	canonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[canonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
+		}
+		return a.info, nil
+	}
+	return nil, fmt.Errorf("AssetInfo %s not found", name)
+}
+
+// AssetDigest returns the digest of the file with the given name. It returns an
+// error if the asset could not be found or the digest could not be loaded.
+func AssetDigest(name string) ([sha256.Size]byte, error) {
+	canonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[canonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return [sha256.Size]byte{}, fmt.Errorf("AssetDigest %s can't read by error: %v", name, err)
+		}
+		return a.digest, nil
+	}
+	return [sha256.Size]byte{}, fmt.Errorf("AssetDigest %s not found", name)
+}
+
+// Digests returns a map of all known files and their checksums.
+func Digests() (map[string][sha256.Size]byte, error) {
+	mp := make(map[string][sha256.Size]byte, len(_bindata))
+	for name := range _bindata {
+		a, err := _bindata[name]()
+		if err != nil {
+			return nil, err
+		}
+		mp[name] = a.digest
+	}
+	return mp, nil
+}
+
+// AssetNames returns the names of the assets.
+func AssetNames() []string {
+	names := make([]string, 0, len(_bindata))
+	for name := range _bindata {
+		names = append(names, name)
+	}
+	return names
+}
+
+// _bindata is a table, holding each asset generator, mapped to its name.
+var _bindata = map[string]func() (*asset, error){
+	"1_messages.down.sql": _1_messagesDownSql,
+	"1_messages.up.sql":   _1_messagesUpSql,
+	"doc.go":              docGo,
+}
+
+// AssetDebug is true if the assets were built with the debug flag enabled.
+const AssetDebug = false
+
+// AssetDir returns the file names below a certain
+// directory embedded in the file by go-bindata.
+// For example if you run go-bindata on data/... and data contains the
+// following hierarchy:
+//     data/
+//       foo.txt
+//       img/
+//         a.png
+//         b.png
+// then AssetDir("data") would return []string{"foo.txt", "img"},
+// AssetDir("data/img") would return []string{"a.png", "b.png"},
+// AssetDir("foo.txt") and AssetDir("notexist") would return an error, and
+// AssetDir("") will return []string{"data"}.
+func AssetDir(name string) ([]string, error) {
+	node := _bintree
+	if len(name) != 0 {
+		canonicalName := strings.Replace(name, "\\", "/", -1)
+		pathList := strings.Split(canonicalName, "/")
+		for _, p := range pathList {
+			node = node.Children[p]
+			if node == nil {
+				return nil, fmt.Errorf("Asset %s not found", name)
+			}
+		}
+	}
+	if node.Func != nil {
+		return nil, fmt.Errorf("Asset %s not found", name)
+	}
+	rv := make([]string, 0, len(node.Children))
+	for childName := range node.Children {
+		rv = append(rv, childName)
+	}
+	return rv, nil
+}
+
+type bintree struct {
+	Func     func() (*asset, error)
+	Children map[string]*bintree
+}
+
+var _bintree = &bintree{nil, map[string]*bintree{
+	"1_messages.down.sql": {_1_messagesDownSql, map[string]*bintree{}},
+	"1_messages.up.sql": {_1_messagesUpSql, map[string]*bintree{}},
+	"doc.go": {docGo, map[string]*bintree{}},
+}}
+
+// RestoreAsset restores an asset under the given directory.
+func RestoreAsset(dir, name string) error {
+	data, err := Asset(name)
+	if err != nil {
+		return err
+	}
+	info, err := AssetInfo(name)
+	if err != nil {
+		return err
+	}
+	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
+	if err != nil {
+		return err
+	}
+	return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
+}
+
+// RestoreAssets restores an asset under the given directory recursively.
+func RestoreAssets(dir, name string) error {
+	children, err := AssetDir(name)
+	// File
+	if err != nil {
+		return RestoreAsset(dir, name)
+	}
+	// Dir
+	for _, child := range children {
+		err = RestoreAssets(dir, filepath.Join(name, child))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func _filePath(dir, name string) string {
+	canonicalName := strings.Replace(name, "\\", "/", -1)
+	return filepath.Join(append([]string{dir}, strings.Split(canonicalName, "/")...)...)
+}

waku/persistence/migrations/migrate.go

@@ -0,0 +1,48 @@
+package migrations
+
+import (
+	"database/sql"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/sqlite3"
+	bindata "github.com/golang-migrate/migrate/v4/source/go_bindata"
+)
+
+// Migrate applies migrations.
+func Migrate(db *sql.DB) error {
+	return migrateDB(db, bindata.Resource(
+		AssetNames(),
+		func(name string) ([]byte, error) {
+			return Asset(name)
+		},
+	))
+}
+
+// Migrate database using provided resources.
+func migrateDB(db *sql.DB, resources *bindata.AssetSource) error {
+	source, err := bindata.WithInstance(resources)
+	if err != nil {
+		return err
+	}
+
+	driver, err := sqlite3.WithInstance(db, &sqlite3.Config{
+		MigrationsTable: "gowaku_" + sqlite3.DefaultMigrationsTable,
+	})
+	if err != nil {
+		return err
+	}
+
+	m, err := migrate.NewWithInstance(
+		"go-bindata",
+		source,
+		"sqlite",
+		driver)
+	if err != nil {
+		return err
+	}
+
+	if err = m.Up(); err != migrate.ErrNoChange {
+		return err
+	}
+	return nil
+}

waku/persistence/migrations/sql/1_messages.down.sql

@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS message_senderTimestamp;
+DROP INDEX IF EXISTS message_receiverTimestamp;
+DROP TABLE IF EXISTS message;

waku/persistence/migrations/sql/1_messages.up.sql

@@ -0,0 +1,13 @@
+CREATE TABLE IF NOT EXISTS message (
+	id BLOB,
+	receiverTimestamp INTEGER NOT NULL,
+	senderTimestamp INTEGER NOT NULL,
+	contentTopic BLOB NOT NULL,
+	pubsubTopic BLOB NOT NULL,
+	payload BLOB,
+	version INTEGER NOT NULL DEFAULT 0,
+	CONSTRAINT messageIndex PRIMARY KEY (id, pubsubTopic)
+) WITHOUT ROWID;
+
+CREATE INDEX IF NOT EXISTS message_senderTimestamp ON message(senderTimestamp);
+CREATE INDEX IF NOT EXISTS message_receiverTimestamp ON message(receiverTimestamp);

waku/persistence/migrations/sql/doc.go

@@ -0,0 +1,3 @@
+package sql
+
+//go:generate go-bindata -pkg migrations -o ../bindata.go ./

waku/persistence/store.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/status-im/go-waku/waku/persistence/migrations"
 	"github.com/status-im/go-waku/waku/v2/protocol"
 	"github.com/status-im/go-waku/waku/v2/protocol/pb"
 	"github.com/status-im/go-waku/waku/v2/utils"
@@ -116,7 +117,7 @@ func NewDBStore(log *zap.Logger, options ...DBOption) (*DBStore, error) {
 		return nil, fmt.Errorf("unable to set journal_mode to WAL. actual mode %s", mode)
 	}
 
-	err = result.createTable()
+	err = migrations.Migrate(result.db)
 	if err != nil {
 		return nil, err
 	}
@@ -132,27 +133,6 @@ func NewDBStore(log *zap.Logger, options ...DBOption) (*DBStore, error) {
 	return result, nil
 }
 
-func (d *DBStore) createTable() error {
-	sqlStmt := `CREATE TABLE IF NOT EXISTS message (
-		id BLOB,
-		receiverTimestamp INTEGER NOT NULL,
-		senderTimestamp INTEGER NOT NULL,
-		contentTopic BLOB NOT NULL,
-		pubsubTopic BLOB NOT NULL,
-		payload BLOB,
-		version INTEGER NOT NULL DEFAULT 0,
-		CONSTRAINT messageIndex PRIMARY KEY (id, pubsubTopic)
-	) WITHOUT ROWID;
-	
-	CREATE INDEX IF NOT EXISTS message_senderTimestamp ON message(senderTimestamp);
-	CREATE INDEX IF NOT EXISTS message_receiverTimestamp ON message(receiverTimestamp);`
-	_, err := d.db.Exec(sqlStmt)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 func (d *DBStore) cleanOlderRecords() error {
 	d.log.Debug("Cleaning older records...")