feat: db migrations (#246)

This commit is contained in:
Richard Ramos 2022-05-30 15:13:27 -04:00 committed by GitHub
parent 7c0206684f
commit b14f4a9aa8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 4823 additions and 111 deletions

View File

@ -67,6 +67,7 @@ test-ci: _before-cc test _after-cc
generate: generate:
go generate ./waku/v2/protocol/pb/generate.go go generate ./waku/v2/protocol/pb/generate.go
go generate ./waku/persistence/migrations/sql
coverage: coverage:
go test -count 1 -coverprofile=coverage.out ./... go test -count 1 -coverprofile=coverage.out ./...

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

10
go.mod
View File

@ -10,11 +10,15 @@ require (
contrib.go.opencensus.io/exporter/prometheus v0.4.1 contrib.go.opencensus.io/exporter/prometheus v0.4.1
github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d github.com/cruxic/go-hmac-drbg v0.0.0-20170206035330-84c46983886d
github.com/ethereum/go-ethereum v1.10.17 github.com/ethereum/go-ethereum v1.10.17
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/gogo/protobuf v1.3.2 github.com/gogo/protobuf v1.3.2
github.com/golang-migrate/migrate/v4 v4.15.2
github.com/golang/protobuf v1.5.2 github.com/golang/protobuf v1.5.2
github.com/gorilla/rpc v1.2.0 github.com/gorilla/rpc v1.2.0
github.com/ipfs/go-ds-sql v0.3.0 github.com/ipfs/go-ds-sql v0.3.0
github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log v1.0.5
github.com/kr/pretty v0.3.0 // indirect
github.com/lib/pq v1.10.3 // indirect
github.com/libp2p/go-libp2p v0.18.0 github.com/libp2p/go-libp2p v0.18.0
github.com/libp2p/go-libp2p-connmgr v0.3.1 github.com/libp2p/go-libp2p-connmgr v0.3.1
github.com/libp2p/go-libp2p-core v0.14.0 github.com/libp2p/go-libp2p-core v0.14.0
@ -24,8 +28,11 @@ require (
github.com/libp2p/go-msgio v0.1.0 github.com/libp2p/go-msgio v0.1.0
github.com/libp2p/go-tcp-transport v0.5.1 github.com/libp2p/go-tcp-transport v0.5.1
github.com/libp2p/go-ws-transport v0.6.1-0.20220221074654-eeaddb3c061d github.com/libp2p/go-ws-transport v0.6.1-0.20220221074654-eeaddb3c061d
github.com/mattn/go-sqlite3 v1.14.12 github.com/mattn/go-sqlite3 v2.0.2+incompatible
github.com/multiformats/go-multiaddr v0.5.0 github.com/multiformats/go-multiaddr v0.5.0
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/gomega v1.17.0 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/status-im/go-discover v0.0.0-20220406135310-85a2ce36f63e github.com/status-im/go-discover v0.0.0-20220406135310-85a2ce36f63e
github.com/status-im/go-waku-rendezvous v0.0.0-20211018070416-a93f3b70c432 github.com/status-im/go-waku-rendezvous v0.0.0-20211018070416-a93f3b70c432
github.com/stretchr/testify v1.7.1 github.com/stretchr/testify v1.7.1
@ -34,5 +41,6 @@ require (
go.opencensus.io v0.23.0 go.opencensus.io v0.23.0
go.uber.org/zap v1.21.0 go.uber.org/zap v1.21.0
golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29
golang.org/x/tools v0.1.7 // indirect
google.golang.org/protobuf v1.28.0 // indirect google.golang.org/protobuf v1.28.0 // indirect
) )

1129
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,320 @@
// Code generated by go-bindata. DO NOT EDIT.
// sources:
// 1_messages.down.sql (0)
// 1_messages.up.sql (464B)
// doc.go (74B)
package migrations
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
)
func bindataRead(data []byte, name string) ([]byte, error) {
gz, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return nil, fmt.Errorf("read %q: %w", name, err)
}
var buf bytes.Buffer
_, err = io.Copy(&buf, gz)
clErr := gz.Close()
if err != nil {
return nil, fmt.Errorf("read %q: %w", name, err)
}
if clErr != nil {
return nil, err
}
return buf.Bytes(), nil
}
type asset struct {
bytes []byte
info os.FileInfo
digest [sha256.Size]byte
}
type bindataFileInfo struct {
name string
size int64
mode os.FileMode
modTime time.Time
}
func (fi bindataFileInfo) Name() string {
return fi.name
}
func (fi bindataFileInfo) Size() int64 {
return fi.size
}
func (fi bindataFileInfo) Mode() os.FileMode {
return fi.mode
}
func (fi bindataFileInfo) ModTime() time.Time {
return fi.modTime
}
func (fi bindataFileInfo) IsDir() bool {
return false
}
func (fi bindataFileInfo) Sys() interface{} {
return nil
}
var __1_messagesDownSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x01\x00\x00\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00")
func _1_messagesDownSqlBytes() ([]byte, error) {
return bindataRead(
__1_messagesDownSql,
"1_messages.down.sql",
)
}
func _1_messagesDownSql() (*asset, error) {
bytes, err := _1_messagesDownSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "1_messages.down.sql", size: 0, mode: os.FileMode(0664), modTime: time.Unix(1653588136, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}
return a, nil
}
var __1_messagesUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x90\x41\x4f\x83\x40\x10\x85\xcf\xec\xaf\x98\x23\x24\x1c\xbc\x73\x82\xb2\xd5\x89\xeb\xae\x59\x86\xb4\x3d\x19\x0a\x13\xb3\x89\x2c\x84\xa5\x8d\xfe\x7b\x93\xc6\x1a\x52\x34\x7a\xfe\x76\xdf\x9b\xf7\x6d\xac\xcc\x49\x02\xe5\x85\x92\x80\x5b\xd0\x86\x40\xee\xb1\xa2\x0a\x7a\x0e\xa1\x79\x65\x88\x45\xe4\x3a\x28\x94\x29\x52\x11\x4d\xdc\xb2\x3b\xf3\x44\xae\xe7\x30\x37\xfd\x08\xa8\x49\xde\x4b\x7b\xf9\xa9\x6b\xa5\x52\x11\x05\xf6\xdd\x1f\x4f\xda\xc1\xcf\xec\x67\x1a\x46\xd7\x5e\xb2\x97\x70\x3c\x1d\xc3\xe9\xf8\x0b\x6b\x3e\xde\x86\xe6\xfb\x9e\x33\x4f\xc1\x0d\x7e\x55\x01\xa5\xdc\xe6\xb5\x22\xb8\x4b\x45\xb4\x31\xba\x22\x9b\xa3\xa6\xeb\x28\xf4\x1d\xbf\xc3\xb3\xc5\xa7\xdc\x1e\xe0\x51\x1e\x20\x76\x5d\x0a\x8b\xe2\x44\x24\xb0\x43\x7a\x30\x35\x81\x35\x3b\x2c\x33\x21\xbe\x64\xa1\x2e\xe5\xfe\x67\x59\x2f\xb7\xd3\x8d\xbe\xa2\xf8\x06\x25\xd9\x7f\xf2\xd6\xbe\x17\x89\x2b\x98\x64\x9f\x01\x00\x00\xff\xff\x59\xcd\x67\xb6\xd0\x01\x00\x00")
func _1_messagesUpSqlBytes() ([]byte, error) {
return bindataRead(
__1_messagesUpSql,
"1_messages.up.sql",
)
}
func _1_messagesUpSql() (*asset, error) {
bytes, err := _1_messagesUpSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "1_messages.up.sql", size: 464, mode: os.FileMode(0664), modTime: time.Unix(1653581124, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x4, 0xd8, 0x47, 0x7b, 0xe, 0x47, 0x2a, 0x4b, 0x48, 0x36, 0x23, 0x93, 0x28, 0xb3, 0x1e, 0x5, 0x76, 0x64, 0x73, 0xb, 0x2b, 0x5b, 0x10, 0x62, 0x36, 0x21, 0x6f, 0xa3, 0x3c, 0xdd, 0xe2, 0xcf}}
return a, nil
}
var _docGo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x2c\xc9\xb1\x0d\xc4\x20\x0c\x05\xd0\x9e\x29\xfe\x02\xd8\xfd\x6d\xe3\x4b\xac\x2f\x44\x82\x09\x78\x7f\xa5\x49\xfd\xa6\x1d\xdd\xe8\xd8\xcf\x55\x8a\x2a\xe3\x47\x1f\xbe\x2c\x1d\x8c\xfa\x6f\xe3\xb4\x34\xd4\xd9\x89\xbb\x71\x59\xb6\x18\x1b\x35\x20\xa2\x9f\x0a\x03\xa2\xe5\x0d\x00\x00\xff\xff\x60\xcd\x06\xbe\x4a\x00\x00\x00")
func docGoBytes() ([]byte, error) {
return bindataRead(
_docGo,
"doc.go",
)
}
func docGo() (*asset, error) {
bytes, err := docGoBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "doc.go", size: 74, mode: os.FileMode(0664), modTime: time.Unix(1653581182, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xde, 0x7c, 0x28, 0xcd, 0x47, 0xf2, 0xfa, 0x7c, 0x51, 0x2d, 0xd8, 0x38, 0xb, 0xb0, 0x34, 0x9d, 0x4c, 0x62, 0xa, 0x9e, 0x28, 0xc3, 0x31, 0x23, 0xd9, 0xbb, 0x89, 0x9f, 0xa0, 0x89, 0x1f, 0xe8}}
return a, nil
}
// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
canonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[canonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
}
return a.bytes, nil
}
return nil, fmt.Errorf("Asset %s not found", name)
}
// AssetString returns the asset contents as a string (instead of a []byte).
func AssetString(name string) (string, error) {
data, err := Asset(name)
return string(data), err
}
// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
a, err := Asset(name)
if err != nil {
panic("asset: Asset(" + name + "): " + err.Error())
}
return a
}
// MustAssetString is like AssetString but panics when Asset would return an
// error. It simplifies safe initialization of global variables.
func MustAssetString(name string) string {
return string(MustAsset(name))
}
// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
canonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[canonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
}
return a.info, nil
}
return nil, fmt.Errorf("AssetInfo %s not found", name)
}
// AssetDigest returns the digest of the file with the given name. It returns an
// error if the asset could not be found or the digest could not be loaded.
func AssetDigest(name string) ([sha256.Size]byte, error) {
canonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[canonicalName]; ok {
a, err := f()
if err != nil {
return [sha256.Size]byte{}, fmt.Errorf("AssetDigest %s can't read by error: %v", name, err)
}
return a.digest, nil
}
return [sha256.Size]byte{}, fmt.Errorf("AssetDigest %s not found", name)
}
// Digests returns a map of all known files and their checksums.
func Digests() (map[string][sha256.Size]byte, error) {
mp := make(map[string][sha256.Size]byte, len(_bindata))
for name := range _bindata {
a, err := _bindata[name]()
if err != nil {
return nil, err
}
mp[name] = a.digest
}
return mp, nil
}
// AssetNames returns the names of the assets.
func AssetNames() []string {
names := make([]string, 0, len(_bindata))
for name := range _bindata {
names = append(names, name)
}
return names
}
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
"1_messages.down.sql": _1_messagesDownSql,
"1_messages.up.sql": _1_messagesUpSql,
"doc.go": docGo,
}
// AssetDebug is true if the assets were built with the debug flag enabled.
const AssetDebug = false
// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
// data/
// foo.txt
// img/
// a.png
// b.png
// then AssetDir("data") would return []string{"foo.txt", "img"},
// AssetDir("data/img") would return []string{"a.png", "b.png"},
// AssetDir("foo.txt") and AssetDir("notexist") would return an error, and
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
node := _bintree
if len(name) != 0 {
canonicalName := strings.Replace(name, "\\", "/", -1)
pathList := strings.Split(canonicalName, "/")
for _, p := range pathList {
node = node.Children[p]
if node == nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
}
}
if node.Func != nil {
return nil, fmt.Errorf("Asset %s not found", name)
}
rv := make([]string, 0, len(node.Children))
for childName := range node.Children {
rv = append(rv, childName)
}
return rv, nil
}
type bintree struct {
Func func() (*asset, error)
Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
"1_messages.down.sql": {_1_messagesDownSql, map[string]*bintree{}},
"1_messages.up.sql": {_1_messagesUpSql, map[string]*bintree{}},
"doc.go": {docGo, map[string]*bintree{}},
}}
// RestoreAsset restores an asset under the given directory.
func RestoreAsset(dir, name string) error {
data, err := Asset(name)
if err != nil {
return err
}
info, err := AssetInfo(name)
if err != nil {
return err
}
err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
if err != nil {
return err
}
err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
if err != nil {
return err
}
return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
}
// RestoreAssets restores an asset under the given directory recursively.
func RestoreAssets(dir, name string) error {
children, err := AssetDir(name)
// File
if err != nil {
return RestoreAsset(dir, name)
}
// Dir
for _, child := range children {
err = RestoreAssets(dir, filepath.Join(name, child))
if err != nil {
return err
}
}
return nil
}
func _filePath(dir, name string) string {
canonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(canonicalName, "/")...)...)
}

View File

@ -0,0 +1,48 @@
package migrations
import (
"database/sql"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/sqlite3"
bindata "github.com/golang-migrate/migrate/v4/source/go_bindata"
)
// Migrate applies migrations.
func Migrate(db *sql.DB) error {
return migrateDB(db, bindata.Resource(
AssetNames(),
func(name string) ([]byte, error) {
return Asset(name)
},
))
}
// Migrate database using provided resources.
func migrateDB(db *sql.DB, resources *bindata.AssetSource) error {
source, err := bindata.WithInstance(resources)
if err != nil {
return err
}
driver, err := sqlite3.WithInstance(db, &sqlite3.Config{
MigrationsTable: "gowaku_" + sqlite3.DefaultMigrationsTable,
})
if err != nil {
return err
}
m, err := migrate.NewWithInstance(
"go-bindata",
source,
"sqlite",
driver)
if err != nil {
return err
}
if err = m.Up(); err != migrate.ErrNoChange {
return err
}
return nil
}

View File

@ -0,0 +1,3 @@
DROP INDEX IF EXISTS message_senderTimestamp;
DROP INDEX IF EXISTS message_receiverTimestamp;
DROP TABLE IF EXISTS message;

View File

@ -0,0 +1,13 @@
CREATE TABLE IF NOT EXISTS message (
id BLOB,
receiverTimestamp INTEGER NOT NULL,
senderTimestamp INTEGER NOT NULL,
contentTopic BLOB NOT NULL,
pubsubTopic BLOB NOT NULL,
payload BLOB,
version INTEGER NOT NULL DEFAULT 0,
CONSTRAINT messageIndex PRIMARY KEY (id, pubsubTopic)
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS message_senderTimestamp ON message(senderTimestamp);
CREATE INDEX IF NOT EXISTS message_receiverTimestamp ON message(receiverTimestamp);

View File

@ -0,0 +1,3 @@
package sql
//go:generate go-bindata -pkg migrations -o ../bindata.go ./

View File

@ -8,6 +8,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/status-im/go-waku/waku/persistence/migrations"
"github.com/status-im/go-waku/waku/v2/protocol" "github.com/status-im/go-waku/waku/v2/protocol"
"github.com/status-im/go-waku/waku/v2/protocol/pb" "github.com/status-im/go-waku/waku/v2/protocol/pb"
"github.com/status-im/go-waku/waku/v2/utils" "github.com/status-im/go-waku/waku/v2/utils"
@ -116,7 +117,7 @@ func NewDBStore(log *zap.Logger, options ...DBOption) (*DBStore, error) {
return nil, fmt.Errorf("unable to set journal_mode to WAL. actual mode %s", mode) return nil, fmt.Errorf("unable to set journal_mode to WAL. actual mode %s", mode)
} }
err = result.createTable() err = migrations.Migrate(result.db)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -132,27 +133,6 @@ func NewDBStore(log *zap.Logger, options ...DBOption) (*DBStore, error) {
return result, nil return result, nil
} }
func (d *DBStore) createTable() error {
sqlStmt := `CREATE TABLE IF NOT EXISTS message (
id BLOB,
receiverTimestamp INTEGER NOT NULL,
senderTimestamp INTEGER NOT NULL,
contentTopic BLOB NOT NULL,
pubsubTopic BLOB NOT NULL,
payload BLOB,
version INTEGER NOT NULL DEFAULT 0,
CONSTRAINT messageIndex PRIMARY KEY (id, pubsubTopic)
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS message_senderTimestamp ON message(senderTimestamp);
CREATE INDEX IF NOT EXISTS message_receiverTimestamp ON message(receiverTimestamp);`
_, err := d.db.Exec(sqlStmt)
if err != nil {
return err
}
return nil
}
func (d *DBStore) cleanOlderRecords() error { func (d *DBStore) cleanOlderRecords() error {
d.log.Debug("Cleaning older records...") d.log.Debug("Cleaning older records...")