status-im/go-waku
2
0
Fork 0
mirror of https://github.com/status-im/go-waku.git synced 2025-01-26 21:49:26 +00:00
Code Issues Projects Releases Wiki Activity

chore: remove --store-message-db-vacuum (#883)

Browse Source
This commit is contained in:
richΛrd 2023-11-09 16:10:40 -04:00 committed by GitHub
parent 712febd32f
commit ff94b1faf0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 12 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/waku/flags.go
View File

@ -330,12 +330,6 @@ var (
Destination: &options.Store.DatabaseURL,
EnvVars: []string{"WAKUNODE2_STORE_MESSAGE_DB_URL"},
})
StoreMessageDBVacuum = altsrc.NewBoolFlag(&cli.BoolFlag{
Name: "store-message-db-vacuum",
Usage: "Enable database vacuuming at start.",
Destination: &options.Store.Vacuum,
EnvVars: []string{"WAKUNODE2_STORE_MESSAGE_DB_VACUUM"},
})
StoreMessageDBMigration = altsrc.NewBoolFlag(&cli.BoolFlag{
Name: "store-message-db-migration",
Usage: "Enable database migration at start.",

1
cmd/waku/main.go
View File

@ -65,7 +65,6 @@ func main() {
StoreMessageDBURL,
StoreMessageRetentionTime,
StoreMessageRetentionCapacity,
StoreMessageDBVacuum,
StoreMessageDBMigration,
FilterFlag,
FilterNode,

4
cmd/waku/node.go
View File

@ -110,9 +110,7 @@ func Execute(options NodeOptions) error {
var db *sql.DB
var migrationFn func(*sql.DB) error
if requiresDB(options) && options.Store.Migration {
dbSettings := dbutils.DBSettings{
Vacuum: options.Store.Vacuum,
}
dbSettings := dbutils.DBSettings{}
db, migrationFn, err = dbutils.ExtractDBAndMigration(options.Store.DatabaseURL, dbSettings, logger)
if err != nil {
return nonRecoverErrorMsg("could not connect to DB: %w", err)

1
cmd/waku/options.go
View File

@ -81,7 +81,6 @@ type StoreOptions struct {
RetentionMaxMessages int
//ResumeNodes []multiaddr.Multiaddr
Nodes []multiaddr.Multiaddr
Vacuum bool
Migration bool
}

2
cmd/waku/server/rest/utils_test.go
View File

@ -13,7 +13,7 @@ import (
func MemoryDB(t *testing.T) *persistence.DBStore {
var db *sql.DB
db, err := sqlite.NewDB(":memory:", false, utils.Logger())
db, err := sqlite.NewDB(":memory:", utils.Logger())
require.NoError(t, err)
dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations))

2
library/node.go
View File

@ -128,7 +128,7 @@ func NewNode(configJSON string) error {
if *config.EnableStore {
var db *sql.DB
var migrationFn func(*sql.DB) error
db, migrationFn, err = dbutils.ExtractDBAndMigration(*config.DatabaseURL, dbutils.DBSettings{Vacuum: true}, utils.Logger())
db, migrationFn, err = dbutils.ExtractDBAndMigration(*config.DatabaseURL, dbutils.DBSettings{}, utils.Logger())
if err != nil {
return err
}

46
waku/persistence/postgres/postgres.go
View File

@ -1,10 +1,8 @@
package postgres
import (
"context"
"database/sql"
"fmt"
"time"
"github.com/golang-migrate/migrate/v4/database"
"github.com/golang-migrate/migrate/v4/database/pgx"
@ -15,55 +13,13 @@ import (
"go.uber.org/zap"
)
func executeVacuum(db *sql.DB, logger *zap.Logger) error {
logger.Info("starting PostgreSQL database vacuuming")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
errCh := make(chan error)
go func() {
defer cancel()
_, err := db.Exec("VACUUM FULL")
if err != nil {
errCh <- err
}
}()
t := time.NewTicker(2 * time.Minute)
defer t.Stop()
loop:
for {
select {
case <-ctx.Done():
break loop
case err := <-errCh:
return err
case <-t.C:
logger.Info("still vacuuming...")
}
}
logger.Info("finished PostgreSQL database vacuuming")
return nil
}
// NewDB connects to postgres DB in the specified path
func NewDB(dburl string, shouldVacuum bool, logger *zap.Logger) (*sql.DB, error) {
func NewDB(dburl string, logger *zap.Logger) (*sql.DB, error) {
db, err := sql.Open("pgx", dburl)
if err != nil {
return nil, err
}
if shouldVacuum {
err := executeVacuum(db, logger)
if err != nil {
return nil, err
}
}
return db, nil
}

46
waku/persistence/sqlite/sqlite.go
View File

@ -1,11 +1,9 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"github.com/golang-migrate/migrate/v4/database"
"github.com/golang-migrate/migrate/v4/database/sqlite3"
@ -32,43 +30,8 @@ func addSqliteURLDefaults(dburl string) string {
return dburl
}
func executeVacuum(db *sql.DB, logger *zap.Logger) error {
logger.Info("starting sqlite database vacuuming")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
errCh := make(chan error)
go func() {
defer cancel()
_, err := db.Exec("VACUUM")
if err != nil {
errCh <- err
}
}()
t := time.NewTicker(2 * time.Minute)
defer t.Stop()
loop:
for {
select {
case <-ctx.Done():
break loop
case err := <-errCh:
return err
case <-t.C:
logger.Info("still vacuuming...")
}
}
logger.Info("finished sqlite database vacuuming")
return nil
}
// NewDB creates a sqlite3 DB in the specified path
func NewDB(dburl string, shouldVacuum bool, logger *zap.Logger) (*sql.DB, error) {
func NewDB(dburl string, logger *zap.Logger) (*sql.DB, error) {
db, err := sql.Open("sqlite3", addSqliteURLDefaults(dburl))
if err != nil {
return nil, err
@ -77,13 +40,6 @@ func NewDB(dburl string, shouldVacuum bool, logger *zap.Logger) (*sql.DB, error)
// Disable concurrent access as not supported by the driver
db.SetMaxOpenConns(1)
if shouldVacuum {
err := executeVacuum(db, logger)
if err != nil {
return nil, err
}
}
return db, nil
}

6
waku/persistence/utils/db.go
View File

@ -22,7 +22,7 @@ func validateDBUrl(val string) error {
// DBSettings hold db specific configuration settings required during the db initialization
type DBSettings struct {
Vacuum bool
// TODO: add any DB specific setting here
}
// ExtractDBAndMigration will return a database connection, and migration function that should be used depending on a database connection string
@ -50,10 +50,10 @@ func ExtractDBAndMigration(databaseURL string, dbSettings DBSettings, logger *za
dbParams := dbURLParts[1]
switch dbEngine {
case "sqlite3":
db, err = sqlite.NewDB(dbParams, dbSettings.Vacuum, logger)
db, err = sqlite.NewDB(dbParams, logger)
migrationFn = sqlite.Migrations
case "postgresql":
db, err = postgres.NewDB(dbURL, dbSettings.Vacuum, logger)
db, err = postgres.NewDB(dbURL, logger)
migrationFn = postgres.Migrations
default:
err = errors.New("unsupported database engine")

2
waku/v2/node/connectedness_test.go
View File

@ -70,7 +70,7 @@ func TestConnectionStatusChanges(t *testing.T) {
err = node2.Start(ctx)
require.NoError(t, err)
db, err := sqlite.NewDB(":memory:", false, utils.Logger())
db, err := sqlite.NewDB(":memory:", utils.Logger())
require.NoError(t, err)
dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations))
require.NoError(t, err)

2
waku/v2/node/wakunode2_test.go
View File

@ -242,7 +242,7 @@ func TestDecoupledStoreFromRelay(t *testing.T) {
defer subs[0].Unsubscribe()
// NODE2: Filter Client/Store
db, err := sqlite.NewDB(":memory:", false, utils.Logger())
db, err := sqlite.NewDB(":memory:", utils.Logger())
require.NoError(t, err)
dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations))
require.NoError(t, err)

2
waku/v2/protocol/store/utils_test.go
View File

@ -13,7 +13,7 @@ import (
func MemoryDB(t *testing.T) *persistence.DBStore {
var db *sql.DB
db, err := sqlite.NewDB(":memory:", false, utils.Logger())
db, err := sqlite.NewDB(":memory:", utils.Logger())
require.NoError(t, err)
dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations))

2
waku/v2/rendezvous/rendezvous_test.go
View File

@ -45,7 +45,7 @@ func TestRendezvous(t *testing.T) {
host1, err := tests.MakeHost(ctx, port1, rand.Reader)
require.NoError(t, err)
db, err := sqlite.NewDB(":memory:", false, utils.Logger())
db, err := sqlite.NewDB(":memory:", utils.Logger())
require.NoError(t, err)
err = sqlite.Migrations(db)