From ff94b1faf035bab19d1ac0a12075d4374736d8d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?rich=CE=9Brd?= Date: Thu, 9 Nov 2023 16:10:40 -0400 Subject: [PATCH] chore: remove --store-message-db-vacuum (#883) --- cmd/waku/flags.go | 6 ---- cmd/waku/main.go | 1 - cmd/waku/node.go | 4 +-- cmd/waku/options.go | 1 - cmd/waku/server/rest/utils_test.go | 2 +- library/node.go | 2 +- waku/persistence/postgres/postgres.go | 46 +-------------------------- waku/persistence/sqlite/sqlite.go | 46 +-------------------------- waku/persistence/utils/db.go | 6 ++-- waku/v2/node/connectedness_test.go | 2 +- waku/v2/node/wakunode2_test.go | 2 +- waku/v2/protocol/store/utils_test.go | 2 +- waku/v2/rendezvous/rendezvous_test.go | 2 +- 13 files changed, 12 insertions(+), 110 deletions(-) diff --git a/cmd/waku/flags.go b/cmd/waku/flags.go index 860d3eda..a981253a 100644 --- a/cmd/waku/flags.go +++ b/cmd/waku/flags.go @@ -330,12 +330,6 @@ var ( Destination: &options.Store.DatabaseURL, EnvVars: []string{"WAKUNODE2_STORE_MESSAGE_DB_URL"}, }) - StoreMessageDBVacuum = altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: "store-message-db-vacuum", - Usage: "Enable database vacuuming at start.", - Destination: &options.Store.Vacuum, - EnvVars: []string{"WAKUNODE2_STORE_MESSAGE_DB_VACUUM"}, - }) StoreMessageDBMigration = altsrc.NewBoolFlag(&cli.BoolFlag{ Name: "store-message-db-migration", Usage: "Enable database migration at start.", diff --git a/cmd/waku/main.go b/cmd/waku/main.go index 3e74632a..ce6c3a17 100644 --- a/cmd/waku/main.go +++ b/cmd/waku/main.go @@ -65,7 +65,6 @@ func main() { StoreMessageDBURL, StoreMessageRetentionTime, StoreMessageRetentionCapacity, - StoreMessageDBVacuum, StoreMessageDBMigration, FilterFlag, FilterNode, diff --git a/cmd/waku/node.go b/cmd/waku/node.go index 534b1e00..a3bd6c4e 100644 --- a/cmd/waku/node.go +++ b/cmd/waku/node.go @@ -110,9 +110,7 @@ func Execute(options NodeOptions) error { var db *sql.DB var migrationFn func(*sql.DB) error if requiresDB(options) && options.Store.Migration { - dbSettings := dbutils.DBSettings{ - Vacuum: options.Store.Vacuum, - } + dbSettings := dbutils.DBSettings{} db, migrationFn, err = dbutils.ExtractDBAndMigration(options.Store.DatabaseURL, dbSettings, logger) if err != nil { return nonRecoverErrorMsg("could not connect to DB: %w", err) diff --git a/cmd/waku/options.go b/cmd/waku/options.go index 8fd48eb8..d85f094a 100644 --- a/cmd/waku/options.go +++ b/cmd/waku/options.go @@ -81,7 +81,6 @@ type StoreOptions struct { RetentionMaxMessages int //ResumeNodes []multiaddr.Multiaddr Nodes []multiaddr.Multiaddr - Vacuum bool Migration bool } diff --git a/cmd/waku/server/rest/utils_test.go b/cmd/waku/server/rest/utils_test.go index 7f3f5a02..12797f4a 100644 --- a/cmd/waku/server/rest/utils_test.go +++ b/cmd/waku/server/rest/utils_test.go @@ -13,7 +13,7 @@ import ( func MemoryDB(t *testing.T) *persistence.DBStore { var db *sql.DB - db, err := sqlite.NewDB(":memory:", false, utils.Logger()) + db, err := sqlite.NewDB(":memory:", utils.Logger()) require.NoError(t, err) dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations)) diff --git a/library/node.go b/library/node.go index 2596a2a1..eaacd8da 100644 --- a/library/node.go +++ b/library/node.go @@ -128,7 +128,7 @@ func NewNode(configJSON string) error { if *config.EnableStore { var db *sql.DB var migrationFn func(*sql.DB) error - db, migrationFn, err = dbutils.ExtractDBAndMigration(*config.DatabaseURL, dbutils.DBSettings{Vacuum: true}, utils.Logger()) + db, migrationFn, err = dbutils.ExtractDBAndMigration(*config.DatabaseURL, dbutils.DBSettings{}, utils.Logger()) if err != nil { return err } diff --git a/waku/persistence/postgres/postgres.go b/waku/persistence/postgres/postgres.go index 565ee8f4..e3ec57bc 100644 --- a/waku/persistence/postgres/postgres.go +++ b/waku/persistence/postgres/postgres.go @@ -1,10 +1,8 @@ package postgres import ( - "context" "database/sql" "fmt" - "time" "github.com/golang-migrate/migrate/v4/database" "github.com/golang-migrate/migrate/v4/database/pgx" @@ -15,55 +13,13 @@ import ( "go.uber.org/zap" ) -func executeVacuum(db *sql.DB, logger *zap.Logger) error { - logger.Info("starting PostgreSQL database vacuuming") - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - errCh := make(chan error) - - go func() { - defer cancel() - _, err := db.Exec("VACUUM FULL") - if err != nil { - errCh <- err - } - }() - - t := time.NewTicker(2 * time.Minute) - defer t.Stop() - -loop: - for { - select { - case <-ctx.Done(): - break loop - case err := <-errCh: - return err - case <-t.C: - logger.Info("still vacuuming...") - } - } - - logger.Info("finished PostgreSQL database vacuuming") - return nil -} - // NewDB connects to postgres DB in the specified path -func NewDB(dburl string, shouldVacuum bool, logger *zap.Logger) (*sql.DB, error) { +func NewDB(dburl string, logger *zap.Logger) (*sql.DB, error) { db, err := sql.Open("pgx", dburl) if err != nil { return nil, err } - if shouldVacuum { - err := executeVacuum(db, logger) - if err != nil { - return nil, err - } - } - return db, nil } diff --git a/waku/persistence/sqlite/sqlite.go b/waku/persistence/sqlite/sqlite.go index 9a718199..b902294b 100644 --- a/waku/persistence/sqlite/sqlite.go +++ b/waku/persistence/sqlite/sqlite.go @@ -1,11 +1,9 @@ package sqlite import ( - "context" "database/sql" "fmt" "strings" - "time" "github.com/golang-migrate/migrate/v4/database" "github.com/golang-migrate/migrate/v4/database/sqlite3" @@ -32,43 +30,8 @@ func addSqliteURLDefaults(dburl string) string { return dburl } -func executeVacuum(db *sql.DB, logger *zap.Logger) error { - logger.Info("starting sqlite database vacuuming") - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - errCh := make(chan error) - - go func() { - defer cancel() - _, err := db.Exec("VACUUM") - if err != nil { - errCh <- err - } - }() - - t := time.NewTicker(2 * time.Minute) - defer t.Stop() - -loop: - for { - select { - case <-ctx.Done(): - break loop - case err := <-errCh: - return err - case <-t.C: - logger.Info("still vacuuming...") - } - } - - logger.Info("finished sqlite database vacuuming") - return nil -} - // NewDB creates a sqlite3 DB in the specified path -func NewDB(dburl string, shouldVacuum bool, logger *zap.Logger) (*sql.DB, error) { +func NewDB(dburl string, logger *zap.Logger) (*sql.DB, error) { db, err := sql.Open("sqlite3", addSqliteURLDefaults(dburl)) if err != nil { return nil, err @@ -77,13 +40,6 @@ func NewDB(dburl string, shouldVacuum bool, logger *zap.Logger) (*sql.DB, error) // Disable concurrent access as not supported by the driver db.SetMaxOpenConns(1) - if shouldVacuum { - err := executeVacuum(db, logger) - if err != nil { - return nil, err - } - } - return db, nil } diff --git a/waku/persistence/utils/db.go b/waku/persistence/utils/db.go index ad7e3a95..72b0ac85 100644 --- a/waku/persistence/utils/db.go +++ b/waku/persistence/utils/db.go @@ -22,7 +22,7 @@ func validateDBUrl(val string) error { // DBSettings hold db specific configuration settings required during the db initialization type DBSettings struct { - Vacuum bool + // TODO: add any DB specific setting here } // ExtractDBAndMigration will return a database connection, and migration function that should be used depending on a database connection string @@ -50,10 +50,10 @@ func ExtractDBAndMigration(databaseURL string, dbSettings DBSettings, logger *za dbParams := dbURLParts[1] switch dbEngine { case "sqlite3": - db, err = sqlite.NewDB(dbParams, dbSettings.Vacuum, logger) + db, err = sqlite.NewDB(dbParams, logger) migrationFn = sqlite.Migrations case "postgresql": - db, err = postgres.NewDB(dbURL, dbSettings.Vacuum, logger) + db, err = postgres.NewDB(dbURL, logger) migrationFn = postgres.Migrations default: err = errors.New("unsupported database engine") diff --git a/waku/v2/node/connectedness_test.go b/waku/v2/node/connectedness_test.go index 192d4a2a..6e1aea23 100644 --- a/waku/v2/node/connectedness_test.go +++ b/waku/v2/node/connectedness_test.go @@ -70,7 +70,7 @@ func TestConnectionStatusChanges(t *testing.T) { err = node2.Start(ctx) require.NoError(t, err) - db, err := sqlite.NewDB(":memory:", false, utils.Logger()) + db, err := sqlite.NewDB(":memory:", utils.Logger()) require.NoError(t, err) dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations)) require.NoError(t, err) diff --git a/waku/v2/node/wakunode2_test.go b/waku/v2/node/wakunode2_test.go index 73bf43f0..d8c78b0c 100644 --- a/waku/v2/node/wakunode2_test.go +++ b/waku/v2/node/wakunode2_test.go @@ -242,7 +242,7 @@ func TestDecoupledStoreFromRelay(t *testing.T) { defer subs[0].Unsubscribe() // NODE2: Filter Client/Store - db, err := sqlite.NewDB(":memory:", false, utils.Logger()) + db, err := sqlite.NewDB(":memory:", utils.Logger()) require.NoError(t, err) dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations)) require.NoError(t, err) diff --git a/waku/v2/protocol/store/utils_test.go b/waku/v2/protocol/store/utils_test.go index 4824afec..6fd883c8 100644 --- a/waku/v2/protocol/store/utils_test.go +++ b/waku/v2/protocol/store/utils_test.go @@ -13,7 +13,7 @@ import ( func MemoryDB(t *testing.T) *persistence.DBStore { var db *sql.DB - db, err := sqlite.NewDB(":memory:", false, utils.Logger()) + db, err := sqlite.NewDB(":memory:", utils.Logger()) require.NoError(t, err) dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations)) diff --git a/waku/v2/rendezvous/rendezvous_test.go b/waku/v2/rendezvous/rendezvous_test.go index 3aeb6685..d354e3c7 100644 --- a/waku/v2/rendezvous/rendezvous_test.go +++ b/waku/v2/rendezvous/rendezvous_test.go @@ -45,7 +45,7 @@ func TestRendezvous(t *testing.T) { host1, err := tests.MakeHost(ctx, port1, rand.Reader) require.NoError(t, err) - db, err := sqlite.NewDB(":memory:", false, utils.Logger()) + db, err := sqlite.NewDB(":memory:", utils.Logger()) require.NoError(t, err) err = sqlite.Migrations(db)