From 5d0692b339853abbb57a3aa4874c6e5a63557272 Mon Sep 17 00:00:00 2001 From: harsh-98 Date: Tue, 3 Oct 2023 23:02:23 +0700 Subject: [PATCH] test(store): add fixture for sqlite and postgres --- waku/persistence/{ => postgres}/mock.go | 9 ++-- waku/persistence/postgres/postgres.go | 2 +- waku/persistence/sql_queries.go | 3 +- waku/persistence/sqlite/mock.go | 17 ++++++ waku/persistence/sqlite/sqlite.go | 2 +- waku/persistence/sqlite/sqlite_test.go | 15 +----- waku/persistence/store.go | 15 +++++- waku/persistence/{ => utils}/store_test.go | 60 +++++++++++++--------- 8 files changed, 75 insertions(+), 48 deletions(-) rename waku/persistence/{ => postgres}/mock.go (81%) create mode 100644 waku/persistence/sqlite/mock.go rename waku/persistence/{ => utils}/store_test.go (77%) diff --git a/waku/persistence/mock.go b/waku/persistence/postgres/mock.go similarity index 81% rename from waku/persistence/mock.go rename to waku/persistence/postgres/mock.go index 183c3a3c..7e6a8b21 100644 --- a/waku/persistence/mock.go +++ b/waku/persistence/postgres/mock.go @@ -1,19 +1,17 @@ -package persistence +package postgres import ( "database/sql" "fmt" "log" "os" - - _ "github.com/jackc/pgx/v5/stdlib" // Blank import to register the postgres driver ) // var dbUrlTemplate = "postgres://postgres@localhost:%s/%s?sslmode=disable" var dbUrlTemplate = "postgres://harshjain@localhost:%s/%s?sslmode=disable" func ResetDefaultTestPostgresDB(dropDBUrl string) error { - db, err := sql.Open("postgres", dropDBUrl) + db, err := sql.Open("pgx", dropDBUrl) if err != nil { return err } @@ -41,9 +39,8 @@ func NewMockPgDB() *sql.DB { // dropDBUrl := fmt.Sprintf(dbUrlTemplate, mockPgDBPort, "template1") - fmt.Println(dropDBUrl) if err := ResetDefaultTestPostgresDB(dropDBUrl); err != nil { - log.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + log.Fatalf("an error '%s' while reseting the db", err) } mockDBUrl := fmt.Sprintf(dbUrlTemplate, mockPgDBPort, "postgres") db, err := sql.Open("pgx", mockDBUrl) diff --git a/waku/persistence/postgres/postgres.go b/waku/persistence/postgres/postgres.go index bda79b54..cfc372fe 100644 --- a/waku/persistence/postgres/postgres.go +++ b/waku/persistence/postgres/postgres.go @@ -71,5 +71,5 @@ func NewQueries(tbl string, db *sql.DB) (*persistence.Queries, error) { if err != nil { return nil, err } - return persistence.CreateQueries(tbl, db), nil + return persistence.CreateQueries(tbl), nil } diff --git a/waku/persistence/sql_queries.go b/waku/persistence/sql_queries.go index 2329187e..730f0c95 100644 --- a/waku/persistence/sql_queries.go +++ b/waku/persistence/sql_queries.go @@ -1,7 +1,6 @@ package persistence import ( - "database/sql" "fmt" ) @@ -20,7 +19,7 @@ type Queries struct { // CreateQueries Function creates a set of queries for an SQL table. // Note: Do not use this function to create queries for a table, rather use .NewQueries to create table as well as queries. -func CreateQueries(tbl string, db *sql.DB) *Queries { +func CreateQueries(tbl string) *Queries { return &Queries{ deleteQuery: fmt.Sprintf("DELETE FROM %s WHERE key = $1", tbl), existsQuery: fmt.Sprintf("SELECT exists(SELECT 1 FROM %s WHERE key=$1)", tbl), diff --git a/waku/persistence/sqlite/mock.go b/waku/persistence/sqlite/mock.go new file mode 100644 index 00000000..be17cfac --- /dev/null +++ b/waku/persistence/sqlite/mock.go @@ -0,0 +1,17 @@ +package sqlite + +import ( + "database/sql" + "log" + + _ "github.com/mattn/go-sqlite3" +) + +func NewMockSqliteDB() *sql.DB { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + log.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + + return db +} diff --git a/waku/persistence/sqlite/sqlite.go b/waku/persistence/sqlite/sqlite.go index c7b845c4..75181608 100644 --- a/waku/persistence/sqlite/sqlite.go +++ b/waku/persistence/sqlite/sqlite.go @@ -91,5 +91,5 @@ func NewQueries(tbl string, db *sql.DB) (*persistence.Queries, error) { if err != nil { return nil, err } - return persistence.CreateQueries(tbl, db), nil + return persistence.CreateQueries(tbl), nil } diff --git a/waku/persistence/sqlite/sqlite_test.go b/waku/persistence/sqlite/sqlite_test.go index 025d1b43..8692f329 100644 --- a/waku/persistence/sqlite/sqlite_test.go +++ b/waku/persistence/sqlite/sqlite_test.go @@ -1,24 +1,13 @@ package sqlite import ( - "database/sql" - "log" "testing" "github.com/stretchr/testify/require" ) -func NewMock() *sql.DB { - db, err := sql.Open("sqlite3", ":memory:") - if err != nil { - log.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - - return db -} - func TestQueries(t *testing.T) { - db := NewMock() + db := NewMockSqliteDB() queries, err := NewQueries("test_queries", db) require.NoError(t, err) @@ -51,7 +40,7 @@ func TestQueries(t *testing.T) { } func TestCreateTable(t *testing.T) { - db := NewMock() + db := NewMockSqliteDB() err := CreateTable(db, "test_create_table") require.NoError(t, err) diff --git a/waku/persistence/store.go b/waku/persistence/store.go index cb7a0ab6..5964db8b 100644 --- a/waku/persistence/store.go +++ b/waku/persistence/store.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "reflect" "strings" "sync" "time" @@ -224,8 +225,8 @@ func (d *DBStore) cleanOlderRecords(ctx context.Context) error { // Limit number of records to a max N if d.maxMessages > 0 { start := time.Now() - sqlStmt := `DELETE FROM message WHERE id IN (SELECT id FROM message ORDER BY receiverTimestamp DESC OFFSET $1)` - _, err := d.db.Exec(sqlStmt, d.maxMessages) + + _, err := d.db.Exec(d.getDeleteOldRowsQuery(), d.maxMessages) if err != nil { d.metrics.RecordError(retPolicyFailure) return err @@ -238,6 +239,16 @@ func (d *DBStore) cleanOlderRecords(ctx context.Context) error { return nil } +func (d *DBStore) getDeleteOldRowsQuery() string { + sqlStmt := `DELETE FROM message WHERE id IN (SELECT id FROM message ORDER BY receiverTimestamp DESC %s OFFSET $1)` + switch reflect.TypeOf(d.db.Driver()).String() { + case "*sqlite3.SQLiteDriver": + sqlStmt = fmt.Sprintf(sqlStmt, "LIMIT -1") + case "*stdlib.Driver": + sqlStmt = fmt.Sprintf(sqlStmt, "") + } + return sqlStmt +} func (d *DBStore) checkForOlderRecords(ctx context.Context, t time.Duration) { defer d.wg.Done() diff --git a/waku/persistence/store_test.go b/waku/persistence/utils/store_test.go similarity index 77% rename from waku/persistence/store_test.go rename to waku/persistence/utils/store_test.go index 249d9772..6b7b8bba 100644 --- a/waku/persistence/store_test.go +++ b/waku/persistence/utils/store_test.go @@ -1,7 +1,7 @@ //go:build include_postgres_tests // +build include_postgres_tests -package persistence +package utils import ( "context" @@ -10,32 +10,49 @@ import ( "testing" "time" - "github.com/golang-migrate/migrate/v4/database/postgres" - _ "github.com/mattn/go-sqlite3" // Blank import to register the sqlite3 driver "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" "github.com/waku-org/go-waku/tests" - "github.com/waku-org/go-waku/waku/persistence/migrate" - postgresmigration "github.com/waku-org/go-waku/waku/persistence/postgres/migrations" + "github.com/waku-org/go-waku/waku/persistence" + "github.com/waku-org/go-waku/waku/persistence/postgres" + "github.com/waku-org/go-waku/waku/persistence/sqlite" "github.com/waku-org/go-waku/waku/v2/protocol" "github.com/waku-org/go-waku/waku/v2/protocol/store/pb" "github.com/waku-org/go-waku/waku/v2/timesource" "github.com/waku-org/go-waku/waku/v2/utils" ) -func Migrate(db *sql.DB) error { - migrationDriver, err := postgres.WithInstance(db, &postgres.Config{ - MigrationsTable: "gowaku_" + postgres.DefaultMigrationsTable, - }) - if err != nil { - return err +func TestStore(t *testing.T) { + tests := []struct { + name string + fn func(t *testing.T, db *sql.DB, migrationFn func(*sql.DB) error) + }{ + {"testDbStore", testDbStore}, + {"testStoreRetention", testStoreRetention}, + {"testQuery", testQuery}, + } + for _, driverName := range []string{"postgres", "sqlite"} { + // all tests are run for each db + for _, tc := range tests { + db, migrationFn := getDB(driverName) + t.Run(driverName+"_"+tc.name, func(t *testing.T) { + tc.fn(t, db, migrationFn) + }) + } } - return migrate.Migrate(db, migrationDriver, postgresmigration.AssetNames(), postgresmigration.Asset) } -func TestDbStore(t *testing.T) { - db := NewMockPgDB() - store, err := NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), WithDB(db), WithMigrations(Migrate)) +func getDB(driver string) (*sql.DB, func(*sql.DB) error) { + switch driver { + case "postgres": + return postgres.NewMockPgDB(), postgres.Migrations + case "sqlite": + return sqlite.NewMockSqliteDB(), sqlite.Migrations + } + return nil, nil +} +func testDbStore(t *testing.T, db *sql.DB, migrationFn func(*sql.DB) error) { + store, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(migrationFn)) require.NoError(t, err) err = store.Start(context.Background(), timesource.NewDefaultClock()) @@ -53,9 +70,8 @@ func TestDbStore(t *testing.T) { require.NotEmpty(t, res) } -func TestStoreRetention(t *testing.T) { - db := NewMockPgDB() - store, err := NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), WithDB(db), WithMigrations(Migrate), WithRetentionPolicy(5, 20*time.Second)) +func testStoreRetention(t *testing.T, db *sql.DB, migrationFn func(*sql.DB) error) { + store, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(migrationFn), persistence.WithRetentionPolicy(5, 20*time.Second)) require.NoError(t, err) err = store.Start(context.Background(), timesource.NewDefaultClock()) @@ -78,7 +94,7 @@ func TestStoreRetention(t *testing.T) { // This step simulates starting go-waku again from scratch - store, err = NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), WithDB(db), WithRetentionPolicy(5, 40*time.Second)) + store, err = persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithRetentionPolicy(5, 40*time.Second)) require.NoError(t, err) err = store.Start(context.Background(), timesource.NewDefaultClock()) @@ -96,10 +112,8 @@ func TestStoreRetention(t *testing.T) { require.Equal(t, msgCount, 3) } -func TestQuery(t *testing.T) { - db := NewMockPgDB() - - store, err := NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), WithDB(db), WithMigrations(Migrate), WithRetentionPolicy(5, 20*time.Second)) +func testQuery(t *testing.T, db *sql.DB, migrationFn func(*sql.DB) error) { + store, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(migrationFn), persistence.WithRetentionPolicy(5, 20*time.Second)) require.NoError(t, err) insertTime := time.Now()