From 4f9422ec855f3add8c07d638c6fbe6d60ce65011 Mon Sep 17 00:00:00 2001 From: Taylor Wrobel Date: Sat, 24 Jun 2017 10:36:59 -0700 Subject: [PATCH 01/11] Update docker dependency version to fix Tavis builds Tavis has updated its docker images, and as part of the update, no longer includes support for the (deprevated) docker-engine package, moving instead to docker-ce. See https://blog.travis-ci.com/2017-06-21-trusty-updates-2017-Q2-launch for details of the package change. This updates the travis configuration to pin the docker library version to 17.05.0 and installs the corresponding version of docker-ce in the install script. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6581f68..50dbf36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,8 @@ services: install: - make deps - - (cd $GOPATH/src/github.com/docker/docker && git fetch --all --tags --prune && git checkout v1.13.0) - - sudo apt-get update && sudo apt-get install docker-engine=1.13.0* + - (cd $GOPATH/src/github.com/docker/docker && git fetch --all --tags --prune && git checkout v17.05.0-ce) + - sudo apt-get update && sudo apt-get install docker-ce=17.05.0* - go get github.com/mattn/goveralls script: From 2361a7aa2429f4184e9313e806bf5debf5fb5065 Mon Sep 17 00:00:00 2001 From: Martin Magakian Date: Mon, 3 Jul 2017 15:29:15 +0200 Subject: [PATCH 02/11] Update README.md Code snippet don't compile. It's missing a step (see https://github.com/mattes/migrate/blob/master/source/go-bindata/go-bindata_test.go) --- source/go-bindata/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/go-bindata/README.md b/source/go-bindata/README.md index e034564..e74a0aa 100644 --- a/source/go-bindata/README.md +++ b/source/go-bindata/README.md @@ -24,8 +24,13 @@ func main() { func(name string) ([]byte, error) { return migrations.Asset(name) }) + + d, err := WithInstance(s) + if err != nil { + t.Fatal(err) + } - m, err := migrate.NewWithSourceInstance("go-bindata", s, "database://foobar") + m, err := migrate.NewWithSourceInstance("go-bindata", d, "database://foobar") m.Up() // run your migrations and handle the errors above of course } ``` From 180c57747dc8bc52f7171ac6fcafefd3225d5ed6 Mon Sep 17 00:00:00 2001 From: Martin Magakian Date: Mon, 3 Jul 2017 15:34:43 +0200 Subject: [PATCH 03/11] Update README.md Forget package --- source/go-bindata/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/go-bindata/README.md b/source/go-bindata/README.md index e74a0aa..c7be7b0 100644 --- a/source/go-bindata/README.md +++ b/source/go-bindata/README.md @@ -25,7 +25,7 @@ func main() { return migrations.Asset(name) }) - d, err := WithInstance(s) + d, err := bindata.WithInstance(s) if err != nil { t.Fatal(err) } From 1485e567f1e26e298d9c1adb95b20ea9b09c7ad2 Mon Sep 17 00:00:00 2001 From: Martin Magakian Date: Mon, 3 Jul 2017 15:35:43 +0200 Subject: [PATCH 04/11] Update README.md Some format :-p --- source/go-bindata/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/go-bindata/README.md b/source/go-bindata/README.md index c7be7b0..1ffab20 100644 --- a/source/go-bindata/README.md +++ b/source/go-bindata/README.md @@ -25,10 +25,10 @@ func main() { return migrations.Asset(name) }) - d, err := bindata.WithInstance(s) - if err != nil { - t.Fatal(err) - } + d, err := bindata.WithInstance(s) + if err != nil { + t.Fatal(err) + } m, err := migrate.NewWithSourceInstance("go-bindata", d, "database://foobar") m.Up() // run your migrations and handle the errors above of course From 56a56527c66b756cda11b6796f10df1c797017cf Mon Sep 17 00:00:00 2001 From: Martin Magakian Date: Mon, 3 Jul 2017 15:45:41 +0200 Subject: [PATCH 05/11] Update README.md Late edit I promise... --- source/go-bindata/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/source/go-bindata/README.md b/source/go-bindata/README.md index 1ffab20..879face 100644 --- a/source/go-bindata/README.md +++ b/source/go-bindata/README.md @@ -25,11 +25,7 @@ func main() { return migrations.Asset(name) }) - d, err := bindata.WithInstance(s) - if err != nil { - t.Fatal(err) - } - + d, err := bindata.WithInstance(s) m, err := migrate.NewWithSourceInstance("go-bindata", d, "database://foobar") m.Up() // run your migrations and handle the errors above of course } From 35af42a0dd19e2ba23db18bcfbe09ad08f9b5489 Mon Sep 17 00:00:00 2001 From: Martin Magakian Date: Mon, 3 Jul 2017 15:45:57 +0200 Subject: [PATCH 06/11] Update README.md --- source/go-bindata/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/go-bindata/README.md b/source/go-bindata/README.md index 879face..cd9dd4b 100644 --- a/source/go-bindata/README.md +++ b/source/go-bindata/README.md @@ -25,7 +25,7 @@ func main() { return migrations.Asset(name) }) - d, err := bindata.WithInstance(s) + d, err := bindata.WithInstance(s) m, err := migrate.NewWithSourceInstance("go-bindata", d, "database://foobar") m.Up() // run your migrations and handle the errors above of course } From 1d8a881941a734a90362addb54d8d38cda6ff9a8 Mon Sep 17 00:00:00 2001 From: Taylor Wrobel Date: Sun, 18 Jun 2017 18:53:52 -0700 Subject: [PATCH 07/11] Add CockroachDB Support Adds support for CockroachDB. Cockroach uses the postges wire protocol and has a large amount of common SQL functionality shared with Postgres, so much of the postgres code was able to be copied and modified. Since the protocol is used in determining the driver, and the Postgres protocol is also used by Cockroach, new connect string prefixes were added: cockroach:// cockroachdb:// and crdb-postgres://. These fake protocol strings are replaced in the connect function with the correct `postgres://` protocol. TODO: Tests needed (Cockroach has a docker image, so this shouldn't be too hard) --- Makefile | 2 +- README.md | 1 + cli/build_cockroachdb.go | 7 + database/cockroachdb/cockroachdb.go | 319 ++++++++++++++++++ .../1085649617_create_users_table.down.sql | 1 + .../1085649617_create_users_table.up.sql | 5 + .../1185749658_add_city_to_users.down.sql | 1 + .../1185749658_add_city_to_users.up.sql | 1 + ...85849751_add_index_on_user_emails.down.sql | 1 + ...1285849751_add_index_on_user_emails.up.sql | 3 + .../1385949617_create_books_table.down.sql | 1 + .../1385949617_create_books_table.up.sql | 5 + .../1485949617_create_movies_table.down.sql | 1 + .../1485949617_create_movies_table.up.sql | 5 + .../1585849751_just_a_comment.up.sql | 1 + .../1685849751_another_comment.up.sql | 1 + .../1785849751_another_comment.up.sql | 1 + .../1885849751_another_comment.up.sql | 1 + 18 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 cli/build_cockroachdb.go create mode 100644 database/cockroachdb/cockroachdb.go create mode 100644 database/cockroachdb/examples/migrations/1085649617_create_users_table.down.sql create mode 100644 database/cockroachdb/examples/migrations/1085649617_create_users_table.up.sql create mode 100644 database/cockroachdb/examples/migrations/1185749658_add_city_to_users.down.sql create mode 100644 database/cockroachdb/examples/migrations/1185749658_add_city_to_users.up.sql create mode 100644 database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.down.sql create mode 100644 database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.up.sql create mode 100644 database/cockroachdb/examples/migrations/1385949617_create_books_table.down.sql create mode 100644 database/cockroachdb/examples/migrations/1385949617_create_books_table.up.sql create mode 100644 database/cockroachdb/examples/migrations/1485949617_create_movies_table.down.sql create mode 100644 database/cockroachdb/examples/migrations/1485949617_create_movies_table.up.sql create mode 100644 database/cockroachdb/examples/migrations/1585849751_just_a_comment.up.sql create mode 100644 database/cockroachdb/examples/migrations/1685849751_another_comment.up.sql create mode 100644 database/cockroachdb/examples/migrations/1785849751_another_comment.up.sql create mode 100644 database/cockroachdb/examples/migrations/1885849751_another_comment.up.sql diff --git a/Makefile b/Makefile index f804816..8fb5ca7 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go-bindata github aws-s3 google-cloud-storage -DATABASE ?= postgres mysql redshift cassandra sqlite3 spanner +DATABASE ?= postgres mysql redshift cassandra sqlite3 spanner cockroachdb VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/README.md b/README.md index f8c8993..371fa8e 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170)) * [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171)) * [Google Cloud Spanner](database/spanner) + * [CockroachDB](database/cockroachdb) diff --git a/cli/build_cockroachdb.go b/cli/build_cockroachdb.go new file mode 100644 index 0000000..e5fdf07 --- /dev/null +++ b/cli/build_cockroachdb.go @@ -0,0 +1,7 @@ +// +build cockroachdb + +package main + +import ( + _ "github.com/mattes/migrate/database/cockroachdb" +) diff --git a/database/cockroachdb/cockroachdb.go b/database/cockroachdb/cockroachdb.go new file mode 100644 index 0000000..9d11620 --- /dev/null +++ b/database/cockroachdb/cockroachdb.go @@ -0,0 +1,319 @@ +package cockroachdb + +import ( + "database/sql" + "fmt" + "io" + "io/ioutil" + nurl "net/url" + + "github.com/cockroachdb/cockroach-go/crdb" + "github.com/lib/pq" + "github.com/mattes/migrate" + "github.com/mattes/migrate/database" + "regexp" +) + +func init() { + db := CockroachDb{} + database.Register("cockroach", &db) + database.Register("cockroachdb", &db) + database.Register("crdb-postgres", &db) +} + +var DefaultMigrationsTable = "schema_migrations" +var DefaultLockTable = "schema_lock" + +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + LockTable string + DatabaseName string +} + +type CockroachDb struct { + db *sql.DB + isLocked bool + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + query := `SELECT current_database()` + var databaseName string + if err := instance.QueryRow(query).Scan(&databaseName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(databaseName) == 0 { + return nil, ErrNoDatabaseName + } + + config.DatabaseName = databaseName + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + if len(config.LockTable) == 0 { + config.LockTable = DefaultLockTable + } + + px := &CockroachDb{ + db: instance, + config: config, + } + + if err := px.ensureVersionTable(); err != nil { + return nil, err + } + + if err := px.ensureLockTable(); err != nil { + return nil, err + } + + return px, nil +} + +func (c *CockroachDb) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + // As Cockroach uses the postgres protocol, and 'postgres' is already a registered database, we need to replace the + // connect prefix, with the actual protocol, so that the library can differentiate between the implementations + re := regexp.MustCompile("^(cockroach(db)?|crdb-postgres)") + connectString := re.ReplaceAllString(migrate.FilterCustomQuery(purl).String(), "postgres") + + db, err := sql.Open("postgres", connectString) + if err != nil { + return nil, err + } + + migrationsTable := purl.Query().Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + + lockTable := purl.Query().Get("x-lock-table") + if len(lockTable) == 0 { + lockTable = DefaultLockTable + } + + px, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + LockTable: lockTable, + }) + if err != nil { + return nil, err + } + + return px, nil +} + +func (c *CockroachDb) Close() error { + return c.db.Close() +} + +// Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed +// See: https://github.com/cockroachdb/cockroach/issues/13546 +func (c *CockroachDb) Lock() error { + err := crdb.ExecuteTx(c.db, func(tx *sql.Tx) error { + aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName) + if err != nil { + return err + } + + query := "SELECT * FROM " + c.config.LockTable + " WHERE lock_id = $1" + rows, err := tx.Query(query, aid) + if err != nil { + return database.Error{OrigErr: err, Err: "failed to fetch migration lock", Query: []byte(query)} + } + defer rows.Close() + + // If row exists at all, lock is present + locked := rows.Next() + if locked { + return database.Error{Err: "lock could not be acquired; already locked", Query: []byte(query)} + } + + query = "INSERT INTO " + c.config.LockTable + " (lock_id) VALUES ($1)" + if _, err := tx.Exec(query, aid) ; err != nil { + return database.Error{OrigErr: err, Err: "failed to set migration lock", Query: []byte(query)} + } + + return nil + }) + + if err != nil { + return err + } else { + c.isLocked = true + return nil + } +} + +// Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed +// See: https://github.com/cockroachdb/cockroach/issues/13546 +func (c *CockroachDb) Unlock() error { + aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName) + if err != nil { + return err + } + + // In the event of an implementation (non-migration) error, it is possible for the lock to not be released. Until + // a better locking mechanism is added, a manual purging of the lock table may be required in such circumstances + query := "DELETE FROM " + c.config.LockTable + " WHERE lock_id = $1" + if _, err := c.db.Exec(query, aid); err != nil { + return database.Error{OrigErr: err, Err: "failed to release migration lock", Query: []byte(query)} + } + + c.isLocked = false + return nil +} + +func (c *CockroachDb) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + + // run migration + query := string(migr[:]) + if _, err := c.db.Exec(query); err != nil { + return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + } + + return nil +} + +func (c *CockroachDb) SetVersion(version int, dirty bool) error { + return crdb.ExecuteTx(c.db, func(tx *sql.Tx) error { + if _, err := tx.Exec( `TRUNCATE "` + c.config.MigrationsTable + `"`); err != nil { + return err + } + + if version >= 0 { + if _, err := tx.Exec(`INSERT INTO "` + c.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)`, version, dirty); err != nil { + return err + } + } + + return nil + }) +} + +func (c *CockroachDb) Version() (version int, dirty bool, err error) { + query := `SELECT version, dirty FROM "` + c.config.MigrationsTable + `" LIMIT 1` + err = c.db.QueryRow(query).Scan(&version, &dirty) + + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + + case err != nil: + if e, ok := err.(*pq.Error); ok { + // 42P01 is "UndefinedTableError" in CockroachDB + // https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go + if e.Code == "42P01" { + return database.NilVersion, false, nil + } + } + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + + default: + return version, dirty, nil + } +} + +func (c *CockroachDb) Drop() error { + // select all tables in current schema + query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema())` + tables, err := c.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer tables.Close() + + // delete one table after another + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + + if len(tableNames) > 0 { + // delete one by one ... + for _, t := range tableNames { + query = `DROP TABLE IF EXISTS ` + t + ` CASCADE` + if _, err := c.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + if err := c.ensureVersionTable(); err != nil { + return err + } + } + + return nil +} + +func (c *CockroachDb) ensureVersionTable() error { + // check if migration table exists + var count int + query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1` + if err := c.db.QueryRow(query, c.config.MigrationsTable).Scan(&count); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if count == 1 { + return nil + } + + // if not, create the empty migration table + query = `CREATE TABLE "` + c.config.MigrationsTable + `" (version INT NOT NULL PRIMARY KEY, dirty BOOL NOT NULL)` + if _, err := c.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil +} + + +func (c *CockroachDb) ensureLockTable() error { + // check if lock table exists + var count int + query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1` + if err := c.db.QueryRow(query, c.config.LockTable).Scan(&count); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if count == 1 { + return nil + } + + // if not, create the empty lock table + query = `CREATE TABLE "` + c.config.LockTable + `" (lock_id INT NOT NULL PRIMARY KEY)` + if _, err := c.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} diff --git a/database/cockroachdb/examples/migrations/1085649617_create_users_table.down.sql b/database/cockroachdb/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 0000000..c99ddcd --- /dev/null +++ b/database/cockroachdb/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/database/cockroachdb/examples/migrations/1085649617_create_users_table.up.sql b/database/cockroachdb/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 0000000..fc32101 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE users ( + user_id INT UNIQUE, + name STRING(40), + email STRING(40) +); diff --git a/database/cockroachdb/examples/migrations/1185749658_add_city_to_users.down.sql b/database/cockroachdb/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 0000000..940c607 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE users DROP COLUMN IF EXISTS city; diff --git a/database/cockroachdb/examples/migrations/1185749658_add_city_to_users.up.sql b/database/cockroachdb/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 0000000..46204b0 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1 @@ +ALTER TABLE users ADD COLUMN city TEXT; diff --git a/database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 0000000..3e87dd2 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS users_email_index; diff --git a/database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 0000000..61f8ba0 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1,3 @@ +CREATE UNIQUE INDEX IF NOT EXISTS users_email_index ON users (email); + +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/cockroachdb/examples/migrations/1385949617_create_books_table.down.sql b/database/cockroachdb/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 0000000..1a0b1a2 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS books; diff --git a/database/cockroachdb/examples/migrations/1385949617_create_books_table.up.sql b/database/cockroachdb/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 0000000..0d3b999 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE books ( + user_id INT, + name STRING(40), + author STRING(40) +); diff --git a/database/cockroachdb/examples/migrations/1485949617_create_movies_table.down.sql b/database/cockroachdb/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 0000000..3a51876 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS movies; diff --git a/database/cockroachdb/examples/migrations/1485949617_create_movies_table.up.sql b/database/cockroachdb/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 0000000..d533be9 --- /dev/null +++ b/database/cockroachdb/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE movies ( + user_id INT, + name STRING(40), + director STRING(40) +); diff --git a/database/cockroachdb/examples/migrations/1585849751_just_a_comment.up.sql b/database/cockroachdb/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 0000000..9b6b57a --- /dev/null +++ b/database/cockroachdb/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/cockroachdb/examples/migrations/1685849751_another_comment.up.sql b/database/cockroachdb/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 0000000..9b6b57a --- /dev/null +++ b/database/cockroachdb/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/cockroachdb/examples/migrations/1785849751_another_comment.up.sql b/database/cockroachdb/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 0000000..9b6b57a --- /dev/null +++ b/database/cockroachdb/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/cockroachdb/examples/migrations/1885849751_another_comment.up.sql b/database/cockroachdb/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 0000000..9b6b57a --- /dev/null +++ b/database/cockroachdb/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. From 88115dedbfb969327fdc981f577d34c4d0afbe3c Mon Sep 17 00:00:00 2001 From: Taylor Wrobel Date: Sat, 24 Jun 2017 00:32:09 -0700 Subject: [PATCH 08/11] Support running docker with commands and handling multiple exposed ports Adds the ability to specify a series of commands to run as part of the docker image execution, and allows for retrieving a mapping of an exposed via the port bound within the container. --- database/cassandra/cassandra_test.go | 6 ++-- database/mysql/mysql_test.go | 8 +++--- testing/docker.go | 43 +++++++++++++++++++++++++++- testing/testing.go | 4 ++- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/database/cassandra/cassandra_test.go b/database/cassandra/cassandra_test.go index 4e9150f..4ca764a 100644 --- a/database/cassandra/cassandra_test.go +++ b/database/cassandra/cassandra_test.go @@ -11,8 +11,8 @@ import ( ) var versions = []mt.Version{ - {"cassandra:3.0.10", []string{}}, - {"cassandra:3.0", []string{}}, + {Image: "cassandra:3.0.10"}, + {Image: "cassandra:3.0"}, } func isReady(i mt.Instance) bool { @@ -50,4 +50,4 @@ func Test(t *testing.T) { } dt.Test(t, d, []byte("SELECT table_name from system_schema.tables")) }) -} \ No newline at end of file +} diff --git a/database/mysql/mysql_test.go b/database/mysql/mysql_test.go index 3eb22ef..f2b12e8 100644 --- a/database/mysql/mysql_test.go +++ b/database/mysql/mysql_test.go @@ -14,10 +14,10 @@ import ( ) var versions = []mt.Version{ - {"mysql:8", []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, - {"mysql:5.7", []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, - {"mysql:5.6", []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, - {"mysql:5.5", []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, + {Image: "mysql:8", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, + {Image: "mysql:5.7", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, + {Image: "mysql:5.6", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, + {Image: "mysql:5.5", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, } func isReady(i mt.Instance) bool { diff --git a/testing/docker.go b/testing/docker.go index de93bbf..345d2dc 100644 --- a/testing/docker.go +++ b/testing/docker.go @@ -18,17 +18,22 @@ import ( dockerclient "github.com/docker/docker/client" ) -func NewDockerContainer(t testing.TB, image string, env []string) (*DockerContainer, error) { +func NewDockerContainer(t testing.TB, image string, env []string, cmd []string) (*DockerContainer, error) { c, err := dockerclient.NewEnvClient() if err != nil { return nil, err } + if cmd == nil { + cmd = make([]string, 0) + } + contr := &DockerContainer{ t: t, client: c, ImageName: image, ENV: env, + Cmd: cmd, } if err := contr.PullImage(); err != nil { @@ -48,6 +53,7 @@ type DockerContainer struct { client *dockerclient.Client ImageName string ENV []string + Cmd []string ContainerId string ContainerName string ContainerJSON dockertypes.ContainerJSON @@ -86,6 +92,7 @@ func (d *DockerContainer) Start() error { Image: d.ImageName, Labels: map[string]string{"migrate_test": "true"}, Env: d.ENV, + Cmd: d.Cmd, }, &dockercontainer.HostConfig{ PublishAllPorts: true, @@ -159,6 +166,32 @@ func (d *DockerContainer) Logs() (io.ReadCloser, error) { }) } +func (d *DockerContainer) mappingForPort(cPort int) (containerPort uint, hostIP string, hostPort uint, err error) { + if !d.containerInspected { + if err := d.Inspect(); err != nil { + d.t.Fatal(err) + } + } + + for port, bindings := range d.ContainerJSON.NetworkSettings.Ports { + if port.Int() != cPort { + // Skip ahead until we find the port we want + continue + } + for _, binding := range bindings { + + hostPortUint, err := strconv.ParseUint(binding.HostPort, 10, 64) + if err != nil { + return 0, "", 0, err + } + + return uint(port.Int()), binding.HostIP, uint(hostPortUint), nil + } + } + + return 0, "", 0, fmt.Errorf("specified port not bound") +} + func (d *DockerContainer) firstPortMapping() (containerPort uint, hostIP string, hostPort uint, err error) { if !d.containerInspected { if err := d.Inspect(); err != nil { @@ -201,6 +234,14 @@ func (d *DockerContainer) Port() uint { return port } +func (d *DockerContainer) PortFor(cPort int) uint { + _, _, port, err := d.mappingForPort(cPort) + if err != nil { + d.t.Fatal(err) + } + return port +} + func (d *DockerContainer) NetworkSettings() dockertypes.NetworkSettings { netSettings := d.ContainerJSON.NetworkSettings return *netSettings diff --git a/testing/testing.go b/testing/testing.go index 0d03432..64e0a64 100644 --- a/testing/testing.go +++ b/testing/testing.go @@ -17,6 +17,7 @@ type TestFunc func(*testing.T, Instance) type Version struct { Image string ENV []string + Cmd []string } func ParallelTest(t *testing.T, versions []Version, readyFn IsReadyFunc, testFn TestFunc) { @@ -38,7 +39,7 @@ func ParallelTest(t *testing.T, versions []Version, readyFn IsReadyFunc, testFn t.Parallel() // create new container - container, err := NewDockerContainer(t, version.Image, version.ENV) + container, err := NewDockerContainer(t, version.Image, version.ENV, version.Cmd) if err != nil { t.Fatalf("%v\n%s", err, containerLogs(t, container)) } @@ -89,6 +90,7 @@ func containerLogs(t *testing.T, c *DockerContainer) []byte { type Instance interface { Host() string Port() uint + PortFor(int) uint NetworkSettings() dockertypes.NetworkSettings KeepForDebugging() } From c2925c40c74027bc15f8a57553d8fd3c15e59db4 Mon Sep 17 00:00:00 2001 From: Taylor Wrobel Date: Sat, 24 Jun 2017 00:34:16 -0700 Subject: [PATCH 09/11] Add tests for CockroachDB --- database/cockroachdb/cockroachdb_test.go | 93 ++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 database/cockroachdb/cockroachdb_test.go diff --git a/database/cockroachdb/cockroachdb_test.go b/database/cockroachdb/cockroachdb_test.go new file mode 100644 index 0000000..5d7fb68 --- /dev/null +++ b/database/cockroachdb/cockroachdb_test.go @@ -0,0 +1,93 @@ +package cockroachdb + +// error codes https://github.com/lib/pq/blob/master/error.go + +import ( + //"bytes" + "database/sql" + "fmt" + "io" + "testing" + + "github.com/lib/pq" + dt "github.com/mattes/migrate/database/testing" + mt "github.com/mattes/migrate/testing" + "bytes" +) + +var versions = []mt.Version{ + {Image: "cockroachdb/cockroach:v1.0.2", Cmd: []string{"start", "--insecure"}}, +} + +func isReady(i mt.Instance) bool { + fmt.Println("Checking readiness") + db, err := sql.Open("postgres", fmt.Sprintf("postgres://root@%v:%v?sslmode=disable", i.Host(), i.PortFor(26257))) + if err != nil { + return false + } + defer db.Close() + err = db.Ping() + if err == io.EOF { + _, err = db.Exec("CREATE DATABASE migrate") + return err == nil; + } else if e, ok := err.(*pq.Error); ok { + if e.Code.Name() == "cannot_connect_now" { + return false + } + } + + _, err = db.Exec("CREATE DATABASE migrate") + return err == nil; + + return true +} + +func Test(t *testing.T) { + mt.ParallelTest(t, versions, isReady, + func(t *testing.T, i mt.Instance) { + c := &CockroachDb{} + fmt.Println("Connecting in test") + addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", i.Host(), i.PortFor(26257)) + d, err := c.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + dt.Test(t, d, []byte("SELECT 1")) + }) +} + +func TestMultiStatement(t *testing.T) { + mt.ParallelTest(t, versions, isReady, + func(t *testing.T, i mt.Instance) { + c := &CockroachDb{} + addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", i.Host(), i.Port()) + d, err := c.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"))); err != nil { + t.Fatalf("expected err to be nil, got %v", err) + } + + // make sure second table exists + var exists bool + if err := d.(*CockroachDb).db.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected table bar to exist") + } + }) +} + +func TestFilterCustomQuery(t *testing.T) { + mt.ParallelTest(t, versions, isReady, + func(t *testing.T, i mt.Instance) { + c := &CockroachDb{} + addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable&x-custom=foobar", i.Host(), i.PortFor(26257)) + _, err := c.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + }) +} From aebc0c3cdda209569d2dcb61450f0f0b1b77c363 Mon Sep 17 00:00:00 2001 From: Taylor Wrobel Date: Sat, 24 Jun 2017 09:53:02 -0700 Subject: [PATCH 10/11] Merge port mapping helper functions into one --- testing/docker.go | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/testing/docker.go b/testing/docker.go index 345d2dc..f7a7c41 100644 --- a/testing/docker.go +++ b/testing/docker.go @@ -166,7 +166,7 @@ func (d *DockerContainer) Logs() (io.ReadCloser, error) { }) } -func (d *DockerContainer) mappingForPort(cPort int) (containerPort uint, hostIP string, hostPort uint, err error) { +func (d *DockerContainer) portMapping(selectFirst bool, cPort int) (containerPort uint, hostIP string, hostPort uint, err error) { if !d.containerInspected { if err := d.Inspect(); err != nil { d.t.Fatal(err) @@ -174,7 +174,7 @@ func (d *DockerContainer) mappingForPort(cPort int) (containerPort uint, hostIP } for port, bindings := range d.ContainerJSON.NetworkSettings.Ports { - if port.Int() != cPort { + if !selectFirst && port.Int() != cPort { // Skip ahead until we find the port we want continue } @@ -189,32 +189,15 @@ func (d *DockerContainer) mappingForPort(cPort int) (containerPort uint, hostIP } } - return 0, "", 0, fmt.Errorf("specified port not bound") -} - -func (d *DockerContainer) firstPortMapping() (containerPort uint, hostIP string, hostPort uint, err error) { - if !d.containerInspected { - if err := d.Inspect(); err != nil { - d.t.Fatal(err) - } + if selectFirst { + return 0, "", 0, fmt.Errorf("no port binding") + } else { + return 0, "", 0, fmt.Errorf("specified port not bound") } - - for port, bindings := range d.ContainerJSON.NetworkSettings.Ports { - for _, binding := range bindings { - - hostPortUint, err := strconv.ParseUint(binding.HostPort, 10, 64) - if err != nil { - return 0, "", 0, err - } - - return uint(port.Int()), binding.HostIP, uint(hostPortUint), nil - } - } - return 0, "", 0, fmt.Errorf("no port binding") } func (d *DockerContainer) Host() string { - _, hostIP, _, err := d.firstPortMapping() + _, hostIP, _, err := d.portMapping(true, -1) if err != nil { d.t.Fatal(err) } @@ -227,7 +210,7 @@ func (d *DockerContainer) Host() string { } func (d *DockerContainer) Port() uint { - _, _, port, err := d.firstPortMapping() + _, _, port, err := d.portMapping(true, -1) if err != nil { d.t.Fatal(err) } @@ -235,7 +218,7 @@ func (d *DockerContainer) Port() uint { } func (d *DockerContainer) PortFor(cPort int) uint { - _, _, port, err := d.mappingForPort(cPort) + _, _, port, err := d.portMapping(false, cPort) if err != nil { d.t.Fatal(err) } From 2cab3592224c08e7285b015091eafd2ec90a230c Mon Sep 17 00:00:00 2001 From: Taylor Wrobel Date: Fri, 7 Jul 2017 23:13:55 -0700 Subject: [PATCH 11/11] Remove debugging output from cockroachDB tests --- database/cockroachdb/cockroachdb_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/database/cockroachdb/cockroachdb_test.go b/database/cockroachdb/cockroachdb_test.go index 5d7fb68..e2dc1f8 100644 --- a/database/cockroachdb/cockroachdb_test.go +++ b/database/cockroachdb/cockroachdb_test.go @@ -20,7 +20,6 @@ var versions = []mt.Version{ } func isReady(i mt.Instance) bool { - fmt.Println("Checking readiness") db, err := sql.Open("postgres", fmt.Sprintf("postgres://root@%v:%v?sslmode=disable", i.Host(), i.PortFor(26257))) if err != nil { return false @@ -46,7 +45,6 @@ func Test(t *testing.T) { mt.ParallelTest(t, versions, isReady, func(t *testing.T, i mt.Instance) { c := &CockroachDb{} - fmt.Println("Connecting in test") addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", i.Host(), i.PortFor(26257)) d, err := c.Open(addr) if err != nil {