From 5f2a8b7f305e73d9bcb3745f308cc91ab1997a85 Mon Sep 17 00:00:00 2001 From: Fuyuan Bie Date: Mon, 15 May 2017 21:50:41 -0700 Subject: [PATCH 1/9] Use both db name and migration table name for lock hash --- database/mysql/mysql.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/database/mysql/mysql.go b/database/mysql/mysql.go index a559e11..480bcb4 100644 --- a/database/mysql/mysql.go +++ b/database/mysql/mysql.go @@ -156,7 +156,8 @@ func (m *Mysql) Lock() error { return database.ErrLocked } - aid, err := database.GenerateAdvisoryLockId(m.config.DatabaseName) + aid, err := database.GenerateAdvisoryLockId( + fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable) if err != nil { return err } @@ -180,7 +181,8 @@ func (m *Mysql) Unlock() error { return nil } - aid, err := database.GenerateAdvisoryLockId(m.config.DatabaseName) + aid, err := database.GenerateAdvisoryLockId( + fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable) if err != nil { return err } From bcdbe0f01862bc59c03c6ec717df736f86742d54 Mon Sep 17 00:00:00 2001 From: Fuyuan Bie Date: Mon, 15 May 2017 22:01:45 -0700 Subject: [PATCH 2/9] Fix parenthesis problem. --- database/mysql/mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/mysql/mysql.go b/database/mysql/mysql.go index 480bcb4..f00f886 100644 --- a/database/mysql/mysql.go +++ b/database/mysql/mysql.go @@ -157,7 +157,7 @@ func (m *Mysql) Lock() error { } aid, err := database.GenerateAdvisoryLockId( - fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable) + fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable)) if err != nil { return err } @@ -182,7 +182,7 @@ func (m *Mysql) Unlock() error { } aid, err := database.GenerateAdvisoryLockId( - fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable) + fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable)) if err != nil { return err } From 8b4ce58a1def3f7eafd857e1d78d597ab4640ca4 Mon Sep 17 00:00:00 2001 From: maxvw Date: Sat, 27 May 2017 08:56:05 +0200 Subject: [PATCH 3/9] adapted the sqlite driver for v3 (#165) --- Makefile | 2 +- README.md | 2 +- database/{sqlite => sqlite3}/README.md | 0 .../migration/33_create_table.down.sql | 1 + .../sqlite3/migration/33_create_table.up.sql | 3 + .../sqlite3/migration/44_alter_table.down.sql | 1 + .../sqlite3/migration/44_alter_table.up.sql | 1 + database/sqlite3/sqlite3.go | 214 ++++++++++++++++++ database/sqlite3/sqlite3_test.go | 61 +++++ 9 files changed, 283 insertions(+), 2 deletions(-) rename database/{sqlite => sqlite3}/README.md (100%) create mode 100644 database/sqlite3/migration/33_create_table.down.sql create mode 100644 database/sqlite3/migration/33_create_table.up.sql create mode 100644 database/sqlite3/migration/44_alter_table.down.sql create mode 100644 database/sqlite3/migration/44_alter_table.up.sql create mode 100644 database/sqlite3/sqlite3.go create mode 100644 database/sqlite3/sqlite3_test.go diff --git a/Makefile b/Makefile index 34e737e..408cc30 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go-bindata github aws-s3 google-cloud-storage -DATABASE ?= postgres mysql redshift +DATABASE ?= postgres mysql redshift sqlite3 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/README.md b/README.md index 2327f7d..ae301df 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [Redshift](database/redshift) * [Ql](database/ql) * [Cassandra](database/cassandra) ([todo #164](https://github.com/mattes/migrate/issues/164)) - * [SQLite](database/sqlite) ([todo #165](https://github.com/mattes/migrate/issues/165)) + * [SQLite](database/sqlite3) * [MySQL/ MariaDB](database/mysql) * [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167)) * [MongoDB](database/mongodb) ([todo #169](https://github.com/mattes/migrate/issues/169)) diff --git a/database/sqlite/README.md b/database/sqlite3/README.md similarity index 100% rename from database/sqlite/README.md rename to database/sqlite3/README.md diff --git a/database/sqlite3/migration/33_create_table.down.sql b/database/sqlite3/migration/33_create_table.down.sql new file mode 100644 index 0000000..72d18c5 --- /dev/null +++ b/database/sqlite3/migration/33_create_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/sqlite3/migration/33_create_table.up.sql b/database/sqlite3/migration/33_create_table.up.sql new file mode 100644 index 0000000..5ad3404 --- /dev/null +++ b/database/sqlite3/migration/33_create_table.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE pets ( + name string +); \ No newline at end of file diff --git a/database/sqlite3/migration/44_alter_table.down.sql b/database/sqlite3/migration/44_alter_table.down.sql new file mode 100644 index 0000000..72d18c5 --- /dev/null +++ b/database/sqlite3/migration/44_alter_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/sqlite3/migration/44_alter_table.up.sql b/database/sqlite3/migration/44_alter_table.up.sql new file mode 100644 index 0000000..f0682fc --- /dev/null +++ b/database/sqlite3/migration/44_alter_table.up.sql @@ -0,0 +1 @@ +ALTER TABLE pets ADD predator bool; diff --git a/database/sqlite3/sqlite3.go b/database/sqlite3/sqlite3.go new file mode 100644 index 0000000..bfd1a5b --- /dev/null +++ b/database/sqlite3/sqlite3.go @@ -0,0 +1,214 @@ +package sqlite3 + +import ( + "database/sql" + "fmt" + "github.com/mattes/migrate" + "github.com/mattes/migrate/database" + _ "github.com/mattn/go-sqlite3" + "io" + "io/ioutil" + nurl "net/url" + "strings" +) + +func init() { + database.Register("sqlite3", &Sqlite{}) +} + +var DefaultMigrationsTable = "schema_migrations" +var ( + ErrDatabaseDirty = fmt.Errorf("database is dirty") + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + DatabaseName string +} + +type Sqlite struct { + db *sql.DB + isLocked bool + + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + mx := &Sqlite{ + db: instance, + config: config, + } + if err := mx.ensureVersionTable(); err != nil { + return nil, err + } + return mx, nil +} + +func (m *Sqlite) ensureVersionTable() error { + + query := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); + CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); + `, DefaultMigrationsTable, DefaultMigrationsTable) + + if _, err := m.db.Exec(query); err != nil { + return err + } + return nil +} + +func (m *Sqlite) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite3://", "", 1) + db, err := sql.Open("sqlite3", dbfile) + if err != nil { + return nil, err + } + + migrationsTable := purl.Query().Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + mx, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + }) + if err != nil { + return nil, err + } + return mx, nil +} + +func (m *Sqlite) Close() error { + return m.db.Close() +} + +func (m *Sqlite) Drop() error { + query := `SELECT name FROM sqlite_master WHERE type = 'table';` + tables, err := m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer tables.Close() + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if len(tableNames) > 0 { + for _, t := range tableNames { + query := "DROP TABLE " + t + err = m.executeQuery(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + if err := m.ensureVersionTable(); err != nil { + return err + } + query := "VACUUM" + _, err = m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + return nil +} + +func (m *Sqlite) Lock() error { + if m.isLocked { + return database.ErrLocked + } + m.isLocked = true + return nil +} + +func (m *Sqlite) Unlock() error { + if !m.isLocked { + return nil + } + m.isLocked = false + return nil +} + +func (m *Sqlite) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + query := string(migr[:]) + + return m.executeQuery(query) +} + +func (m *Sqlite) executeQuery(query string) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + if _, err := tx.Exec(query); err != nil { + tx.Rollback() + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + return nil +} + +func (m *Sqlite) SetVersion(version int, dirty bool) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := "DELETE FROM " + m.config.MigrationsTable + if _, err := tx.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if version >= 0 { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (%d, '%t')`, m.config.MigrationsTable, version, dirty) + if _, err := tx.Exec(query); err != nil { + tx.Rollback() + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (m *Sqlite) Version() (version int, dirty bool, err error) { + query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" + err = m.db.QueryRow(query).Scan(&version, &dirty) + if err != nil { + return database.NilVersion, false, nil + } + return version, dirty, nil +} diff --git a/database/sqlite3/sqlite3_test.go b/database/sqlite3/sqlite3_test.go new file mode 100644 index 0000000..6a5c5c8 --- /dev/null +++ b/database/sqlite3/sqlite3_test.go @@ -0,0 +1,61 @@ +package sqlite3 + +import ( + "database/sql" + "fmt" + "github.com/mattes/migrate" + dt "github.com/mattes/migrate/database/testing" + _ "github.com/mattes/migrate/source/file" + _ "github.com/mattn/go-sqlite3" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func Test(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return + } + defer func() { + os.RemoveAll(dir) + }() + fmt.Printf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite3://%s", filepath.Join(dir, "sqlite3.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + + db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) + driver, err := WithInstance(db, &Config{}) + if err != nil { + t.Fatalf("%v", err) + } + if err := d.Drop(); err != nil { + t.Fatal(err) + } + + m, err := migrate.NewWithDatabaseInstance( + "file://./migration", + "ql", driver) + if err != nil { + t.Fatalf("%v", err) + } + fmt.Println("UP") + err = m.Up() + if err != nil { + t.Fatalf("%v", err) + } +} From decee4abb28da39c047b9ceb1af7e22437b9fea9 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 30 May 2017 14:06:59 +0200 Subject: [PATCH 4/9] added file cli/build_sqlite3.go --- cli/build_sqlite3.go | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 cli/build_sqlite3.go diff --git a/cli/build_sqlite3.go b/cli/build_sqlite3.go new file mode 100644 index 0000000..48ae8eb --- /dev/null +++ b/cli/build_sqlite3.go @@ -0,0 +1,7 @@ +// +build sqlite3 + +package main + +import ( + _ "github.com/mattes/migrate/database/sqlite3" +) From b5c2f0f2bb50eb62537062b486ce37bb78f33293 Mon Sep 17 00:00:00 2001 From: Pavlina Drosos Date: Tue, 30 May 2017 18:52:26 +0300 Subject: [PATCH 5/9] Add code sample for use with existing MySQL client with multistatements=true --- database/mysql/README.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/database/mysql/README.md b/database/mysql/README.md index fc79c98..490e90b 100644 --- a/database/mysql/README.md +++ b/database/mysql/README.md @@ -1,4 +1,4 @@ -# mysql +# MySQL `mysql://user:password@tcp(host:port)/dbname?query` @@ -15,6 +15,35 @@ | `x-tls-key` | | Key file location. | | `x-tls-insecure-skip-verify` | | Whether or not to use SSL (true\|false) | +## Use with existing client + +If you use the MySQL driver with existing database client, you must create the client with parameter `multiStatements=true`: + +```go +package main + +import ( + "database/sql" + + _ "github.com/go-sql-driver/mysql" + "github.com/mattes/migrate" + "github.com/mattes/migrate/database/mysql" + _ "github.com/mattes/migrate/source/file" +) + +func main() { + db, _ := sql.Open("mysql", "user:password@tcp(host:port)/dbname?multiStatements=true") + driver, _ := mysql.WithInstance(db, &mysql.Config{}) + m, _ := migrate.NewWithDatabaseInstance( + "file:///migrations", + "mysql", + driver, + ) + + m.Steps(2) +} +``` + ## Upgrading from v1 1. Write down the current migration version from schema_migrations From 28524d977cfab88cf219819f052563a230ce2862 Mon Sep 17 00:00:00 2001 From: Jolan Malassigne Date: Wed, 24 May 2017 09:59:18 +0200 Subject: [PATCH 6/9] add cassandra driver and function to retrieve networkSettings to get port bound to 9042 --- Makefile | 8 +- README.md | 2 +- cli/build_cassandra.go | 7 + database/cassandra/README.md | 29 ++++ database/cassandra/cassandra.go | 223 +++++++++++++++++++++++++++ database/cassandra/cassandra_test.go | 53 +++++++ database/driver.go | 4 +- testing/docker.go | 8 +- testing/testing.go | 7 +- 9 files changed, 331 insertions(+), 10 deletions(-) create mode 100644 cli/build_cassandra.go create mode 100644 database/cassandra/cassandra.go create mode 100644 database/cassandra/cassandra_test.go diff --git a/Makefile b/Makefile index 34e737e..c2725ab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go-bindata github aws-s3 google-cloud-storage -DATABASE ?= postgres mysql redshift +DATABASE ?= postgres mysql redshift cassandra VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") @@ -7,9 +7,9 @@ REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") build-cli: clean -mkdir ./cli/build - cd ./cli && GOOS=linux GOARCH=amd64 go build -a -o build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . - cd ./cli && GOOS=darwin GOARCH=amd64 go build -a -o build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . - cd ./cli && GOOS=windows GOARCH=amd64 go build -a -o build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . + cd ./cli && GOOS=linux GOARCH=amd64 go build -a -o build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . + cd ./cli && GOOS=darwin GOARCH=amd64 go build -a -o build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . + cd ./cli && GOOS=windows GOARCH=amd64 go build -a -o build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {} cd ./cli/build && shasum -a 256 * > sha256sum.txt cat ./cli/build/sha256sum.txt diff --git a/README.md b/README.md index 2327f7d..2ed3a37 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [PostgreSQL](database/postgres) * [Redshift](database/redshift) * [Ql](database/ql) - * [Cassandra](database/cassandra) ([todo #164](https://github.com/mattes/migrate/issues/164)) + * [Cassandra](database/cassandra) * [SQLite](database/sqlite) ([todo #165](https://github.com/mattes/migrate/issues/165)) * [MySQL/ MariaDB](database/mysql) * [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167)) diff --git a/cli/build_cassandra.go b/cli/build_cassandra.go new file mode 100644 index 0000000..319b52d --- /dev/null +++ b/cli/build_cassandra.go @@ -0,0 +1,7 @@ +// +build cassandra + +package main + +import ( + _ "github.com/mattes/migrate/database/cassandra" +) diff --git a/database/cassandra/README.md b/database/cassandra/README.md index e69de29..e7765e2 100644 --- a/database/cassandra/README.md +++ b/database/cassandra/README.md @@ -0,0 +1,29 @@ +# Cassandra + +* Drop command will not work on Cassandra 2.X because it rely on +system_schema table which comes with 3.X +* Other commands should work properly but are **not tested** + + +## Usage +`cassandra://host:port/keyspace?param1=value¶m2=value2` + + +| URL Query | Default value | Description | +|------------|-------------|-----------| +| `x-migrations-table` | schema_migrations | Name of the migrations table | +| `port` | 9042 | The port to bind to | +| `consistency` | ALL | Migration consistency +| `protocol` | | Cassandra protocol version (3 or 4) +| `timeout` | 1 minute | Migration timeout + + +`timeout` is parsed using [time.ParseDuration(s string)](https://golang.org/pkg/time/#ParseDuration) + + +## Upgrading from v1 + +1. Write down the current migration version from schema_migrations +2. `DROP TABLE schema_migrations` +4. Download and install the latest migrate version. +5. Force the current migration version with `migrate force `. diff --git a/database/cassandra/cassandra.go b/database/cassandra/cassandra.go new file mode 100644 index 0000000..3dd9379 --- /dev/null +++ b/database/cassandra/cassandra.go @@ -0,0 +1,223 @@ +package cassandra + +import ( + "fmt" + "io" + "io/ioutil" + nurl "net/url" + "github.com/gocql/gocql" + "time" + "github.com/mattes/migrate/database" + "strconv" +) + +func init() { + db := new(Cassandra) + database.Register("cassandra", db) +} + +var DefaultMigrationsTable = "schema_migrations" +var dbLocked = false + +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoKeyspace = fmt.Errorf("no keyspace provided") + ErrDatabaseDirty = fmt.Errorf("database is dirty") +) + +type Config struct { + MigrationsTable string + KeyspaceName string +} + +type Cassandra struct { + session *gocql.Session + isLocked bool + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func (p *Cassandra) Open(url string) (database.Driver, error) { + u, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + // Check for missing mandatory attributes + if len(u.Path) == 0 { + return nil, ErrNoKeyspace + } + + migrationsTable := u.Query().Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + + p.config = &Config{ + KeyspaceName: u.Path, + MigrationsTable: migrationsTable, + } + + cluster := gocql.NewCluster(u.Host) + cluster.Keyspace = u.Path[1:len(u.Path)] + cluster.Consistency = gocql.All + cluster.Timeout = 1 * time.Minute + + + // Retrieve query string configuration + if len(u.Query().Get("consistency")) > 0 { + var consistency gocql.Consistency + consistency, err = parseConsistency(u.Query().Get("consistency")) + if err != nil { + return nil, err + } + + cluster.Consistency = consistency + } + if len(u.Query().Get("protocol")) > 0 { + var protoversion int + protoversion, err = strconv.Atoi(u.Query().Get("protocol")) + if err != nil { + return nil, err + } + cluster.ProtoVersion = protoversion + } + if len(u.Query().Get("timeout")) > 0 { + var timeout time.Duration + timeout, err = time.ParseDuration(u.Query().Get("timeout")) + if err != nil { + return nil, err + } + cluster.Timeout = timeout + } + + p.session, err = cluster.CreateSession() + + if err != nil { + return nil, err + } + + if err := p.ensureVersionTable(); err != nil { + return nil, err + } + + return p, nil +} + +func (p *Cassandra) Close() error { + p.session.Close() + return nil +} + +func (p *Cassandra) Lock() error { + if (dbLocked) { + return database.ErrLocked + } + dbLocked = true + return nil +} + +func (p *Cassandra) Unlock() error { + dbLocked = false + return nil +} + +func (p *Cassandra) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + // run migration + query := string(migr[:]) + if err := p.session.Query(query).Exec(); err != nil { + // TODO: cast to Cassandra error and get line number + return database.Error{OrigErr: err, Err: "migration failed", Query: migr} + } + + return nil +} + +func (p *Cassandra) SetVersion(version int, dirty bool) error { + query := `TRUNCATE "` + p.config.MigrationsTable + `"` + if err := p.session.Query(query).Exec(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if version >= 0 { + query = `INSERT INTO "` + p.config.MigrationsTable + `" (version, dirty) VALUES (?, ?)` + if err := p.session.Query(query, version, dirty).Exec(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + return nil +} + + +// Return current keyspace version +func (p *Cassandra) Version() (version int, dirty bool, err error) { + query := `SELECT version, dirty FROM "` + p.config.MigrationsTable + `" LIMIT 1` + err = p.session.Query(query).Scan(&version, &dirty) + switch { + case err == gocql.ErrNotFound: + return database.NilVersion, false, nil + + case err != nil: + if _, ok := err.(*gocql.Error); ok { + return database.NilVersion, false, nil + } + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + + default: + return version, dirty, nil + } +} + +func (p *Cassandra) Drop() error { + // select all tables in current schema + query := fmt.Sprintf(`SELECT table_name from system_schema.tables WHERE keyspace_name='%s'`, p.config.KeyspaceName[1:]) // Skip '/' character + iter := p.session.Query(query).Iter() + var tableName string + for iter.Scan(&tableName) { + err := p.session.Query(fmt.Sprintf(`DROP TABLE %s`, tableName)).Exec() + if err != nil { + return err + } + } + // Re-create the version table + if err := p.ensureVersionTable(); err != nil { + return err + } + return nil +} + + +// Ensure version table exists +func (p *Cassandra) ensureVersionTable() error { + err := p.session.Query(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version bigint, dirty boolean, PRIMARY KEY(version))", p.config.MigrationsTable)).Exec() + if err != nil { + return err + } + if _, _, err = p.Version(); err != nil { + return err + } + return nil +} + + +// ParseConsistency wraps gocql.ParseConsistency +// to return an error instead of a panicking. +func parseConsistency(consistencyStr string) (consistency gocql.Consistency, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + err = fmt.Errorf("Failed to parse consistency \"%s\": %v", consistencyStr, r) + } + } + }() + consistency = gocql.ParseConsistency(consistencyStr) + + return consistency, nil +} diff --git a/database/cassandra/cassandra_test.go b/database/cassandra/cassandra_test.go new file mode 100644 index 0000000..4e9150f --- /dev/null +++ b/database/cassandra/cassandra_test.go @@ -0,0 +1,53 @@ +package cassandra + +import ( + "fmt" + "testing" + dt "github.com/mattes/migrate/database/testing" + mt "github.com/mattes/migrate/testing" + "github.com/gocql/gocql" + "time" + "strconv" +) + +var versions = []mt.Version{ + {"cassandra:3.0.10", []string{}}, + {"cassandra:3.0", []string{}}, +} + +func isReady(i mt.Instance) bool { + // Cassandra exposes 5 ports (7000, 7001, 7199, 9042 & 9160) + // We only need the port bound to 9042, but we can only access to the first one + // through 'i.Port()' (which calls DockerContainer.firstPortMapping()) + // So we need to get port mapping to retrieve correct port number bound to 9042 + portMap := i.NetworkSettings().Ports + port, _ := strconv.Atoi(portMap["9042/tcp"][0].HostPort) + + cluster := gocql.NewCluster(i.Host()) + cluster.Port = port + //cluster.ProtoVersion = 4 + cluster.Consistency = gocql.All + cluster.Timeout = 1 * time.Minute + p, err := cluster.CreateSession() + if err != nil { + return false + } + // Create keyspace for tests + p.Query("CREATE KEYSPACE testks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor':1}").Exec() + return true +} + +func Test(t *testing.T) { + mt.ParallelTest(t, versions, isReady, + func(t *testing.T, i mt.Instance) { + p := &Cassandra{} + portMap := i.NetworkSettings().Ports + port, _ := strconv.Atoi(portMap["9042/tcp"][0].HostPort) + addr := fmt.Sprintf("cassandra://%v:%v/testks", i.Host(), port) + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + dt.Test(t, d, []byte("SELECT table_name from system_schema.tables")) + }) +} \ No newline at end of file diff --git a/database/driver.go b/database/driver.go index 53d5240..3983a6a 100644 --- a/database/driver.go +++ b/database/driver.go @@ -32,6 +32,8 @@ var drivers = make(map[string]Driver) // All other functions are tested by tests in database/testing. // Saves you some time and makes sure all database drivers behave the same way. // 5. Call Register in init(). +// 6. Create a migrate/cli/build_.go file +// 7. Add driver name in 'DATABASE' variable in Makefile // // Guidelines: // * Don't try to correct user input. Don't assume things. @@ -71,7 +73,7 @@ type Driver interface { // Dirty means, a previous migration failed and user interaction is required. Version() (version int, dirty bool, err error) - // Drop deletes everyting in the database. + // Drop deletes everything in the database. Drop() error } diff --git a/testing/docker.go b/testing/docker.go index 9017e97..de93bbf 100644 --- a/testing/docker.go +++ b/testing/docker.go @@ -3,7 +3,7 @@ package testing import ( "bufio" - "context" // TODO: is issue with go < 1.7? + "context" "encoding/json" "fmt" "io" @@ -12,7 +12,6 @@ import ( "strings" "testing" "time" - dockertypes "github.com/docker/docker/api/types" dockercontainer "github.com/docker/docker/api/types/container" dockernetwork "github.com/docker/docker/api/types/network" @@ -202,6 +201,11 @@ func (d *DockerContainer) Port() uint { return port } +func (d *DockerContainer) NetworkSettings() dockertypes.NetworkSettings { + netSettings := d.ContainerJSON.NetworkSettings + return *netSettings +} + type dockerImagePullOutput struct { Status string `json:"status"` ProgressDetails struct { diff --git a/testing/testing.go b/testing/testing.go index fb04ad7..0d03432 100644 --- a/testing/testing.go +++ b/testing/testing.go @@ -6,6 +6,8 @@ import ( "strconv" "testing" "time" + + dockertypes "github.com/docker/docker/api/types" ) type IsReadyFunc func(Instance) bool @@ -46,8 +48,8 @@ func ParallelTest(t *testing.T, versions []Version, readyFn IsReadyFunc, testFn // wait until database is ready tick := time.Tick(1000 * time.Millisecond) - timeout := time.After(time.Duration(delay+60) * time.Second) - outer: + timeout := time.After(time.Duration(delay + 60) * time.Second) + outer: for { select { case <-tick: @@ -87,5 +89,6 @@ func containerLogs(t *testing.T, c *DockerContainer) []byte { type Instance interface { Host() string Port() uint + NetworkSettings() dockertypes.NetworkSettings KeepForDebugging() } From a9bcd83ff0358ec6f1f957cc368241a445522338 Mon Sep 17 00:00:00 2001 From: Matthias Kadenbach Date: Mon, 5 Jun 2017 00:59:53 -0700 Subject: [PATCH 7/9] enable cgo --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index bf0584f..3cb43c8 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,9 @@ REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") build-cli: clean -mkdir ./cli/build - cd ./cli && GOOS=linux GOARCH=amd64 go build -a -o build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . - cd ./cli && GOOS=darwin GOARCH=amd64 go build -a -o build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . - cd ./cli && GOOS=windows GOARCH=amd64 go build -a -o build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . + cd ./cli && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . + cd ./cli && CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -a -o build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . + cd ./cli && CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -o build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {} cd ./cli/build && shasum -a 256 * > sha256sum.txt cat ./cli/build/sha256sum.txt From 39a5a378221f2c696245e7e5d63f825ca9d1b274 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 5 Jun 2017 12:14:10 -0400 Subject: [PATCH 8/9] Fix typo forgotton => forgotten --- database/driver.go | 2 +- source/driver.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/database/driver.go b/database/driver.go index 3983a6a..016eedc 100644 --- a/database/driver.go +++ b/database/driver.go @@ -92,7 +92,7 @@ func Open(url string) (Driver, error) { d, ok := drivers[u.Scheme] driversMu.RUnlock() if !ok { - return nil, fmt.Errorf("database driver: unknown driver %v (forgotton import?)", u.Scheme) + return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", u.Scheme) } return d.Open(url) diff --git a/source/driver.go b/source/driver.go index 103138d..b9c052c 100644 --- a/source/driver.go +++ b/source/driver.go @@ -87,7 +87,7 @@ func Open(url string) (Driver, error) { d, ok := drivers[u.Scheme] driversMu.RUnlock() if !ok { - return nil, fmt.Errorf("source driver: unknown driver %v (forgotton import?)", u.Scheme) + return nil, fmt.Errorf("source driver: unknown driver %v (forgotten import?)", u.Scheme) } return d.Open(url) From 9a19439e311e1ac09c3964bc9ee55b27401ee5f0 Mon Sep 17 00:00:00 2001 From: Stephanie Ng Date: Sat, 10 Jun 2017 20:03:17 +0800 Subject: [PATCH 9/9] add create command to cli --- cli/README.md | 2 ++ cli/commands.go | 15 +++++++++++++++ cli/main.go | 27 +++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/cli/README.md b/cli/README.md index b3b38d2..74865dc 100644 --- a/cli/README.md +++ b/cli/README.md @@ -54,6 +54,8 @@ Options: -help Print usage Commands: + create [-ext E] [-dir D] NAME + Create a set of timestamped up/down migrations titled NAME, in directory D with extension E goto V Migrate to version V up [N] Apply all or N up migrations down [N] Apply all or N down migrations diff --git a/cli/commands.go b/cli/commands.go index e264621..703896d 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -4,8 +4,23 @@ import ( "github.com/mattes/migrate" _ "github.com/mattes/migrate/database/stub" // TODO remove again _ "github.com/mattes/migrate/source/file" + "os" + "fmt" ) +func createCmd(dir string, timestamp int64, name string, ext string) { + base := fmt.Sprintf("%v%v_%v.", dir, timestamp, name) + os.MkdirAll(dir, os.ModePerm) + createFile(base + "up" + ext) + createFile(base + "down" + ext) +} + +func createFile(fname string) { + if _, err := os.Create(fname); err != nil { + log.fatalErr(err) + } +} + func gotoCmd(m *migrate.Migrate, v uint) { if err := m.Migrate(v); err != nil { if err != migrate.ErrNoChange { diff --git a/cli/main.go b/cli/main.go index 58b17c1..f019b5c 100644 --- a/cli/main.go +++ b/cli/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "strings" "os" "os/signal" "strconv" @@ -41,6 +42,8 @@ Options: -help Print usage Commands: + create [-ext E] [-dir D] NAME + Create a set of timestamped up/down migrations titled NAME, in directory D with extension E goto V Migrate to version V up [N] Apply all or N up migrations down [N] Apply all or N down migrations @@ -101,6 +104,30 @@ Commands: startTime := time.Now() switch flag.Arg(0) { + case "create": + args := flag.Args()[1:] + + createFlagSet := flag.NewFlagSet("create", flag.ExitOnError) + extPtr := createFlagSet.String("ext", "", "File extension") + dirPtr := createFlagSet.String("dir", "", "Directory to place file in (default: current working directory)") + createFlagSet.Parse(args) + + if createFlagSet.NArg() == 0 { + log.fatal("error: please specify name") + } + name := createFlagSet.Arg(0) + + if *extPtr != "" { + *extPtr = "." + strings.TrimPrefix(*extPtr, ".") + } + if *dirPtr != "" { + *dirPtr = strings.Trim(*dirPtr, "/") + "/" + } + + timestamp := startTime.Unix() + + createCmd(*dirPtr, timestamp, name, *extPtr) + case "goto": if migraterErr != nil { log.fatalErr(migraterErr)