From 907d8cd935838b4f2a19ce309c84c4504d92bc09 Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 11:48:39 +0200 Subject: [PATCH 1/4] Added driver for crate.io database --- docker-compose.yml | 5 +- driver/crate/crate.go | 105 +++++++++++++++++++++++++++++++++++++ driver/crate/crate_test.go | 87 ++++++++++++++++++++++++++++++ main.go | 1 + 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 driver/crate/crate.go create mode 100644 driver/crate/crate_test.go diff --git a/docker-compose.yml b/docker-compose.yml index cac3abc..2b13d8f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,7 @@ go-test: - postgres - mysql - cassandra + - crate go-build: <<: *go command: sh -c 'go get -v && go build -ldflags ''-s'' -o migrater' @@ -21,6 +22,8 @@ mysql: image: mysql environment: MYSQL_DATABASE: migratetest - MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" cassandra: image: cassandra:2.2 +crate: + image: crate diff --git a/driver/crate/crate.go b/driver/crate/crate.go new file mode 100644 index 0000000..8773b66 --- /dev/null +++ b/driver/crate/crate.go @@ -0,0 +1,105 @@ +// Package crate implements a driver for the Crate.io database +package crate + +import ( + "database/sql" + "fmt" + "strings" + + _ "github.com/herenow/go-crate" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" +) + +func init() { + driver.RegisterDriver("crate", &Driver{}) +} + +type Driver struct { + db *sql.DB +} + +const tableName = "schema_migrations" + +func (driver *Driver) Initialize(url string) error { + db, err := sql.Open("crate", url) + if err != nil { + return err + } + if err := db.Ping(); err != nil { + return err + } + driver.db = db + + if err := driver.ensureVersionTableExists(); err != nil { + return err + } + return nil +} + +func (driver *Driver) Close() error { + if err := driver.db.Close(); err != nil { + return err + } + return nil +} + +func (driver *Driver) FilenameExtension() string { + return "sql" +} + +func (driver *Driver) Version() (uint64, error) { + var version uint64 + err := driver.db.QueryRow("SELECT version FROM " + tableName + " ORDER BY version DESC LIMIT 1").Scan(&version) + switch { + case err == sql.ErrNoRows: + return 0, nil + case err != nil: + return 0, err + default: + return version, nil + } +} + +func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + if err := f.ReadContent(); err != nil { + pipe <- err + return + } + + lines := strings.Split(string(f.Content), ";") + for _, line := range lines { + query := strings.TrimSpace(line) + query = strings.Replace(query, ";", "", -1) + if query != "" { + _, err := driver.db.Exec(query) + if err != nil { + pipe <- err + return + } + } + } + + if f.Direction == direction.Up { + if _, err := driver.db.Exec("INSERT INTO "+tableName+" (version) VALUES (?)", f.Version); err != nil { + pipe <- err + return + } + } else if f.Direction == direction.Down { + if _, err := driver.db.Exec("DELETE FROM "+tableName+" WHERE version=?", f.Version); err != nil { + pipe <- err + return + } + } +} + +func (driver *Driver) ensureVersionTableExists() error { + if _, err := driver.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version INTEGER PRIMARY KEY)", tableName)); err != nil { + return err + } + return nil +} diff --git a/driver/crate/crate_test.go b/driver/crate/crate_test.go new file mode 100644 index 0000000..05367dd --- /dev/null +++ b/driver/crate/crate_test.go @@ -0,0 +1,87 @@ +package crate + +import ( + "fmt" + "os" + "testing" + + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" +) + +func TestMigrate(t *testing.T) { + host := os.Getenv("CRATE_PORT_4200_TCP_ADDR") + port := os.Getenv("CRATE_PORT_4200_TCP_PORT") + + url := fmt.Sprintf("http://%s:%s", host, port) + + driver := &Driver{} + + if err := driver.Initialize(url); err != nil { + t.Fatal(err) + } + + successFiles := []file.File{ + { + Path: "/foobar", + FileName: "001_foobar.up.sql", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + CREATE TABLE yolo ( + id integer primary key, + msg string + ); + `), + }, + { + Path: "/foobar", + FileName: "002_foobar.down.sql", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + DROP TABLE yolo; + `), + }, + } + + failFiles := []file.File{ + { + Path: "/foobar", + FileName: "002_foobar.up.sql", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + CREATE TABLE error ( + id THIS WILL CAUSE AN ERROR + ) + `), + }, + } + + for _, file := range successFiles { + pipe := pipep.New() + go driver.Migrate(file, pipe) + errs := pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + } + + for _, file := range failFiles { + pipe := pipep.New() + go driver.Migrate(file, pipe) + errs := pipep.ReadErrors(pipe) + if len(errs) == 0 { + t.Fatal("Migration should have failed but succeeded") + } + } + + if err := driver.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/main.go b/main.go index b6da438..49f3e52 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "github.com/fatih/color" _ "github.com/mattes/migrate/driver/bash" _ "github.com/mattes/migrate/driver/cassandra" + _ "github.com/mattes/migrate/driver/crate" _ "github.com/mattes/migrate/driver/mysql" _ "github.com/mattes/migrate/driver/postgres" _ "github.com/mattes/migrate/driver/sqlite3" From a7fa9eb3fbac6dedba484ea47b770c2d1dae5d04 Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 13:08:04 +0200 Subject: [PATCH 2/4] Added README.md for Crate driver --- driver/crate/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 driver/crate/README.md diff --git a/driver/crate/README.md b/driver/crate/README.md new file mode 100644 index 0000000..a43d013 --- /dev/null +++ b/driver/crate/README.md @@ -0,0 +1,15 @@ +# Crate driver + +This is a driver for the [Crate](https://crate.io) database. It is based on the Crate +sql driver by [herenow](https://github.com/herenow/go-crate). + +This driver does not use transactions! This is not a limitation of the driver, but a +limitation of Crate. So handle situations with failed migrations with care! + +## Usage + +```bash +migrate -url http://host:port -path ./db/migrations create add_field_to_table +migrate -url http://host:port -path ./db/migrations up +migrate help # for more info +``` \ No newline at end of file From 6612806e067b1f502e4f40d489156e3bb7b53636 Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 13:47:21 +0200 Subject: [PATCH 3/4] Crate driver was using wrong url scheme. The migrate CLI never had a chance to select the correct driver --- driver/crate/crate.go | 1 + driver/crate/crate_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/driver/crate/crate.go b/driver/crate/crate.go index 8773b66..faf4ef7 100644 --- a/driver/crate/crate.go +++ b/driver/crate/crate.go @@ -23,6 +23,7 @@ type Driver struct { const tableName = "schema_migrations" func (driver *Driver) Initialize(url string) error { + url = strings.Replace(url, "crate", "http", 1) db, err := sql.Open("crate", url) if err != nil { return err diff --git a/driver/crate/crate_test.go b/driver/crate/crate_test.go index 05367dd..1896432 100644 --- a/driver/crate/crate_test.go +++ b/driver/crate/crate_test.go @@ -14,7 +14,7 @@ func TestMigrate(t *testing.T) { host := os.Getenv("CRATE_PORT_4200_TCP_ADDR") port := os.Getenv("CRATE_PORT_4200_TCP_PORT") - url := fmt.Sprintf("http://%s:%s", host, port) + url := fmt.Sprintf("crate://%s:%s", host, port) driver := &Driver{} From cd5f75d47c7532396414617e83c533c5adf53d54 Mon Sep 17 00:00:00 2001 From: Till Klocke Date: Thu, 16 Jun 2016 14:34:50 +0200 Subject: [PATCH 4/4] Splitting of sql is now more robust and better tested --- driver/crate/crate.go | 27 ++++++++++++++++++--------- driver/crate/crate_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/driver/crate/crate.go b/driver/crate/crate.go index faf4ef7..eb30853 100644 --- a/driver/crate/crate.go +++ b/driver/crate/crate.go @@ -72,16 +72,12 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { return } - lines := strings.Split(string(f.Content), ";") + lines := splitContent(string(f.Content)) for _, line := range lines { - query := strings.TrimSpace(line) - query = strings.Replace(query, ";", "", -1) - if query != "" { - _, err := driver.db.Exec(query) - if err != nil { - pipe <- err - return - } + _, err := driver.db.Exec(line) + if err != nil { + pipe <- err + return } } @@ -98,6 +94,19 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { } } +func splitContent(content string) []string { + lines := strings.Split(content, ";") + resultLines := make([]string, 0, len(lines)) + for i, line := range lines { + line = strings.Replace(lines[i], ";", "", -1) + line = strings.TrimSpace(line) + if line != "" { + resultLines = append(resultLines, line) + } + } + return resultLines +} + func (driver *Driver) ensureVersionTableExists() error { if _, err := driver.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version INTEGER PRIMARY KEY)", tableName)); err != nil { return err diff --git a/driver/crate/crate_test.go b/driver/crate/crate_test.go index 1896432..164cd57 100644 --- a/driver/crate/crate_test.go +++ b/driver/crate/crate_test.go @@ -10,6 +10,30 @@ import ( pipep "github.com/mattes/migrate/pipe" ) +func TestContentSplit(t *testing.T) { + content := `CREATE TABLE users (user_id STRING primary key, first_name STRING, last_name STRING, email STRING, password_hash STRING) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); +CREATE TABLE units (unit_id STRING primary key, name STRING, members array(string)) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); +CREATE TABLE available_connectors (technology_id STRING primary key, description STRING, icon STRING, link STRING, configuration_parameters array(object as (name STRING, type STRING))) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0); + ` + + lines := splitContent(content) + if len(lines) != 3 { + t.Errorf("Expected 3 lines, but got %d", len(lines)) + } + + if lines[0] != "CREATE TABLE users (user_id STRING primary key, first_name STRING, last_name STRING, email STRING, password_hash STRING) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } + + if lines[1] != "CREATE TABLE units (unit_id STRING primary key, name STRING, members array(string)) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } + + if lines[2] != "CREATE TABLE available_connectors (technology_id STRING primary key, description STRING, icon STRING, link STRING, configuration_parameters array(object as (name STRING, type STRING))) CLUSTERED INTO 3 shards WITH (number_of_replicas = 0)" { + t.Error("Line does not match expected output") + } +} + func TestMigrate(t *testing.T) { host := os.Getenv("CRATE_PORT_4200_TCP_ADDR") port := os.Getenv("CRATE_PORT_4200_TCP_PORT")