From 8630ea78b78799bd69b6081f18b6ba2652c66709 Mon Sep 17 00:00:00 2001 From: dimag Date: Thu, 4 Aug 2016 15:52:31 +0300 Subject: [PATCH 01/27] changed import paths to point to dimag-jfrog --- driver/bash/bash.go | 4 ++-- driver/cassandra/cassandra.go | 6 +++--- driver/cassandra/cassandra_test.go | 6 +++--- driver/driver.go | 2 +- driver/mysql/mysql.go | 6 +++--- driver/mysql/mysql_test.go | 6 +++--- driver/postgres/postgres.go | 6 +++--- driver/postgres/postgres_test.go | 6 +++--- driver/sqlite3/sqlite3.go | 6 +++--- driver/sqlite3/sqlite3_test.go | 6 +++--- file/file.go | 2 +- file/file_test.go | 2 +- main.go | 20 ++++++++++---------- migrate/migrate.go | 8 ++++---- migrate/migrate_test.go | 4 ++-- 15 files changed, 45 insertions(+), 45 deletions(-) diff --git a/driver/bash/bash.go b/driver/bash/bash.go index 031f9bb..97ba91f 100644 --- a/driver/bash/bash.go +++ b/driver/bash/bash.go @@ -2,8 +2,8 @@ package bash import ( - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" ) type Driver struct { diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index e242de2..ffc76ad 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -9,9 +9,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index ea73794..7d36389 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -7,9 +7,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) func TestMigrate(t *testing.T) { diff --git a/driver/driver.go b/driver/driver.go index e4ecb78..5c2f6fc 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -5,7 +5,7 @@ import ( "fmt" neturl "net/url" // alias to allow `url string` func signature in New - "github.com/mattes/migrate/file" + "github.com/dimag-jfrog/migrate/file" ) // Driver is the interface type that needs to implemented by all drivers. diff --git a/driver/mysql/mysql.go b/driver/mysql/mysql.go index 2bbbc13..cf186f8 100644 --- a/driver/mysql/mysql.go +++ b/driver/mysql/mysql.go @@ -12,9 +12,9 @@ import ( "strings" "github.com/go-sql-driver/mysql" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/mysql/mysql_test.go b/driver/mysql/mysql_test.go index 51cf552..9e08af9 100644 --- a/driver/mysql/mysql_test.go +++ b/driver/mysql/mysql_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/postgres/postgres.go b/driver/postgres/postgres.go index 502c7d1..dbbe4a2 100644 --- a/driver/postgres/postgres.go +++ b/driver/postgres/postgres.go @@ -8,9 +8,9 @@ import ( "strconv" "github.com/lib/pq" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go index ec91b35..552f23d 100644 --- a/driver/postgres/postgres_test.go +++ b/driver/postgres/postgres_test.go @@ -5,9 +5,9 @@ import ( "os" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/sqlite3/sqlite3.go b/driver/sqlite3/sqlite3.go index a92b14d..65e6a2a 100644 --- a/driver/sqlite3/sqlite3.go +++ b/driver/sqlite3/sqlite3.go @@ -7,9 +7,9 @@ import ( "fmt" "strings" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" "github.com/mattn/go-sqlite3" ) diff --git a/driver/sqlite3/sqlite3_test.go b/driver/sqlite3/sqlite3_test.go index 8d51a7d..29a501f 100644 --- a/driver/sqlite3/sqlite3_test.go +++ b/driver/sqlite3/sqlite3_test.go @@ -4,9 +4,9 @@ import ( "database/sql" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate() diff --git a/file/file.go b/file/file.go index 54992a4..14617ba 100644 --- a/file/file.go +++ b/file/file.go @@ -5,7 +5,7 @@ import ( "bytes" "errors" "fmt" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/migrate/direction" "go/token" "io/ioutil" "path" diff --git a/file/file_test.go b/file/file_test.go index b9ddeab..c6bd0b3 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -1,7 +1,7 @@ package file import ( - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/migrate/direction" "io/ioutil" "os" "path" diff --git a/main.go b/main.go index b6da438..51d68a8 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ // Package main is the CLI. // You can use the CLI via Terminal. -// import "github.com/mattes/migrate/migrate" for usage within Go. +// import "github.com/dimag-jfrog/migrate/migrate" for usage within Go. package main import ( @@ -11,15 +11,15 @@ import ( "time" "github.com/fatih/color" - _ "github.com/mattes/migrate/driver/bash" - _ "github.com/mattes/migrate/driver/cassandra" - _ "github.com/mattes/migrate/driver/mysql" - _ "github.com/mattes/migrate/driver/postgres" - _ "github.com/mattes/migrate/driver/sqlite3" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + _ "github.com/dimag-jfrog/migrate/driver/bash" + _ "github.com/dimag-jfrog/migrate/driver/cassandra" + _ "github.com/dimag-jfrog/migrate/driver/mysql" + _ "github.com/dimag-jfrog/migrate/driver/postgres" + _ "github.com/dimag-jfrog/migrate/driver/sqlite3" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) var url = flag.String("url", os.Getenv("MIGRATE_URL"), "") diff --git a/migrate/migrate.go b/migrate/migrate.go index b452aee..54cc42b 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -11,10 +11,10 @@ import ( "strconv" "strings" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // Up applies all available migrations diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 7d2a622..7909826 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -6,8 +6,8 @@ import ( "testing" // Ensure imports for each driver we wish to test - _ "github.com/mattes/migrate/driver/postgres" - _ "github.com/mattes/migrate/driver/sqlite3" + _ "github.com/dimag-jfrog/migrate/driver/postgres" + _ "github.com/dimag-jfrog/migrate/driver/sqlite3" ) // Add Driver URLs here to test basic Up, Down, .. functions. From 64101a13e946a05197e0ff6b5c589827544f2ffc Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 16:29:25 +0300 Subject: [PATCH 02/27] Changes that interfere with the Migrate open source code: - Added go methods migrator, mongo db template: different from the usual driver model. - Added support for bidirectional files (for go methods), appending _up or _down upon context - Added DriverWithFilnameParser for providing custom filename parser functionality that knows to parse bi-directional file names. --- .gitignore | 11 +- driver/driver.go | 9 + driver/gomethods/gomethods_migrator.go | 176 +++++++++ driver/gomethods/gomethods_migrator_test.go | 342 ++++++++++++++++++ driver/gomethods/mongodb/mongodb_template.go | 111 ++++++ driver/gomethods/usage_examples/mongodb.go | 121 +++++++ .../gomethods/usage_examples/mongodb_test.go | 123 +++++++ file/file.go | 106 +++--- file/file_test.go | 50 +-- file/filename_parser.go | 85 +++++ file/filename_parser_test.go | 85 +++++ migrate/direction/direction.go | 1 + migrate/migrate.go | 12 +- 13 files changed, 1128 insertions(+), 104 deletions(-) create mode 100644 driver/gomethods/gomethods_migrator.go create mode 100644 driver/gomethods/gomethods_migrator_test.go create mode 100644 driver/gomethods/mongodb/mongodb_template.go create mode 100644 driver/gomethods/usage_examples/mongodb.go create mode 100644 driver/gomethods/usage_examples/mongodb_test.go create mode 100644 file/filename_parser.go create mode 100644 file/filename_parser_test.go diff --git a/.gitignore b/.gitignore index ee2f41f..9bd9f47 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,11 @@ .DS_Store -test.db \ No newline at end of file +test.db +*.iml +.DS_Store +.idea +*.zip +*.tar.* +*.nuget +*.jar +*.war + diff --git a/driver/driver.go b/driver/driver.go index 5c2f6fc..a668d52 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -34,6 +34,15 @@ type Driver interface { Version() (uint64, error) } +// Driver that has some custom migration file format +// and wants to use a different parsing strategy. +type DriverWithFilenameParser interface { + + Driver + + FilenameParser() file.FilenameParser +} + // New returns Driver and calls Initialize on it func New(url string) (Driver, error) { u, err := neturl.Parse(url) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go new file mode 100644 index 0000000..68b9ff5 --- /dev/null +++ b/driver/gomethods/gomethods_migrator.go @@ -0,0 +1,176 @@ +package gomethods + +import ( + //"bytes" + "reflect" + "fmt" + "strings" + "os" + "path" + "bufio" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" +) + + +type MissingMethodError string +func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } + + +type WrongMethodSignatureError string +func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", e) } + +type MethodInvocationFailedError struct { + MethodName string + Err error +} + +func (e *MethodInvocationFailedError) Error() string { + return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) +} + + +type Migrator struct { + Driver driver.DriverWithFilenameParser + RollbackOnFailure bool +} + +func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { + methods, err := m.getMigrationMethods(f) + if err != nil { + pipe <- err + return err + } + + for i, methodName := range methods { + pipe <- methodName + err := m.Invoke(methodName) + if err != nil { + pipe <- err + if !m.RollbackOnFailure { + return err + } + + // on failure, try to rollback methods in this migration + for j := i-1; j >= 0; j-- { + rollbackToMethodName := getRollbackToMethod(methods[j]) + pipe <- rollbackToMethodName + err = m.Invoke(rollbackToMethodName) + if err != nil { + pipe <- err + break + } + } + return err + } + } + + return nil +} + +func (m *Migrator) IsValid(methodName string) bool { + return reflect.ValueOf(m.Driver).MethodByName(methodName).IsValid() +} + +func (m *Migrator) Invoke(methodName string) error { + name := methodName + migrateMethod := reflect.ValueOf(m.Driver).MethodByName(name) + if !migrateMethod.IsValid() { + return MissingMethodError(methodName) + } + + retValues := migrateMethod.Call([]reflect.Value{}) + if len(retValues) != 1 { + return WrongMethodSignatureError(name) + } + + if !retValues[0].IsNil() { + err, ok := retValues[0].Interface().(error) + if !ok { + return WrongMethodSignatureError(name) + } + return &MethodInvocationFailedError{ MethodName:name, Err:err} + } + + return nil +} + +func reverseInPlace(a []string) { + for i := 0; i < len(a)/2; i++ { + j := len(a) - i - 1 + a[i], a[j] = a[j], a[i] + } +} + +func getRollbackToMethod(methodName string) string { + if strings.HasSuffix(methodName, "_up") { + return strings.TrimSuffix(methodName, "_up") + "_down" + } else { + return strings.TrimSuffix(methodName, "_down") + "_up" + } +} + +func getFileLines(file file.File) ([]string, error) { + if len(file.Content) == 0 { + lines := make([]string, 0) + file, err := os.Open(path.Join(file.Path, file.FileName)) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, nil + } else { + //n := bytes.IndexByte(file.Content, 0) + //n := bytes.Index(file.Content, []byte{0}) + //s := string(file.Content[:n]) + s := string(file.Content) + return strings.Split(s, "\n"), nil + } +} + +func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { + var lines, methods []string + lines, err := getFileLines(f) + if err != nil { + return nil, err + } + + for _, line := range lines { + operationName := strings.TrimSpace(line) + + if operationName == "" || strings.HasPrefix(operationName, "--") { + // an empty line or a comment, ignore + continue + } + + upMethodName := operationName + "_up" + downMethodName := operationName + "_down" + + if !m.IsValid(upMethodName) { + return nil, MissingMethodError(upMethodName) + } + if !m.IsValid(downMethodName) { + return nil, MissingMethodError(downMethodName) + } + + if f.Direction == direction.Up { + methods = append(methods, upMethodName) + } else { + methods = append(methods, downMethodName) + } + } + + _,_,fileType,_ := m.Driver.FilenameParser().Parse(f.FileName) + if fileType == direction.Both && f.Direction == direction.Down { + reverseInPlace(methods) + } + return methods, nil + +} diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go new file mode 100644 index 0000000..58fd4f2 --- /dev/null +++ b/driver/gomethods/gomethods_migrator_test.go @@ -0,0 +1,342 @@ +package gomethods + +import ( + "reflect" + "testing" + + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + + pipep "github.com/dimag-jfrog/migrate/pipe" +) + +type FakeGoMethodsDriver struct { + InvokedMethods []string + Migrator Migrator +} + +func (driver *FakeGoMethodsDriver) Initialize(url string) error { + return nil +} + +func (driver *FakeGoMethodsDriver) Close() error { + return nil +} + +func (driver *FakeGoMethodsDriver) FilenameParser() file.FilenameParser { + return file.UpDownAndBothFilenameParser{ FilenameExtension: driver.FilenameExtension() } +} + +func (driver *FakeGoMethodsDriver) FilenameExtension() string { + return "gm" +} + +func (driver *FakeGoMethodsDriver) Version() (uint64, error) { + return uint64(0), nil +} + +func (driver *FakeGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + return +} + +func (driver *FakeGoMethodsDriver) V001_init_organizations_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_up") + return nil +} + +func (driver *FakeGoMethodsDriver) V001_init_organizations_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_down") + return nil + +} + +func (driver *FakeGoMethodsDriver) V001_init_users_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_up") + return nil +} + +func (driver *FakeGoMethodsDriver) V001_init_users_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_down") + return nil +} + +type SomeError struct{} +func (e SomeError) Error() string { return "Some error happened" } + +func (driver *FakeGoMethodsDriver) V001_some_failing_method_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_up") + return SomeError{} +} + +func (driver *FakeGoMethodsDriver) V001_some_failing_method_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_down") + return SomeError{} +} + +func TestMigrate(t *testing.T) { + cases := []struct { + name string + file file.File + expectedInvokedMethods []string + expectedErrors []error + expectRollback bool + }{ + { + name: "up migration, both directions-file: invokes up methods in order", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, + expectedErrors: []error{}, + }, + { + name: "down migration, both-directions-file: reverts direction of invoked down methods", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, + expectedErrors: []error{}, + }, + { + name: "up migration, up direction-file: invokes up methods in order", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, + expectedErrors: []error{}, + }, + { + name: "down migration, down directions-file: keeps order of invoked down methods", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_down", "V001_init_users_down"}, + expectedErrors: []error{}, + }, + { + name: "up migration: non-existing method causes migration not to execute", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_some_non_existing_method + `), + }, + expectedInvokedMethods: []string{}, + expectedErrors: []error{ MissingMethodError("V001_some_non_existing_method_up") }, + }, + { + name: "up migration: failing method stops execution", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_organizations_up", + "V001_some_failing_method_up", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_up", + Err: SomeError{}, + }}, + }, + { + name: "down migration, both-directions-file: failing method stops migration", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_users_down", + "V001_some_failing_method_down", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_down", + Err: SomeError{}, + }}, + }, + { + name: "up migration: failing method causes rollback in rollback mode", + expectRollback: true, + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_some_failing_method + `), + }, + expectedInvokedMethods: []string{ + "V001_init_organizations_up", + "V001_init_users_up", + "V001_some_failing_method_up", + "V001_init_users_down", + "V001_init_organizations_down", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_up", + Err: SomeError{}, + }}, + }, + { + name: "down migration, both-directions-file: failing method causes rollback in rollback mode", + expectRollback: true, + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_users_down", + "V001_some_failing_method_down", + "V001_init_users_up", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_down", + Err: SomeError{}, + }}, + }, + + } + + for _, c := range cases { + migrator := Migrator{} + d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} + migrator.Driver = d + migrator.RollbackOnFailure = c.expectRollback + + pipe := pipep.New() + go func() { + migrator.Migrate(c.file, pipe) + close(pipe) + }() + errs := pipep.ReadErrors(pipe) + + var failed bool + if !reflect.DeepEqual(d.InvokedMethods, c.expectedInvokedMethods) { + failed = true + t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, d.InvokedMethods) + } + if !reflect.DeepEqual(errs, c.expectedErrors) { + failed = true + t.Errorf("case '%s': FAILED\nexpected errors %v\nbut got %v", c.name, c.expectedErrors, errs) + + } + if !failed { + t.Logf("case '%s': PASSED", c.name) + } + } +} + + + +func TestGetRollbackToMethod(t *testing.T) { + cases := []struct { + method string + expectedRollbackMethod string + }{ + {"some_method_up", "some_method_down"}, + {"some_method_down", "some_method_up"}, + {"up_down_up", "up_down_down"}, + {"down_up", "down_down"}, + {"down_down", "down_up"}, + } + + for _, c := range cases { + actualRollbackMethod := getRollbackToMethod(c.method) + if actualRollbackMethod != c.expectedRollbackMethod { + t.Errorf("Expected rollback method to be %s but got %s", c.expectedRollbackMethod, actualRollbackMethod) + } + } +} + +func TestReverseInPlace(t *testing.T) { + methods := []string { + "method1_down", + "method2_down", + "method3_down", + "method4_down", + "method5_down", + } + + expectedReversedMethods := []string { + "method5_down", + "method4_down", + "method3_down", + "method2_down", + "method1_down", + } + + reverseInPlace(methods) + + if !reflect.DeepEqual(methods, expectedReversedMethods) { + t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) + } +} + diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go new file mode 100644 index 0000000..48817cf --- /dev/null +++ b/driver/gomethods/mongodb/mongodb_template.go @@ -0,0 +1,111 @@ +package mongodb + +import ( + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "errors" + "strings" + "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/driver/gomethods" +) + +const MIGRATE_C = "db_migrations" + + +// This is not a real driver since the Initialize method requires a gomethods.Migrator +// The real driver will contain the DriverTemplate and implement all the custom migration Golang methods +// See example in usage_examples for details +type DriverTemplate struct { + Session *mgo.Session + DbName string + + migrator gomethods.Migrator +} + + +type DbMigration struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Version uint64 `bson:"version"` +} + +func (driver *DriverTemplate) Initialize(url, dbName string, migrator gomethods.Migrator) error { + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) + if len(urlWithoutScheme) != 2 { + return errors.New("invalid mongodb:// scheme") + } + + session, err := mgo.Dial(url) + if err != nil { + return err + } + session.SetMode(mgo.Monotonic, true) + + driver.Session = session + driver.DbName = dbName + driver.migrator = migrator + + return nil +} + +func (driver *DriverTemplate) Close() error { + if driver.Session != nil { + driver.Session.Close() + } + return nil +} + +func (driver *DriverTemplate) FilenameParser() file.FilenameParser { + return file.UpDownAndBothFilenameParser{FilenameExtension: driver.FilenameExtension()} +} + +func (driver *DriverTemplate) FilenameExtension() string { + return "mgo" +} + + +func (driver *DriverTemplate) Version() (uint64, error) { + var latestMigration DbMigration + c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + + err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) + + switch { + case err == mgo.ErrNotFound: + return 0, nil + case err != nil: + return 0, err + default: + return latestMigration.Version, nil + } +} +func (driver *DriverTemplate) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + err := driver.migrator.Migrate(f, pipe) + if err != nil { + return + } + + migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + if f.Direction == direction.Up { + id := bson.NewObjectId() + dbMigration := DbMigration{Id: id, Version: f.Version} + + err := migrate_c.Insert(dbMigration) + if err != nil { + pipe <- err + return + } + + } else if f.Direction == direction.Down { + err := migrate_c.Remove(bson.M{"version": f.Version}) + if err != nil { + pipe <- err + return + } + } +} diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go new file mode 100644 index 0000000..be0ccb2 --- /dev/null +++ b/driver/gomethods/usage_examples/mongodb.go @@ -0,0 +1,121 @@ +package usage_examples + +import ( + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "gopkg.in/mgo.v2/bson" + "time" +) + +// This boilerplate part is necessary and the same +// regardless of the specific mongodb golang methods driver + +type GoMethodsMongoDbDriver struct { + mongodb.DriverTemplate +} + +func (d *GoMethodsMongoDbDriver) Initialize(url string) error { + return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{Driver: d}) +} + +func init() { + driver.RegisterDriver("mongodb", &GoMethodsMongoDbDriver{mongodb.DriverTemplate{}}) +} + + + +// Here goes the specific mongodb golang methods driver logic + +const DB_NAME = "test" +const SHORT_DATE_LAYOUT = "2000-Jan-01" +const USERS_C = "users" +const ORGANIZATIONS_C = "organizations" + +type Organization struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Location string `bson:"location"` + DateFounded time.Time `bson:"date_founded"` +} + +type User struct { + Id bson.ObjectId `bson:"_id"` + Name string `bson:"name"` +} + +func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + orgs := []Organization{ + {Id: bson.NewObjectId(), Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: bson.NewObjectId(), Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: bson.NewObjectId(), Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + } + + for _, org := range orgs { + err := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) + if err != nil { + return err + } + } + return nil +} + +func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { + return m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() +} + +func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { + users := []User{ + {Id: bson.NewObjectId(), Name: "Alex"}, + {Id: bson.NewObjectId(), Name: "Beatrice"}, + {Id: bson.NewObjectId(), Name: "Cleo"}, + } + + for _, user := range users { + err := m.Session.DB(DB_NAME).C(USERS_C).Insert(user) + if err != nil { + return err + } + } + return nil +} + +func (m *GoMethodsMongoDbDriver) V001_init_users_down() error { + return m.Session.DB(DB_NAME).C(USERS_C).DropCollection() +} + +func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_up() error { + c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) + return err +} + +func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_down() error { + c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) + return err +} + +func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_up() error { + c := m.Session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleo"} + change := bson.M{"$set": bson.M{"name": "Cleopatra"}} + + return c.Update(colQuerier, change) +} + +func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_down() error { + c := m.Session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleopatra"} + change := bson.M{"$set": bson.M{"name": "Cleo",}} + + return c.Update(colQuerier, change) +} \ No newline at end of file diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go new file mode 100644 index 0000000..ec1a85d --- /dev/null +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -0,0 +1,123 @@ +package usage_examples + +import ( + "testing" + + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + + pipep "github.com/dimag-jfrog/migrate/pipe" +) + + + +func TestMigrate(t *testing.T) { + //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") + //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") + host := "127.0.0.1" + port := "27017" + driverUrl := "mongodb://" + host + ":" + port + + d := &GoMethodsMongoDbDriver{} + if err := d.Initialize(driverUrl); err != nil { + t.Fatal(err) + } + + content1 := []byte(` + V001_init_organizations + V001_init_users + `) + content2 := []byte(` + V002_organizations_rename_location_field_to_headquarters + V002_change_user_cleo_to_cleopatra + `) + + files := []file.File{ + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: content1, + }, + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: content1, + }, + { + Path: "/foobar", + FileName: "002_foobar.mgo", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: content2, + }, + { + Path: "/foobar", + FileName: "002_foobar.mgo", + Version: 2, + Name: "foobar", + Direction: direction.Down, + Content: content2, + }, + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_non_existing_operation + `), + }, + } + + var pipe chan interface{} + var errs []error + + pipe = pipep.New() + go d.Migrate(files[0], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + pipe = pipep.New() + go d.Migrate(files[2], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + pipe = pipep.New() + go d.Migrate(files[3], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + //pipe = pipep.New() + //go d.Migrate(files[1], pipe) + //errs = pipep.ReadErrors(pipe) + //if len(errs) > 0 { + // t.Fatal(errs) + //} + + pipe = pipep.New() + go d.Migrate(files[4], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) == 0 { + t.Error("Expected test case to fail") + } + + if err := d.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/file/file.go b/file/file.go index 14617ba..794e951 100644 --- a/file/file.go +++ b/file/file.go @@ -9,20 +9,11 @@ import ( "go/token" "io/ioutil" "path" - "regexp" "sort" "strconv" "strings" ) -var filenameRegex = `^([0-9]+)_(.*)\.(up|down)\.%s$` - -// FilenameRegex builds regular expression stmt with given -// filename extension from driver. -func FilenameRegex(filenameExtension string) *regexp.Regexp { - return regexp.MustCompile(fmt.Sprintf(filenameRegex, filenameExtension)) -} - // File represents one file on disk. // Example: 001_initial_plan_to_do_sth.up.sql type File struct { @@ -149,8 +140,17 @@ func (mf *MigrationFiles) From(version uint64, relativeN int) (Files, error) { return files, nil } + +func ReadMigrationFiles(path, filenameExtension string) (MigrationFiles, error){ + return doReadMigrationFiles(path, DefaultFilenameParser{FilenameExtension: filenameExtension}) +} + +func ReadMigrationFilesWithFilenameParser(path string, filenameParser FilenameParser) (MigrationFiles, error){ + return doReadMigrationFiles(path, filenameParser) +} + // ReadMigrationFiles reads all migration files from a given path -func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files MigrationFiles, err error) { +func doReadMigrationFiles(path string, filenameParser FilenameParser) (files MigrationFiles, err error) { // find all migration files in path ioFiles, err := ioutil.ReadDir(path) if err != nil { @@ -165,7 +165,7 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat tmpFiles := make([]*tmpFile, 0) tmpFileMap := map[uint64]map[direction.Direction]tmpFile{} for _, file := range ioFiles { - version, name, d, err := parseFilenameSchema(file.Name(), filenameRegex) + version, name, d, err := filenameParser.Parse(file.Name()) if err == nil { if _, ok := tmpFileMap[version]; !ok { tmpFileMap[version] = map[direction.Direction]tmpFile{} @@ -210,33 +210,56 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat Direction: direction.Down, } lookFordirection = direction.Up + case direction.Both: + migrationFile.UpFile = &File{ + Path: path, + FileName: file.filename, + Version: file.version, + Name: file.name, + Content: nil, + Direction: direction.Up, + } + migrationFile.DownFile = &File{ + Path: path, + FileName: file.filename, + Version: file.version, + Name: file.name, + Content: nil, + Direction: direction.Down, + } default: return nil, errors.New("Unsupported direction.Direction Type") } for _, file2 := range tmpFiles { - if file2.version == file.version && file2.d == lookFordirection { - switch lookFordirection { - case direction.Up: - migrationFile.UpFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Up, + if file2.version == file.version { + if file.d == direction.Both { + if file.d != file2.d { + return nil, errors.New("Incompatible direction.Direction types") } - case direction.Down: - migrationFile.DownFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Down, + } else if file2.d == lookFordirection { + switch lookFordirection { + case direction.Up: + migrationFile.UpFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Up, + } + case direction.Down: + migrationFile.DownFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Down, + } } + break } - break } } @@ -249,29 +272,6 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat return newFiles, nil } -// parseFilenameSchema parses the filename -func parseFilenameSchema(filename string, filenameRegex *regexp.Regexp) (version uint64, name string, d direction.Direction, err error) { - matches := filenameRegex.FindStringSubmatch(filename) - if len(matches) != 4 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - - if matches[3] == "up" { - d = direction.Up - } else if matches[3] == "down" { - d = direction.Down - } else { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) - } - - return version, matches[2], d, nil -} - // Len is the number of elements in the collection. // Required by Sort Interface{} func (mf MigrationFiles) Len() int { diff --git a/file/file_test.go b/file/file_test.go index c6bd0b3..648f554 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -8,52 +8,6 @@ import ( "testing" ) -func TestParseFilenameSchema(t *testing.T) { - var tests = []struct { - filename string - filenameExtension string - expectVersion uint64 - expectName string - expectDirection direction.Direction - expectErr bool - }{ - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - version, name, migrate, err := parseFilenameSchema(test.filename, FilenameRegex(test.filenameExtension)) - if test.expectErr && err == nil { - t.Fatal("Expected error, but got none.", test) - } - if !test.expectErr && err != nil { - t.Fatal("Did not expect error, but got one:", err, test) - } - if err == nil { - if version != test.expectVersion { - t.Error("Wrong version number", test) - } - if name != test.expectName { - t.Error("wrong name", test) - } - if migrate != test.expectDirection { - t.Error("wrong migrate", test) - } - } - } -} - func TestFiles(t *testing.T) { tmpdir, err := ioutil.TempDir("/tmp", "TestLookForMigrationFilesInSearchPath") if err != nil { @@ -77,7 +31,7 @@ func TestFiles(t *testing.T) { ioutil.WriteFile(path.Join(tmpdir, "401_migrationfile.down.sql"), []byte("test"), 0755) - files, err := ReadMigrationFiles(tmpdir, FilenameRegex("sql")) + files, err := ReadMigrationFiles(tmpdir, "sql") if err != nil { t.Fatal(err) } @@ -223,7 +177,7 @@ func TestDuplicateFiles(t *testing.T) { t.Fatal(err) } - _, err = ReadMigrationFiles(root, FilenameRegex("sql")) + _, err = ReadMigrationFiles(root, "sql") if err == nil { t.Fatal("Expected duplicate migration file error") } diff --git a/file/filename_parser.go b/file/filename_parser.go new file mode 100644 index 0000000..6fb8521 --- /dev/null +++ b/file/filename_parser.go @@ -0,0 +1,85 @@ +package file + +import ( + "errors" + "github.com/dimag-jfrog/migrate/migrate/direction" + "regexp" + "fmt" + "strconv" + "strings" +) + + +type FilenameParser interface { + Parse (filename string) (version uint64, name string, d direction.Direction, err error) +} + + +var defaultFilenameRegexTemplate = `^([0-9]+)_(.*)\.(up|down)\.%s$` + +func parseDefaultFilenameSchema(filename, filenameRegex string) (version uint64, name string, d direction.Direction, err error) { + regexp := regexp.MustCompile(filenameRegex) + matches := regexp.FindStringSubmatch(filename) + if len(matches) != 4 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + + name = matches[2] + + if matches[3] == "up" { + d = direction.Up + } else if matches[3] == "down" { + d = direction.Down + } else { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) + } + + return version, name, d, nil +} + +type DefaultFilenameParser struct { + FilenameExtension string +} + +func (parser DefaultFilenameParser) Parse (filename string) (version uint64, name string, d direction.Direction, err error) { + filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) + return parseDefaultFilenameSchema(filename, filenameRegex) +} + + +type UpDownAndBothFilenameParser struct { + FilenameExtension string +} + +func (parser UpDownAndBothFilenameParser) Parse(filename string) (version uint64, name string, d direction.Direction, err error) { + ext := parser.FilenameExtension + if !strings.HasSuffix(filename, ext) { + return 0, "", 0, errors.New("Filename ") + } + + var matches []string + if strings.HasSuffix(filename, ".up." + ext) || strings.HasSuffix(filename, ".down." + ext) { + filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) + return parseDefaultFilenameSchema(filename, filenameRegex) + } + + regex := regexp.MustCompile(fmt.Sprintf(`^([0-9]+)_(.*)\.%s$`, ext)) + matches = regex.FindStringSubmatch(filename) + if len(matches) != 3 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + name = matches[2] + d = direction.Both + + return version, name, d, nil +} diff --git a/file/filename_parser_test.go b/file/filename_parser_test.go new file mode 100644 index 0000000..ea9135d --- /dev/null +++ b/file/filename_parser_test.go @@ -0,0 +1,85 @@ +package file + +import ( + "github.com/dimag-jfrog/migrate/migrate/direction" + "testing" +) + + +type ParsingTest struct { + filename string + filenameExtension string + expectVersion uint64 + expectName string + expectDirection direction.Direction + expectErr bool +} + +func testParser(t *testing.T, parser FilenameParser, test *ParsingTest) { + version, name, migrate, err := parser.Parse(test.filename) + if test.expectErr && err == nil { + t.Fatal("Expected error, but got none.", test) + } + if !test.expectErr && err != nil { + t.Fatal("Did not expect error, but got one:", err, test) + } + if err == nil { + if version != test.expectVersion { + t.Error("Wrong version number", test) + } + if name != test.expectName { + t.Error("wrong name", test) + } + if migrate != test.expectDirection { + t.Error("wrong migrate", test) + } + } +} + +func TestParseDefaultFilenameSchema(t *testing.T) { + var tests = []ParsingTest { + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + parser := DefaultFilenameParser{FilenameExtension:test.filenameExtension} + testParser(t, &parser, &test) + } +} + +func TestParseUpDownAndBothFilenameSchema(t *testing.T) { + var tests = []ParsingTest { + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 100, "test_file", direction.Both, false}, + {"001_test_file.mgo", "mgo", 1, "test_file", direction.Both, false}, + {"-1_test_file.mgo", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + parser := UpDownAndBothFilenameParser{FilenameExtension:test.filenameExtension} + testParser(t, &parser, &test) + } +} diff --git a/migrate/direction/direction.go b/migrate/direction/direction.go index ed5e5ec..3ebc7b8 100644 --- a/migrate/direction/direction.go +++ b/migrate/direction/direction.go @@ -6,4 +6,5 @@ type Direction int const ( Up Direction = +1 Down = -1 + Both = 0 ) diff --git a/migrate/migrate.go b/migrate/migrate.go index 54cc42b..4e3c53d 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -212,7 +212,7 @@ func Create(url, migrationsPath, name string) (*file.MigrationFile, error) { if err != nil { return nil, err } - files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) + files, err := file.ReadMigrationFiles(migrationsPath, d.FilenameExtension()) if err != nil { return nil, err } @@ -268,7 +268,15 @@ func initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath string) (d if err != nil { return nil, nil, 0, err } - files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) + + var files file.MigrationFiles + + if d1, ok := d.(driver.DriverWithFilenameParser); ok { + files, err = file.ReadMigrationFilesWithFilenameParser(migrationsPath, d1.FilenameParser()) + } else { + files, err = file.ReadMigrationFiles(migrationsPath, d.FilenameExtension()) + } + if err != nil { d.Close() // TODO what happens with errors from this func? return nil, nil, 0, err From 5f7b2caee8518c70b5859f3c48e92e83a5340005 Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 21:07:37 +0300 Subject: [PATCH 03/27] - Reset all changes to the upstream branch - Changed logic not to use custom filename parser: -Supporting up and down files only, no both direction files -Using method names as is - Added complete test to the mongo db migration scenarios --- .gitignore | 11 +- driver/driver.go | 9 - driver/gomethods/gomethods_migrator.go | 35 +-- driver/gomethods/gomethods_migrator_test.go | 95 ++----- driver/gomethods/mongodb/mongodb_template.go | 4 - driver/gomethods/usage_examples/mongodb.go | 31 +- .../gomethods/usage_examples/mongodb_test.go | 268 ++++++++++++------ file/file.go | 106 +++---- file/file_test.go | 50 +++- file/filename_parser.go | 85 ------ file/filename_parser_test.go | 85 ------ migrate/direction/direction.go | 1 - migrate/migrate.go | 12 +- 13 files changed, 361 insertions(+), 431 deletions(-) delete mode 100644 file/filename_parser.go delete mode 100644 file/filename_parser_test.go diff --git a/.gitignore b/.gitignore index 9bd9f47..ee2f41f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,2 @@ .DS_Store -test.db -*.iml -.DS_Store -.idea -*.zip -*.tar.* -*.nuget -*.jar -*.war - +test.db \ No newline at end of file diff --git a/driver/driver.go b/driver/driver.go index a668d52..5c2f6fc 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -34,15 +34,6 @@ type Driver interface { Version() (uint64, error) } -// Driver that has some custom migration file format -// and wants to use a different parsing strategy. -type DriverWithFilenameParser interface { - - Driver - - FilenameParser() file.FilenameParser -} - // New returns Driver and calls Initialize on it func New(url string) (Driver, error) { u, err := neturl.Parse(url) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 68b9ff5..d695342 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -10,7 +10,6 @@ import ( "bufio" "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" ) @@ -32,7 +31,7 @@ func (e *MethodInvocationFailedError) Error() string { type Migrator struct { - Driver driver.DriverWithFilenameParser + Driver driver.Driver RollbackOnFailure bool } @@ -55,6 +54,10 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { // on failure, try to rollback methods in this migration for j := i-1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) + if rollbackToMethodName == "" || !m.IsValid(rollbackToMethodName) { + continue + } + pipe <- rollbackToMethodName err = m.Invoke(rollbackToMethodName) if err != nil { @@ -106,8 +109,10 @@ func reverseInPlace(a []string) { func getRollbackToMethod(methodName string) string { if strings.HasSuffix(methodName, "_up") { return strings.TrimSuffix(methodName, "_up") + "_down" - } else { + } else if strings.HasSuffix(methodName, "_down") { return strings.TrimSuffix(methodName, "_down") + "_up" + } else { + return "" } } @@ -143,34 +148,20 @@ func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { } for _, line := range lines { - operationName := strings.TrimSpace(line) + methodName := strings.TrimSpace(line) - if operationName == "" || strings.HasPrefix(operationName, "--") { + if methodName == "" || strings.HasPrefix(methodName, "--") { // an empty line or a comment, ignore continue } - upMethodName := operationName + "_up" - downMethodName := operationName + "_down" - - if !m.IsValid(upMethodName) { - return nil, MissingMethodError(upMethodName) - } - if !m.IsValid(downMethodName) { - return nil, MissingMethodError(downMethodName) + if !m.IsValid(methodName) { + return nil, MissingMethodError(methodName) } - if f.Direction == direction.Up { - methods = append(methods, upMethodName) - } else { - methods = append(methods, downMethodName) - } + methods = append(methods, methodName) } - _,_,fileType,_ := m.Driver.FilenameParser().Parse(f.FileName) - if fileType == direction.Both && f.Direction == direction.Down { - reverseInPlace(methods) - } return methods, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index 58fd4f2..1a5452f 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -23,10 +23,6 @@ func (driver *FakeGoMethodsDriver) Close() error { return nil } -func (driver *FakeGoMethodsDriver) FilenameParser() file.FilenameParser { - return file.UpDownAndBothFilenameParser{ FilenameExtension: driver.FilenameExtension() } -} - func (driver *FakeGoMethodsDriver) FilenameExtension() string { return "gm" } @@ -84,39 +80,7 @@ func TestMigrate(t *testing.T) { expectRollback bool }{ { - name: "up migration, both directions-file: invokes up methods in order", - file: file.File { - Path: "/foobar", - FileName: "001_foobar.gm", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations - V001_init_users - `), - }, - expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, - expectedErrors: []error{}, - }, - { - name: "down migration, both-directions-file: reverts direction of invoked down methods", - file: file.File { - Path: "/foobar", - FileName: "001_foobar.gm", - Version: 1, - Name: "foobar", - Direction: direction.Down, - Content: []byte(` - V001_init_organizations - V001_init_users - `), - }, - expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, - expectedErrors: []error{}, - }, - { - name: "up migration, up direction-file: invokes up methods in order", + name: "up migration invokes up methods", file: file.File { Path: "/foobar", FileName: "001_foobar.up.gm", @@ -124,15 +88,15 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users + V001_init_organizations_up + V001_init_users_up `), }, expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, expectedErrors: []error{}, }, { - name: "down migration, down directions-file: keeps order of invoked down methods", + name: "down migration invoked down methods", file: file.File { Path: "/foobar", FileName: "001_foobar.down.gm", @@ -140,25 +104,25 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_init_users + V001_init_users_down + V001_init_organizations_down `), }, - expectedInvokedMethods: []string{"V001_init_organizations_down", "V001_init_users_down"}, + expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, expectedErrors: []error{}, }, { name: "up migration: non-existing method causes migration not to execute", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users - V001_some_non_existing_method + V001_init_organizations_up + V001_init_users_up + V001_some_non_existing_method_up `), }, expectedInvokedMethods: []string{}, @@ -168,14 +132,14 @@ func TestMigrate(t *testing.T) { name: "up migration: failing method stops execution", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_organizations_up + V001_some_failing_method_up + V001_init_users_up `), }, expectedInvokedMethods: []string{ @@ -188,17 +152,17 @@ func TestMigrate(t *testing.T) { }}, }, { - name: "down migration, both-directions-file: failing method stops migration", + name: "down migration: failing method stops migration", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_users_down + V001_some_failing_method_down + V001_init_organizations_down `), }, expectedInvokedMethods: []string{ @@ -215,14 +179,14 @@ func TestMigrate(t *testing.T) { expectRollback: true, file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users - V001_some_failing_method + V001_init_organizations_up + V001_init_users_up + V001_some_failing_method_up `), }, expectedInvokedMethods: []string{ @@ -238,18 +202,18 @@ func TestMigrate(t *testing.T) { }}, }, { - name: "down migration, both-directions-file: failing method causes rollback in rollback mode", + name: "down migration: failing method causes rollback in rollback mode", expectRollback: true, file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_users_down + V001_some_failing_method_down + V001_init_organizations_down `), }, expectedInvokedMethods: []string{ @@ -306,6 +270,7 @@ func TestGetRollbackToMethod(t *testing.T) { {"up_down_up", "up_down_down"}, {"down_up", "down_down"}, {"down_down", "down_up"}, + {"some_method", ""}, } for _, c := range cases { diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go index 48817cf..1edbe78 100644 --- a/driver/gomethods/mongodb/mongodb_template.go +++ b/driver/gomethods/mongodb/mongodb_template.go @@ -55,10 +55,6 @@ func (driver *DriverTemplate) Close() error { return nil } -func (driver *DriverTemplate) FilenameParser() file.FilenameParser { - return file.UpDownAndBothFilenameParser{FilenameExtension: driver.FilenameExtension()} -} - func (driver *DriverTemplate) FilenameExtension() string { return "mgo" } diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go index be0ccb2..ea7a680 100644 --- a/driver/gomethods/usage_examples/mongodb.go +++ b/driver/gomethods/usage_examples/mongodb.go @@ -39,20 +39,39 @@ type Organization struct { DateFounded time.Time `bson:"date_founded"` } +type Organization_v2 struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Headquarters string `bson:"headquarters"` + DateFounded time.Time `bson:"date_founded"` +} + type User struct { Id bson.ObjectId `bson:"_id"` Name string `bson:"name"` } +var OrganizationIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +var UserIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") orgs := []Organization{ - {Id: bson.NewObjectId(), Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: bson.NewObjectId(), Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: bson.NewObjectId(), Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, } for _, org := range orgs { @@ -70,9 +89,9 @@ func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { users := []User{ - {Id: bson.NewObjectId(), Name: "Alex"}, - {Id: bson.NewObjectId(), Name: "Beatrice"}, - {Id: bson.NewObjectId(), Name: "Cleo"}, + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, } for _, user := range users { diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go index ec1a85d..033876c 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -7,9 +7,69 @@ import ( "github.com/dimag-jfrog/migrate/migrate/direction" pipep "github.com/dimag-jfrog/migrate/pipe" + "reflect" + "time" + "github.com/dimag-jfrog/migrate/driver/gomethods" ) +type ExpectedMigrationResult struct { + Organizations []Organization + Organizations_v2 []Organization_v2 + Users []User + Errors []error +} + +func RunMigrationAndAssertResult( + t *testing.T, + title string, + d *GoMethodsMongoDbDriver, + file file.File, + expected *ExpectedMigrationResult) { + + actualOrganizations := []Organization{} + actualOrganizations_v2 := []Organization_v2{} + actualUsers := []User{} + var err error + var pipe chan interface{} + var errs []error + + pipe = pipep.New() + go d.Migrate(file, pipe) + errs = pipep.ReadErrors(pipe) + + session := d.Session + if len(expected.Organizations) > 0 { + err = session.DB(DB_NAME).C(ORGANIZATIONS_C).Find(nil).All(&actualOrganizations) + } else { + err = session.DB(DB_NAME).C(ORGANIZATIONS_C).Find(nil).All(&actualOrganizations_v2) + } + if err != nil { + t.Fatal("Failed to query Organizations collection") + } + + err = session.DB(DB_NAME).C(USERS_C).Find(nil).All(&actualUsers) + if err != nil { + t.Fatal("Failed to query Users collection") + } + + if !reflect.DeepEqual(expected.Organizations, actualOrganizations) { + t.Fatalf("Migration '%s': FAILED\nexpected organizations %v\nbut got %v", title, expected.Organizations, actualOrganizations) + } + + if !reflect.DeepEqual(expected.Organizations_v2, actualOrganizations_v2) { + t.Fatalf("Migration '%s': FAILED\nexpected organizations v2 %v\nbut got %v", title, expected.Organizations_v2, actualOrganizations_v2) + } + + if !reflect.DeepEqual(expected.Users, actualUsers) { + t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) + + } + if !reflect.DeepEqual(expected.Errors, errs) { + t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) + } +} + func TestMigrate(t *testing.T) { //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") @@ -23,98 +83,148 @@ func TestMigrate(t *testing.T) { t.Fatal(err) } - content1 := []byte(` - V001_init_organizations - V001_init_users - `) - content2 := []byte(` - V002_organizations_rename_location_field_to_headquarters - V002_change_user_cleo_to_cleopatra - `) + // Reset DB + d.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() + d.Session.DB(DB_NAME).C(USERS_C).DropCollection() - files := []file.File{ + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + migrations := []struct { + name string + file file.File + expectedResult ExpectedMigrationResult + }{ { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: content1, + name: "v0 -> v1", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + }, + Organizations_v2: []Organization_v2{}, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Down, - Content: content1, + name: "v1 -> v2", + file: file.File{ + Path: "/foobar", + FileName: "002_foobar.up.gm", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V002_organizations_rename_location_field_to_headquarters_up + V002_change_user_cleo_to_cleopatra_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{ + {Id: OrganizationIds[0], Name: "Amazon", Headquarters:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Headquarters:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Headquarters:"Santa Clara", DateFounded: date3}, + }, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleopatra"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "002_foobar.mgo", - Version: 2, - Name: "foobar", - Direction: direction.Up, - Content: content2, + name: "v2 -> v1", + file: file.File{ + Path: "/foobar", + FileName: "002_foobar.down.gm", + Version: 2, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V002_change_user_cleo_to_cleopatra_down + V002_organizations_rename_location_field_to_headquarters_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + }, + Organizations_v2: []Organization_v2{}, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "002_foobar.mgo", - Version: 2, - Name: "foobar", - Direction: direction.Down, - Content: content2, + name: "v1 -> v0", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_init_organizations_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations - V001_init_users - V001_non_existing_operation - `), + name: "v0 -> v1: with error", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + v001_non_existing_method_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{ gomethods.MissingMethodError("v001_non_existing_method_up") }, + }, }, } - var pipe chan interface{} - var errs []error - - pipe = pipep.New() - go d.Migrate(files[0], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - pipe = pipep.New() - go d.Migrate(files[2], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - pipe = pipep.New() - go d.Migrate(files[3], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - //pipe = pipep.New() - //go d.Migrate(files[1], pipe) - //errs = pipep.ReadErrors(pipe) - //if len(errs) > 0 { - // t.Fatal(errs) - //} - - pipe = pipep.New() - go d.Migrate(files[4], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) == 0 { - t.Error("Expected test case to fail") + for _, m := range migrations { + RunMigrationAndAssertResult(t, m.name, d, m.file, &m.expectedResult) } if err := d.Close(); err != nil { diff --git a/file/file.go b/file/file.go index 794e951..14617ba 100644 --- a/file/file.go +++ b/file/file.go @@ -9,11 +9,20 @@ import ( "go/token" "io/ioutil" "path" + "regexp" "sort" "strconv" "strings" ) +var filenameRegex = `^([0-9]+)_(.*)\.(up|down)\.%s$` + +// FilenameRegex builds regular expression stmt with given +// filename extension from driver. +func FilenameRegex(filenameExtension string) *regexp.Regexp { + return regexp.MustCompile(fmt.Sprintf(filenameRegex, filenameExtension)) +} + // File represents one file on disk. // Example: 001_initial_plan_to_do_sth.up.sql type File struct { @@ -140,17 +149,8 @@ func (mf *MigrationFiles) From(version uint64, relativeN int) (Files, error) { return files, nil } - -func ReadMigrationFiles(path, filenameExtension string) (MigrationFiles, error){ - return doReadMigrationFiles(path, DefaultFilenameParser{FilenameExtension: filenameExtension}) -} - -func ReadMigrationFilesWithFilenameParser(path string, filenameParser FilenameParser) (MigrationFiles, error){ - return doReadMigrationFiles(path, filenameParser) -} - // ReadMigrationFiles reads all migration files from a given path -func doReadMigrationFiles(path string, filenameParser FilenameParser) (files MigrationFiles, err error) { +func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files MigrationFiles, err error) { // find all migration files in path ioFiles, err := ioutil.ReadDir(path) if err != nil { @@ -165,7 +165,7 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig tmpFiles := make([]*tmpFile, 0) tmpFileMap := map[uint64]map[direction.Direction]tmpFile{} for _, file := range ioFiles { - version, name, d, err := filenameParser.Parse(file.Name()) + version, name, d, err := parseFilenameSchema(file.Name(), filenameRegex) if err == nil { if _, ok := tmpFileMap[version]; !ok { tmpFileMap[version] = map[direction.Direction]tmpFile{} @@ -210,56 +210,33 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig Direction: direction.Down, } lookFordirection = direction.Up - case direction.Both: - migrationFile.UpFile = &File{ - Path: path, - FileName: file.filename, - Version: file.version, - Name: file.name, - Content: nil, - Direction: direction.Up, - } - migrationFile.DownFile = &File{ - Path: path, - FileName: file.filename, - Version: file.version, - Name: file.name, - Content: nil, - Direction: direction.Down, - } default: return nil, errors.New("Unsupported direction.Direction Type") } for _, file2 := range tmpFiles { - if file2.version == file.version { - if file.d == direction.Both { - if file.d != file2.d { - return nil, errors.New("Incompatible direction.Direction types") + if file2.version == file.version && file2.d == lookFordirection { + switch lookFordirection { + case direction.Up: + migrationFile.UpFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Up, } - } else if file2.d == lookFordirection { - switch lookFordirection { - case direction.Up: - migrationFile.UpFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Up, - } - case direction.Down: - migrationFile.DownFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Down, - } + case direction.Down: + migrationFile.DownFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Down, } - break } + break } } @@ -272,6 +249,29 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig return newFiles, nil } +// parseFilenameSchema parses the filename +func parseFilenameSchema(filename string, filenameRegex *regexp.Regexp) (version uint64, name string, d direction.Direction, err error) { + matches := filenameRegex.FindStringSubmatch(filename) + if len(matches) != 4 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + + if matches[3] == "up" { + d = direction.Up + } else if matches[3] == "down" { + d = direction.Down + } else { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) + } + + return version, matches[2], d, nil +} + // Len is the number of elements in the collection. // Required by Sort Interface{} func (mf MigrationFiles) Len() int { diff --git a/file/file_test.go b/file/file_test.go index 648f554..c6bd0b3 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -8,6 +8,52 @@ import ( "testing" ) +func TestParseFilenameSchema(t *testing.T) { + var tests = []struct { + filename string + filenameExtension string + expectVersion uint64 + expectName string + expectDirection direction.Direction + expectErr bool + }{ + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + version, name, migrate, err := parseFilenameSchema(test.filename, FilenameRegex(test.filenameExtension)) + if test.expectErr && err == nil { + t.Fatal("Expected error, but got none.", test) + } + if !test.expectErr && err != nil { + t.Fatal("Did not expect error, but got one:", err, test) + } + if err == nil { + if version != test.expectVersion { + t.Error("Wrong version number", test) + } + if name != test.expectName { + t.Error("wrong name", test) + } + if migrate != test.expectDirection { + t.Error("wrong migrate", test) + } + } + } +} + func TestFiles(t *testing.T) { tmpdir, err := ioutil.TempDir("/tmp", "TestLookForMigrationFilesInSearchPath") if err != nil { @@ -31,7 +77,7 @@ func TestFiles(t *testing.T) { ioutil.WriteFile(path.Join(tmpdir, "401_migrationfile.down.sql"), []byte("test"), 0755) - files, err := ReadMigrationFiles(tmpdir, "sql") + files, err := ReadMigrationFiles(tmpdir, FilenameRegex("sql")) if err != nil { t.Fatal(err) } @@ -177,7 +223,7 @@ func TestDuplicateFiles(t *testing.T) { t.Fatal(err) } - _, err = ReadMigrationFiles(root, "sql") + _, err = ReadMigrationFiles(root, FilenameRegex("sql")) if err == nil { t.Fatal("Expected duplicate migration file error") } diff --git a/file/filename_parser.go b/file/filename_parser.go deleted file mode 100644 index 6fb8521..0000000 --- a/file/filename_parser.go +++ /dev/null @@ -1,85 +0,0 @@ -package file - -import ( - "errors" - "github.com/dimag-jfrog/migrate/migrate/direction" - "regexp" - "fmt" - "strconv" - "strings" -) - - -type FilenameParser interface { - Parse (filename string) (version uint64, name string, d direction.Direction, err error) -} - - -var defaultFilenameRegexTemplate = `^([0-9]+)_(.*)\.(up|down)\.%s$` - -func parseDefaultFilenameSchema(filename, filenameRegex string) (version uint64, name string, d direction.Direction, err error) { - regexp := regexp.MustCompile(filenameRegex) - matches := regexp.FindStringSubmatch(filename) - if len(matches) != 4 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - - name = matches[2] - - if matches[3] == "up" { - d = direction.Up - } else if matches[3] == "down" { - d = direction.Down - } else { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) - } - - return version, name, d, nil -} - -type DefaultFilenameParser struct { - FilenameExtension string -} - -func (parser DefaultFilenameParser) Parse (filename string) (version uint64, name string, d direction.Direction, err error) { - filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) - return parseDefaultFilenameSchema(filename, filenameRegex) -} - - -type UpDownAndBothFilenameParser struct { - FilenameExtension string -} - -func (parser UpDownAndBothFilenameParser) Parse(filename string) (version uint64, name string, d direction.Direction, err error) { - ext := parser.FilenameExtension - if !strings.HasSuffix(filename, ext) { - return 0, "", 0, errors.New("Filename ") - } - - var matches []string - if strings.HasSuffix(filename, ".up." + ext) || strings.HasSuffix(filename, ".down." + ext) { - filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) - return parseDefaultFilenameSchema(filename, filenameRegex) - } - - regex := regexp.MustCompile(fmt.Sprintf(`^([0-9]+)_(.*)\.%s$`, ext)) - matches = regex.FindStringSubmatch(filename) - if len(matches) != 3 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - name = matches[2] - d = direction.Both - - return version, name, d, nil -} diff --git a/file/filename_parser_test.go b/file/filename_parser_test.go deleted file mode 100644 index ea9135d..0000000 --- a/file/filename_parser_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package file - -import ( - "github.com/dimag-jfrog/migrate/migrate/direction" - "testing" -) - - -type ParsingTest struct { - filename string - filenameExtension string - expectVersion uint64 - expectName string - expectDirection direction.Direction - expectErr bool -} - -func testParser(t *testing.T, parser FilenameParser, test *ParsingTest) { - version, name, migrate, err := parser.Parse(test.filename) - if test.expectErr && err == nil { - t.Fatal("Expected error, but got none.", test) - } - if !test.expectErr && err != nil { - t.Fatal("Did not expect error, but got one:", err, test) - } - if err == nil { - if version != test.expectVersion { - t.Error("Wrong version number", test) - } - if name != test.expectName { - t.Error("wrong name", test) - } - if migrate != test.expectDirection { - t.Error("wrong migrate", test) - } - } -} - -func TestParseDefaultFilenameSchema(t *testing.T) { - var tests = []ParsingTest { - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - parser := DefaultFilenameParser{FilenameExtension:test.filenameExtension} - testParser(t, &parser, &test) - } -} - -func TestParseUpDownAndBothFilenameSchema(t *testing.T) { - var tests = []ParsingTest { - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 100, "test_file", direction.Both, false}, - {"001_test_file.mgo", "mgo", 1, "test_file", direction.Both, false}, - {"-1_test_file.mgo", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - parser := UpDownAndBothFilenameParser{FilenameExtension:test.filenameExtension} - testParser(t, &parser, &test) - } -} diff --git a/migrate/direction/direction.go b/migrate/direction/direction.go index 3ebc7b8..ed5e5ec 100644 --- a/migrate/direction/direction.go +++ b/migrate/direction/direction.go @@ -6,5 +6,4 @@ type Direction int const ( Up Direction = +1 Down = -1 - Both = 0 ) diff --git a/migrate/migrate.go b/migrate/migrate.go index 4e3c53d..54cc42b 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -212,7 +212,7 @@ func Create(url, migrationsPath, name string) (*file.MigrationFile, error) { if err != nil { return nil, err } - files, err := file.ReadMigrationFiles(migrationsPath, d.FilenameExtension()) + files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) if err != nil { return nil, err } @@ -268,15 +268,7 @@ func initDriverAndReadMigrationFilesAndGetVersion(url, migrationsPath string) (d if err != nil { return nil, nil, 0, err } - - var files file.MigrationFiles - - if d1, ok := d.(driver.DriverWithFilenameParser); ok { - files, err = file.ReadMigrationFilesWithFilenameParser(migrationsPath, d1.FilenameParser()) - } else { - files, err = file.ReadMigrationFiles(migrationsPath, d.FilenameExtension()) - } - + files, err := file.ReadMigrationFiles(migrationsPath, file.FilenameRegex(d.FilenameExtension())) if err != nil { d.Close() // TODO what happens with errors from this func? return nil, nil, 0, err From c71fea90d66a48ce091b694c02bb0c15b2248582 Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 21:34:52 +0300 Subject: [PATCH 04/27] Relaxed constratint that method receiver has to be of Driver interface. --- driver/gomethods/gomethods_migrator.go | 7 +++---- driver/gomethods/gomethods_migrator_test.go | 2 +- driver/gomethods/usage_examples/mongodb.go | 2 +- driver/gomethods/usage_examples/mongodb_test.go | 1 + 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index d695342..0ff848a 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -8,7 +8,6 @@ import ( "os" "path" "bufio" - "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" ) @@ -31,7 +30,7 @@ func (e *MethodInvocationFailedError) Error() string { type Migrator struct { - Driver driver.Driver + MigrationMethodsReceiver interface{} RollbackOnFailure bool } @@ -73,12 +72,12 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { } func (m *Migrator) IsValid(methodName string) bool { - return reflect.ValueOf(m.Driver).MethodByName(methodName).IsValid() + return reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(methodName).IsValid() } func (m *Migrator) Invoke(methodName string) error { name := methodName - migrateMethod := reflect.ValueOf(m.Driver).MethodByName(name) + migrateMethod := reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { return MissingMethodError(methodName) } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index 1a5452f..a50d6c1 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -232,7 +232,7 @@ func TestMigrate(t *testing.T) { for _, c := range cases { migrator := Migrator{} d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} - migrator.Driver = d + migrator.MigrationMethodsReceiver = d migrator.RollbackOnFailure = c.expectRollback pipe := pipep.New() diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go index ea7a680..e7f30c0 100644 --- a/driver/gomethods/usage_examples/mongodb.go +++ b/driver/gomethods/usage_examples/mongodb.go @@ -16,7 +16,7 @@ type GoMethodsMongoDbDriver struct { } func (d *GoMethodsMongoDbDriver) Initialize(url string) error { - return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{Driver: d}) + return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{MigrationMethodsReceiver: d}) } func init() { diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go index 033876c..a57c51d 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -68,6 +68,7 @@ func RunMigrationAndAssertResult( if !reflect.DeepEqual(expected.Errors, errs) { t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) } + t.Logf("Migration '%s': PASSED", title) } From 570e4b42be4f85e573e605e6885bfa64951a49f2 Mon Sep 17 00:00:00 2001 From: dimag Date: Tue, 9 Aug 2016 16:47:34 +0300 Subject: [PATCH 05/27] - redefined mongo db driver to be autonomous and not depending on specific methods - defined methods receivers registration by name and change the migration files format to include them - added extensive testing --- driver/gomethods/gomethods_migrator.go | 105 ++++++------- driver/gomethods/gomethods_migrator_test.go | 144 +++++++----------- driver/gomethods/gomethods_registry.go | 30 ++++ driver/gomethods/mongodb/mongodb.go | 134 ++++++++++++++++ driver/gomethods/mongodb/mongodb_template.go | 107 ------------- .../mongodb_example/methods_receiver.go | 129 ++++++++++++++++ .../mongodb_test.go | 85 +++++++---- driver/gomethods/usage_examples/mongodb.go | 140 ----------------- 8 files changed, 462 insertions(+), 412 deletions(-) create mode 100644 driver/gomethods/gomethods_registry.go create mode 100644 driver/gomethods/mongodb/mongodb.go delete mode 100644 driver/gomethods/mongodb/mongodb_template.go create mode 100644 driver/gomethods/mongodb_example/methods_receiver.go rename driver/gomethods/{usage_examples => mongodb_example}/mongodb_test.go (71%) delete mode 100644 driver/gomethods/usage_examples/mongodb.go diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 0ff848a..2ee0192 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -2,40 +2,57 @@ package gomethods import ( //"bytes" - "reflect" + "bufio" + "database/sql/driver" "fmt" - "strings" + "github.com/dimag-jfrog/migrate/file" "os" "path" - "bufio" - "github.com/dimag-jfrog/migrate/file" + "strings" ) +type UnregisteredMethodsReceiverError string + +func (e UnregisteredMethodsReceiverError) Error() string { + return "Unregistered methods receiver: " + string(e) +} type MissingMethodError string -func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } +func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } type WrongMethodSignatureError string -func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", e) } + +func (e WrongMethodSignatureError) Error() string { + return fmt.Sprintf("Method %s has wrong signature", e) +} type MethodInvocationFailedError struct { MethodName string - Err error + Err error } func (e *MethodInvocationFailedError) Error() string { return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) } +type MigrationMethodInvoker interface { + IsValid(methodName string, methodReceiver interface{}) bool + Invoke(methodName string, methodReceiver interface{}) error +} + +type GoMethodsDriver interface { + driver.Driver + MigrationMethodInvoker +} type Migrator struct { - MigrationMethodsReceiver interface{} RollbackOnFailure bool + MethodInvoker MigrationMethodInvoker } func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { - methods, err := m.getMigrationMethods(f) + methods, methodsReceiver, err := m.getMigrationMethods(f) if err != nil { pipe <- err return err @@ -43,7 +60,7 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for i, methodName := range methods { pipe <- methodName - err := m.Invoke(methodName) + err := m.MethodInvoker.Invoke(methodName, methodsReceiver) if err != nil { pipe <- err if !m.RollbackOnFailure { @@ -51,14 +68,15 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { } // on failure, try to rollback methods in this migration - for j := i-1; j >= 0; j-- { + for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) - if rollbackToMethodName == "" || !m.IsValid(rollbackToMethodName) { + if rollbackToMethodName == "" || + !m.MethodInvoker.IsValid(rollbackToMethodName, methodsReceiver) { continue } pipe <- rollbackToMethodName - err = m.Invoke(rollbackToMethodName) + err = m.MethodInvoker.Invoke(rollbackToMethodName, methodsReceiver) if err != nil { pipe <- err break @@ -71,33 +89,6 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { return nil } -func (m *Migrator) IsValid(methodName string) bool { - return reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(methodName).IsValid() -} - -func (m *Migrator) Invoke(methodName string) error { - name := methodName - migrateMethod := reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(name) - if !migrateMethod.IsValid() { - return MissingMethodError(methodName) - } - - retValues := migrateMethod.Call([]reflect.Value{}) - if len(retValues) != 1 { - return WrongMethodSignatureError(name) - } - - if !retValues[0].IsNil() { - err, ok := retValues[0].Interface().(error) - if !ok { - return WrongMethodSignatureError(name) - } - return &MethodInvocationFailedError{ MethodName:name, Err:err} - } - - return nil -} - func reverseInPlace(a []string) { for i := 0; i < len(a)/2; i++ { j := len(a) - i - 1 @@ -139,28 +130,40 @@ func getFileLines(file file.File) ([]string, error) { } } -func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { - var lines, methods []string - lines, err := getFileLines(f) +func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsReceiver interface{}, err error) { + var lines []string + + lines, err = getFileLines(f) if err != nil { - return nil, err + return nil, nil, err } for _, line := range lines { - methodName := strings.TrimSpace(line) + line := strings.TrimSpace(line) - if methodName == "" || strings.HasPrefix(methodName, "--") { + if line == "" || strings.HasPrefix(line, "--") { // an empty line or a comment, ignore continue } - if !m.IsValid(methodName) { - return nil, MissingMethodError(methodName) - } + if methodsReceiver == nil { + receiverName := line + methodsReceiver = GetMethodsReceiver(receiverName) + if methodsReceiver == nil { + return nil, nil, UnregisteredMethodsReceiverError(receiverName) + } + continue - methods = append(methods, methodName) + } else { + methodName := line + if !m.MethodInvoker.IsValid(methodName, methodsReceiver) { + return nil, nil, MissingMethodError(methodName) + } + + methods = append(methods, methodName) + } } - return methods, nil + return methods, methodsReceiver, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index a50d6c1..eb33103 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -10,133 +10,105 @@ import ( pipep "github.com/dimag-jfrog/migrate/pipe" ) -type FakeGoMethodsDriver struct { +type FakeGoMethodsInvoker struct { InvokedMethods []string - Migrator Migrator } -func (driver *FakeGoMethodsDriver) Initialize(url string) error { - return nil +func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver interface{}) bool { + if methodName == "V001_some_non_existing_method_up" { + return false + } + + return true } -func (driver *FakeGoMethodsDriver) Close() error { - return nil -} +func (invoker *FakeGoMethodsInvoker) Invoke(methodName string, methodReceiver interface{}) error { + invoker.InvokedMethods = append(invoker.InvokedMethods, methodName) -func (driver *FakeGoMethodsDriver) FilenameExtension() string { - return "gm" -} - -func (driver *FakeGoMethodsDriver) Version() (uint64, error) { - return uint64(0), nil -} - -func (driver *FakeGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { - defer close(pipe) - pipe <- f - return -} - -func (driver *FakeGoMethodsDriver) V001_init_organizations_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_up") - return nil -} - -func (driver *FakeGoMethodsDriver) V001_init_organizations_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_down") - return nil - -} - -func (driver *FakeGoMethodsDriver) V001_init_users_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_up") - return nil -} - -func (driver *FakeGoMethodsDriver) V001_init_users_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_down") - return nil + if methodName == "V001_some_failing_method_up" || methodName == "V001_some_failing_method_down" { + return &MethodInvocationFailedError{ + MethodName: methodName, + Err: SomeError{}, + } + } else { + return nil + } } type SomeError struct{} -func (e SomeError) Error() string { return "Some error happened" } -func (driver *FakeGoMethodsDriver) V001_some_failing_method_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_up") - return SomeError{} -} - -func (driver *FakeGoMethodsDriver) V001_some_failing_method_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_down") - return SomeError{} -} +func (e SomeError) Error() string { return "Some error happened" } func TestMigrate(t *testing.T) { cases := []struct { - name string - file file.File + name string + file file.File expectedInvokedMethods []string - expectedErrors []error - expectRollback bool + expectedErrors []error + expectRollback bool }{ { name: "up migration invokes up methods", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up `), }, expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, - expectedErrors: []error{}, + expectedErrors: []error{}, }, { name: "down migration invoked down methods", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_init_organizations_down `), }, expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, - expectedErrors: []error{}, + expectedErrors: []error{}, }, { name: "up migration: non-existing method causes migration not to execute", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_non_existing_method_up `), }, expectedInvokedMethods: []string{}, - expectedErrors: []error{ MissingMethodError("V001_some_non_existing_method_up") }, + expectedErrors: []error{MissingMethodError("V001_some_non_existing_method_up")}, }, { name: "up migration: failing method stops execution", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_some_failing_method_up V001_init_users_up @@ -146,20 +118,21 @@ func TestMigrate(t *testing.T) { "V001_init_organizations_up", "V001_some_failing_method_up", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_up", - Err: SomeError{}, + Err: SomeError{}, }}, }, { name: "down migration: failing method stops migration", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -169,21 +142,22 @@ func TestMigrate(t *testing.T) { "V001_init_users_down", "V001_some_failing_method_down", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_down", - Err: SomeError{}, + Err: SomeError{}, }}, }, { - name: "up migration: failing method causes rollback in rollback mode", + name: "up migration: failing method causes rollback in rollback mode", expectRollback: true, - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_failing_method_up @@ -196,21 +170,22 @@ func TestMigrate(t *testing.T) { "V001_init_users_down", "V001_init_organizations_down", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_up", - Err: SomeError{}, + Err: SomeError{}, }}, }, { - name: "down migration: failing method causes rollback in rollback mode", + name: "down migration: failing method causes rollback in rollback mode", expectRollback: true, - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -221,18 +196,20 @@ func TestMigrate(t *testing.T) { "V001_some_failing_method_down", "V001_init_users_up", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_down", - Err: SomeError{}, + Err: SomeError{}, }}, }, - } + RegisterMethodsReceiver("FakeMethodsReceiver", "") + for _, c := range cases { migrator := Migrator{} - d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} - migrator.MigrationMethodsReceiver = d + fakeInvoker := &FakeGoMethodsInvoker{InvokedMethods: []string{}} + + migrator.MethodInvoker = fakeInvoker migrator.RollbackOnFailure = c.expectRollback pipe := pipep.New() @@ -243,9 +220,9 @@ func TestMigrate(t *testing.T) { errs := pipep.ReadErrors(pipe) var failed bool - if !reflect.DeepEqual(d.InvokedMethods, c.expectedInvokedMethods) { + if !reflect.DeepEqual(fakeInvoker.InvokedMethods, c.expectedInvokedMethods) { failed = true - t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, d.InvokedMethods) + t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, fakeInvoker.InvokedMethods) } if !reflect.DeepEqual(errs, c.expectedErrors) { failed = true @@ -258,11 +235,9 @@ func TestMigrate(t *testing.T) { } } - - func TestGetRollbackToMethod(t *testing.T) { cases := []struct { - method string + method string expectedRollbackMethod string }{ {"some_method_up", "some_method_down"}, @@ -282,7 +257,7 @@ func TestGetRollbackToMethod(t *testing.T) { } func TestReverseInPlace(t *testing.T) { - methods := []string { + methods := []string{ "method1_down", "method2_down", "method3_down", @@ -290,7 +265,7 @@ func TestReverseInPlace(t *testing.T) { "method5_down", } - expectedReversedMethods := []string { + expectedReversedMethods := []string{ "method5_down", "method4_down", "method3_down", @@ -304,4 +279,3 @@ func TestReverseInPlace(t *testing.T) { t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) } } - diff --git a/driver/gomethods/gomethods_registry.go b/driver/gomethods/gomethods_registry.go new file mode 100644 index 0000000..78ac84c --- /dev/null +++ b/driver/gomethods/gomethods_registry.go @@ -0,0 +1,30 @@ +package gomethods + +import ( + "sync" +) + +var methodsReceiversMu sync.Mutex +var methodsReceivers = make(map[string]interface{}) + +// Registers a methods receiver object so it can be created from its name. Users of gomethods migration drivers should +// call this method to register objects with their migration methods before executing the migration +func RegisterMethodsReceiver(name string, receiver interface{}) { + methodsReceiversMu.Lock() + defer methodsReceiversMu.Unlock() + if receiver == nil { + panic("Go methods: Register receiver object is nil") + } + if _, dup := methodsReceivers[name]; dup { + panic("Go methods: Register called twice for receiver " + name) + } + methodsReceivers[name] = receiver +} + +// Retrieves a registered driver by name +func GetMethodsReceiver(name string) interface{} { + methodsReceiversMu.Lock() + defer methodsReceiversMu.Unlock() + receiver := methodsReceivers[name] + return receiver +} diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go new file mode 100644 index 0000000..0acad17 --- /dev/null +++ b/driver/gomethods/mongodb/mongodb.go @@ -0,0 +1,134 @@ +package mongodb + +import ( + "errors" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "reflect" + "strings" +) + +const MIGRATE_DB = "db_migrations" +const MIGRATE_C = "db_migrations" + +type MongoDbGoMethodsDriver struct { + Session *mgo.Session + DbName string + + migrator gomethods.Migrator +} + +func init() { + driver.RegisterDriver("mongodb", &MongoDbGoMethodsDriver{}) +} + +type DbMigration struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Version uint64 `bson:"version"` +} + +func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) + if len(urlWithoutScheme) != 2 { + return errors.New("invalid mongodb:// scheme") + } + + session, err := mgo.Dial(url) + if err != nil { + return err + } + session.SetMode(mgo.Monotonic, true) + + driver.Session = session + driver.DbName = MIGRATE_DB + driver.migrator = gomethods.Migrator{MethodInvoker: driver} + + return nil +} + +func (driver *MongoDbGoMethodsDriver) Close() error { + if driver.Session != nil { + driver.Session.Close() + } + return nil +} + +func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { + return "mgo" +} + +func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { + var latestMigration DbMigration + c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) + + switch { + case err == mgo.ErrNotFound: + return 0, nil + case err != nil: + return 0, err + default: + return latestMigration.Version, nil + } +} +func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + err := driver.migrator.Migrate(f, pipe) + if err != nil { + return + } + + migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + if f.Direction == direction.Up { + id := bson.NewObjectId() + dbMigration := DbMigration{Id: id, Version: f.Version} + + err := migrate_c.Insert(dbMigration) + if err != nil { + pipe <- err + return + } + + } else if f.Direction == direction.Down { + err := migrate_c.Remove(bson.M{"version": f.Version}) + if err != nil { + pipe <- err + return + } + } +} + +func (driver *MongoDbGoMethodsDriver) IsValid(methodName string, methodsReceiver interface{}) bool { + return reflect.ValueOf(methodsReceiver).MethodByName(methodName).IsValid() +} + +func (driver *MongoDbGoMethodsDriver) Invoke(methodName string, methodsReceiver interface{}) error { + name := methodName + migrateMethod := reflect.ValueOf(methodsReceiver).MethodByName(name) + if !migrateMethod.IsValid() { + return gomethods.MissingMethodError(methodName) + } + + retValues := migrateMethod.Call([]reflect.Value{reflect.ValueOf(driver.Session)}) + if len(retValues) != 1 { + return gomethods.WrongMethodSignatureError(name) + } + + if !retValues[0].IsNil() { + err, ok := retValues[0].Interface().(error) + if !ok { + return gomethods.WrongMethodSignatureError(name) + } + return &gomethods.MethodInvocationFailedError{MethodName: name, Err: err} + } + + return nil +} diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go deleted file mode 100644 index 1edbe78..0000000 --- a/driver/gomethods/mongodb/mongodb_template.go +++ /dev/null @@ -1,107 +0,0 @@ -package mongodb - -import ( - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "errors" - "strings" - "github.com/dimag-jfrog/migrate/migrate/direction" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/driver/gomethods" -) - -const MIGRATE_C = "db_migrations" - - -// This is not a real driver since the Initialize method requires a gomethods.Migrator -// The real driver will contain the DriverTemplate and implement all the custom migration Golang methods -// See example in usage_examples for details -type DriverTemplate struct { - Session *mgo.Session - DbName string - - migrator gomethods.Migrator -} - - -type DbMigration struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Version uint64 `bson:"version"` -} - -func (driver *DriverTemplate) Initialize(url, dbName string, migrator gomethods.Migrator) error { - urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) - if len(urlWithoutScheme) != 2 { - return errors.New("invalid mongodb:// scheme") - } - - session, err := mgo.Dial(url) - if err != nil { - return err - } - session.SetMode(mgo.Monotonic, true) - - driver.Session = session - driver.DbName = dbName - driver.migrator = migrator - - return nil -} - -func (driver *DriverTemplate) Close() error { - if driver.Session != nil { - driver.Session.Close() - } - return nil -} - -func (driver *DriverTemplate) FilenameExtension() string { - return "mgo" -} - - -func (driver *DriverTemplate) Version() (uint64, error) { - var latestMigration DbMigration - c := driver.Session.DB(driver.DbName).C(MIGRATE_C) - - - err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) - - switch { - case err == mgo.ErrNotFound: - return 0, nil - case err != nil: - return 0, err - default: - return latestMigration.Version, nil - } -} -func (driver *DriverTemplate) Migrate(f file.File, pipe chan interface{}) { - defer close(pipe) - pipe <- f - - err := driver.migrator.Migrate(f, pipe) - if err != nil { - return - } - - migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) - - if f.Direction == direction.Up { - id := bson.NewObjectId() - dbMigration := DbMigration{Id: id, Version: f.Version} - - err := migrate_c.Insert(dbMigration) - if err != nil { - pipe <- err - return - } - - } else if f.Direction == direction.Down { - err := migrate_c.Remove(bson.M{"version": f.Version}) - if err != nil { - pipe <- err - return - } - } -} diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go new file mode 100644 index 0000000..24433e9 --- /dev/null +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -0,0 +1,129 @@ +package mongodb_example + +import ( + "github.com/dimag-jfrog/migrate/driver/gomethods" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "time" +) + +type MyMgoMethodsReceiver struct { +} + +func init() { + gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) +} + +// Here goes the specific mongodb golang methods driver logic + +const DB_NAME = "test" +const SHORT_DATE_LAYOUT = "2000-Jan-01" +const USERS_C = "users" +const ORGANIZATIONS_C = "organizations" + +type Organization struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Location string `bson:"location"` + DateFounded time.Time `bson:"date_founded"` +} + +type Organization_v2 struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Headquarters string `bson:"headquarters"` + DateFounded time.Time `bson:"date_founded"` +} + +type User struct { + Id bson.ObjectId `bson:"_id"` + Name string `bson:"name"` +} + +var OrganizationIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +var UserIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) error { + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + orgs := []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, + } + + for _, org := range orgs { + err := session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) + if err != nil { + return err + } + } + return nil +} + +func (r *MyMgoMethodsReceiver) V001_init_organizations_down(session *mgo.Session) error { + return session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() +} + +func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { + users := []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + } + + for _, user := range users { + err := session.DB(DB_NAME).C(USERS_C).Insert(user) + if err != nil { + return err + } + } + return nil +} + +func (r *MyMgoMethodsReceiver) V001_init_users_down(session *mgo.Session) error { + return session.DB(DB_NAME).C(USERS_C).DropCollection() +} + +func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { + c := session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) + return err +} + +func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { + c := session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) + return err +} + +func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { + c := session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleo"} + change := bson.M{"$set": bson.M{"name": "Cleopatra"}} + + return c.Update(colQuerier, change) +} + +func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { + c := session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleopatra"} + change := bson.M{"$set": bson.M{"name": "Cleo"}} + + return c.Update(colQuerier, change) +} diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go similarity index 71% rename from driver/gomethods/usage_examples/mongodb_test.go rename to driver/gomethods/mongodb_example/mongodb_test.go index a57c51d..2ef2797 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -1,4 +1,4 @@ -package usage_examples +package mongodb_example import ( "testing" @@ -6,24 +6,24 @@ import ( "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" pipep "github.com/dimag-jfrog/migrate/pipe" "reflect" "time" - "github.com/dimag-jfrog/migrate/driver/gomethods" ) - type ExpectedMigrationResult struct { - Organizations []Organization + Organizations []Organization Organizations_v2 []Organization_v2 - Users []User - Errors []error + Users []User + Errors []error } func RunMigrationAndAssertResult( t *testing.T, title string, - d *GoMethodsMongoDbDriver, + d *mongodb.MongoDbGoMethodsDriver, file file.File, expected *ExpectedMigrationResult) { @@ -53,6 +53,10 @@ func RunMigrationAndAssertResult( t.Fatal("Failed to query Users collection") } + if !reflect.DeepEqual(expected.Errors, errs) { + t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) + } + if !reflect.DeepEqual(expected.Organizations, actualOrganizations) { t.Fatalf("Migration '%s': FAILED\nexpected organizations %v\nbut got %v", title, expected.Organizations, actualOrganizations) } @@ -65,13 +69,9 @@ func RunMigrationAndAssertResult( t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) } - if !reflect.DeepEqual(expected.Errors, errs) { - t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) - } t.Logf("Migration '%s': PASSED", title) } - func TestMigrate(t *testing.T) { //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") @@ -79,7 +79,9 @@ func TestMigrate(t *testing.T) { port := "27017" driverUrl := "mongodb://" + host + ":" + port - d := &GoMethodsMongoDbDriver{} + //gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) + d := &mongodb.MongoDbGoMethodsDriver{} + if err := d.Initialize(driverUrl); err != nil { t.Fatal(err) } @@ -93,8 +95,8 @@ func TestMigrate(t *testing.T) { date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") migrations := []struct { - name string - file file.File + name string + file file.File expectedResult ExpectedMigrationResult }{ { @@ -106,15 +108,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up `), }, expectedResult: ExpectedMigrationResult{ Organizations: []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, }, Organizations_v2: []Organization_v2{}, Users: []User{ @@ -134,6 +137,7 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V002_organizations_rename_location_field_to_headquarters_up V002_change_user_cleo_to_cleopatra_up `), @@ -141,9 +145,9 @@ func TestMigrate(t *testing.T) { expectedResult: ExpectedMigrationResult{ Organizations: []Organization{}, Organizations_v2: []Organization_v2{ - {Id: OrganizationIds[0], Name: "Amazon", Headquarters:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Headquarters:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Headquarters:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Headquarters: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Headquarters: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Headquarters: "Santa Clara", DateFounded: date3}, }, Users: []User{ {Id: UserIds[0], Name: "Alex"}, @@ -162,15 +166,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` + MyMgoMethodsReceiver V002_change_user_cleo_to_cleopatra_down V002_organizations_rename_location_field_to_headquarters_down `), }, expectedResult: ExpectedMigrationResult{ Organizations: []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, }, Organizations_v2: []Organization_v2{}, Users: []User{ @@ -190,15 +195,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` + MyMgoMethodsReceiver V001_init_users_down V001_init_organizations_down `), }, expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, + Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{}, + Users: []User{}, + Errors: []error{}, }, }, { @@ -210,16 +216,37 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up v001_non_existing_method_up `), }, expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, + Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{ gomethods.MissingMethodError("v001_non_existing_method_up") }, + Users: []User{}, + Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, + }, + }, + { + name: "v0 -> v1: not defined message receiver", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.UnregisteredMethodsReceiverError("V001_init_organizations_up")}, }, }, } diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go deleted file mode 100644 index e7f30c0..0000000 --- a/driver/gomethods/usage_examples/mongodb.go +++ /dev/null @@ -1,140 +0,0 @@ -package usage_examples - -import ( - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" - "gopkg.in/mgo.v2/bson" - "time" -) - -// This boilerplate part is necessary and the same -// regardless of the specific mongodb golang methods driver - -type GoMethodsMongoDbDriver struct { - mongodb.DriverTemplate -} - -func (d *GoMethodsMongoDbDriver) Initialize(url string) error { - return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{MigrationMethodsReceiver: d}) -} - -func init() { - driver.RegisterDriver("mongodb", &GoMethodsMongoDbDriver{mongodb.DriverTemplate{}}) -} - - - -// Here goes the specific mongodb golang methods driver logic - -const DB_NAME = "test" -const SHORT_DATE_LAYOUT = "2000-Jan-01" -const USERS_C = "users" -const ORGANIZATIONS_C = "organizations" - -type Organization struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Name string `bson:"name"` - Location string `bson:"location"` - DateFounded time.Time `bson:"date_founded"` -} - -type Organization_v2 struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Name string `bson:"name"` - Headquarters string `bson:"headquarters"` - DateFounded time.Time `bson:"date_founded"` -} - -type User struct { - Id bson.ObjectId `bson:"_id"` - Name string `bson:"name"` -} - -var OrganizationIds []bson.ObjectId = []bson.ObjectId{ - bson.NewObjectId(), - bson.NewObjectId(), - bson.NewObjectId(), -} - -var UserIds []bson.ObjectId = []bson.ObjectId{ - bson.NewObjectId(), - bson.NewObjectId(), - bson.NewObjectId(), -} - -func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { - date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") - date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") - date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") - - orgs := []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, - } - - for _, org := range orgs { - err := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) - if err != nil { - return err - } - } - return nil -} - -func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { - return m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() -} - -func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { - users := []User{ - {Id: UserIds[0], Name: "Alex"}, - {Id: UserIds[1], Name: "Beatrice"}, - {Id: UserIds[2], Name: "Cleo"}, - } - - for _, user := range users { - err := m.Session.DB(DB_NAME).C(USERS_C).Insert(user) - if err != nil { - return err - } - } - return nil -} - -func (m *GoMethodsMongoDbDriver) V001_init_users_down() error { - return m.Session.DB(DB_NAME).C(USERS_C).DropCollection() -} - -func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_up() error { - c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) - - _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) - return err -} - -func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_down() error { - c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) - - _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) - return err -} - -func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_up() error { - c := m.Session.DB(DB_NAME).C(USERS_C) - - colQuerier := bson.M{"name": "Cleo"} - change := bson.M{"$set": bson.M{"name": "Cleopatra"}} - - return c.Update(colQuerier, change) -} - -func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_down() error { - c := m.Session.DB(DB_NAME).C(USERS_C) - - colQuerier := bson.M{"name": "Cleopatra"} - change := bson.M{"$set": bson.M{"name": "Cleo",}} - - return c.Update(colQuerier, change) -} \ No newline at end of file From 866ff82ffe692054b79b92c21a5b31a24c669e23 Mon Sep 17 00:00:00 2001 From: dimag Date: Tue, 9 Aug 2016 21:16:21 +0300 Subject: [PATCH 06/27] - registering method receiver directly for the driver - currently each driver contains only a single method receiver: - enforcing method receiver pre-registration on go methods driver initialization - Method receiver name can be removed from files format - passing the DbName parameter inside the method receiver for the go methods driver --- driver/gomethods/gomethods_migrator.go | 53 ++++++------------ driver/gomethods/gomethods_migrator_test.go | 13 +---- driver/gomethods/gomethods_registry.go | 39 ++++++++----- driver/gomethods/mongodb/mongodb.go | 55 +++++++++++++++---- .../mongodb_example/methods_receiver.go | 20 +++++-- .../gomethods/mongodb_example/mongodb_test.go | 40 +++++--------- 6 files changed, 117 insertions(+), 103 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 2ee0192..70d57c3 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -1,22 +1,15 @@ package gomethods import ( - //"bytes" "bufio" - "database/sql/driver" "fmt" + "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" "os" "path" "strings" ) -type UnregisteredMethodsReceiverError string - -func (e UnregisteredMethodsReceiverError) Error() string { - return "Unregistered methods receiver: " + string(e) -} - type MissingMethodError string func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } @@ -37,13 +30,16 @@ func (e *MethodInvocationFailedError) Error() string { } type MigrationMethodInvoker interface { - IsValid(methodName string, methodReceiver interface{}) bool - Invoke(methodName string, methodReceiver interface{}) error + IsValid(methodName string) bool + Invoke(methodName string) error } type GoMethodsDriver interface { driver.Driver + MigrationMethodInvoker + MethodsReceiver() interface{} + SetMethodsReceiver(r interface{}) error } type Migrator struct { @@ -52,7 +48,7 @@ type Migrator struct { } func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { - methods, methodsReceiver, err := m.getMigrationMethods(f) + methods, err := m.getMigrationMethods(f) if err != nil { pipe <- err return err @@ -60,7 +56,7 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for i, methodName := range methods { pipe <- methodName - err := m.MethodInvoker.Invoke(methodName, methodsReceiver) + err := m.MethodInvoker.Invoke(methodName) if err != nil { pipe <- err if !m.RollbackOnFailure { @@ -71,12 +67,12 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) if rollbackToMethodName == "" || - !m.MethodInvoker.IsValid(rollbackToMethodName, methodsReceiver) { + !m.MethodInvoker.IsValid(rollbackToMethodName) { continue } pipe <- rollbackToMethodName - err = m.MethodInvoker.Invoke(rollbackToMethodName, methodsReceiver) + err = m.MethodInvoker.Invoke(rollbackToMethodName) if err != nil { pipe <- err break @@ -122,20 +118,17 @@ func getFileLines(file file.File) ([]string, error) { } return lines, nil } else { - //n := bytes.IndexByte(file.Content, 0) - //n := bytes.Index(file.Content, []byte{0}) - //s := string(file.Content[:n]) s := string(file.Content) return strings.Split(s, "\n"), nil } } -func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsReceiver interface{}, err error) { +func (m *Migrator) getMigrationMethods(f file.File) (methods []string, err error) { var lines []string lines, err = getFileLines(f) if err != nil { - return nil, nil, err + return nil, err } for _, line := range lines { @@ -146,24 +139,14 @@ func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsRe continue } - if methodsReceiver == nil { - receiverName := line - methodsReceiver = GetMethodsReceiver(receiverName) - if methodsReceiver == nil { - return nil, nil, UnregisteredMethodsReceiverError(receiverName) - } - continue - - } else { - methodName := line - if !m.MethodInvoker.IsValid(methodName, methodsReceiver) { - return nil, nil, MissingMethodError(methodName) - } - - methods = append(methods, methodName) + methodName := line + if !m.MethodInvoker.IsValid(methodName) { + return nil, MissingMethodError(methodName) } + + methods = append(methods, methodName) } - return methods, methodsReceiver, nil + return methods, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index eb33103..db2a817 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -14,7 +14,7 @@ type FakeGoMethodsInvoker struct { InvokedMethods []string } -func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver interface{}) bool { +func (invoker *FakeGoMethodsInvoker) IsValid(methodName string) bool { if methodName == "V001_some_non_existing_method_up" { return false } @@ -22,7 +22,7 @@ func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver i return true } -func (invoker *FakeGoMethodsInvoker) Invoke(methodName string, methodReceiver interface{}) error { +func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { invoker.InvokedMethods = append(invoker.InvokedMethods, methodName) if methodName == "V001_some_failing_method_up" || methodName == "V001_some_failing_method_down" { @@ -56,7 +56,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up `), @@ -73,7 +72,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_init_organizations_down `), @@ -90,7 +88,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_non_existing_method_up @@ -108,7 +105,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_some_failing_method_up V001_init_users_up @@ -132,7 +128,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -157,7 +152,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_failing_method_up @@ -185,7 +179,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -203,8 +196,6 @@ func TestMigrate(t *testing.T) { }, } - RegisterMethodsReceiver("FakeMethodsReceiver", "") - for _, c := range cases { migrator := Migrator{} fakeInvoker := &FakeGoMethodsInvoker{InvokedMethods: []string{}} diff --git a/driver/gomethods/gomethods_registry.go b/driver/gomethods/gomethods_registry.go index 78ac84c..5ceb128 100644 --- a/driver/gomethods/gomethods_registry.go +++ b/driver/gomethods/gomethods_registry.go @@ -1,30 +1,39 @@ package gomethods import ( + "fmt" + "github.com/dimag-jfrog/migrate/driver" "sync" ) var methodsReceiversMu sync.Mutex -var methodsReceivers = make(map[string]interface{}) -// Registers a methods receiver object so it can be created from its name. Users of gomethods migration drivers should -// call this method to register objects with their migration methods before executing the migration -func RegisterMethodsReceiver(name string, receiver interface{}) { +// Registers a methods receiver for go methods driver +// Users of gomethods migration drivers should call this method +// to register objects with their migration methods before executing the migration +func RegisterMethodsReceiverForDriver(driverName string, receiver interface{}) { methodsReceiversMu.Lock() defer methodsReceiversMu.Unlock() if receiver == nil { panic("Go methods: Register receiver object is nil") } - if _, dup := methodsReceivers[name]; dup { - panic("Go methods: Register called twice for receiver " + name) - } - methodsReceivers[name] = receiver -} -// Retrieves a registered driver by name -func GetMethodsReceiver(name string) interface{} { - methodsReceiversMu.Lock() - defer methodsReceiversMu.Unlock() - receiver := methodsReceivers[name] - return receiver + driver := driver.GetDriver(driverName) + if driver == nil { + panic("Go methods: Trying to register receiver for not registered driver " + driverName) + } + + methodsDriver, ok := driver.(GoMethodsDriver) + if !ok { + panic("Go methods: Trying to register receiver for non go methods driver " + driverName) + } + + if methodsDriver.MethodsReceiver() != nil { + panic("Go methods: Methods receiver already registered for driver " + driverName) + } + + if err := methodsDriver.SetMethodsReceiver(receiver); err != nil { + panic(fmt.Sprintf("Go methods: Failed to set methods receiver for driver %s\nError: %v", + driverName, err)) + } } diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go index 0acad17..acbc256 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/gomethods/mongodb/mongodb.go @@ -12,14 +12,46 @@ import ( "strings" ) -const MIGRATE_DB = "db_migrations" +type UnregisteredMethodsReceiverError string + +func (e UnregisteredMethodsReceiverError) Error() string { + return "Unregistered methods receiver for driver: " + string(e) +} + +type WrongMethodsReceiverTypeError string + +func (e WrongMethodsReceiverTypeError) Error() string { + return "Wrong methods receiver type for driver: " + string(e) +} + const MIGRATE_C = "db_migrations" +const DRIVER_NAME = "gomethods.mongodb" type MongoDbGoMethodsDriver struct { Session *mgo.Session - DbName string - migrator gomethods.Migrator + methodsReceiver MethodsReceiver + migrator gomethods.Migrator +} + +var _ gomethods.GoMethodsDriver = (*MongoDbGoMethodsDriver)(nil) + +type MethodsReceiver interface { + DbName() string +} + +func (d *MongoDbGoMethodsDriver) MethodsReceiver() interface{} { + return d.methodsReceiver +} + +func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { + r1, ok := r.(MethodsReceiver) + if !ok { + return WrongMethodsReceiverTypeError(DRIVER_NAME) + } + + d.methodsReceiver = r1 + return nil } func init() { @@ -32,6 +64,10 @@ type DbMigration struct { } func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { + if driver.methodsReceiver == nil { + return UnregisteredMethodsReceiverError(DRIVER_NAME) + } + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) if len(urlWithoutScheme) != 2 { return errors.New("invalid mongodb:// scheme") @@ -44,7 +80,6 @@ func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { session.SetMode(mgo.Monotonic, true) driver.Session = session - driver.DbName = MIGRATE_DB driver.migrator = gomethods.Migrator{MethodInvoker: driver} return nil @@ -63,7 +98,7 @@ func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { var latestMigration DbMigration - c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) @@ -85,7 +120,7 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} return } - migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + migrate_c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) if f.Direction == direction.Up { id := bson.NewObjectId() @@ -106,13 +141,13 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) IsValid(methodName string, methodsReceiver interface{}) bool { - return reflect.ValueOf(methodsReceiver).MethodByName(methodName).IsValid() +func (driver *MongoDbGoMethodsDriver) IsValid(methodName string) bool { + return reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName).IsValid() } -func (driver *MongoDbGoMethodsDriver) Invoke(methodName string, methodsReceiver interface{}) error { +func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { name := methodName - migrateMethod := reflect.ValueOf(methodsReceiver).MethodByName(name) + migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { return gomethods.MissingMethodError(methodName) } diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go index 24433e9..6749ae7 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -2,6 +2,8 @@ package mongodb_example import ( "github.com/dimag-jfrog/migrate/driver/gomethods" + _ "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" @@ -10,16 +12,24 @@ import ( type MyMgoMethodsReceiver struct { } +func (r *MyMgoMethodsReceiver) DbName() string { + return DB_NAME +} + +var _ mongodb.MethodsReceiver = (*MyMgoMethodsReceiver)(nil) + func init() { - gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) + gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMgoMethodsReceiver{}) } // Here goes the specific mongodb golang methods driver logic -const DB_NAME = "test" -const SHORT_DATE_LAYOUT = "2000-Jan-01" -const USERS_C = "users" -const ORGANIZATIONS_C = "organizations" +const ( + DB_NAME = "test" + SHORT_DATE_LAYOUT = "2000-Jan-01" + USERS_C = "users" + ORGANIZATIONS_C = "organizations" +) type Organization struct { Id bson.ObjectId `bson:"_id,omitempty"` diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go index 2ef2797..fc21c9c 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -6,6 +6,7 @@ import ( "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/driver/gomethods" "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" pipep "github.com/dimag-jfrog/migrate/pipe" @@ -73,20 +74,30 @@ func RunMigrationAndAssertResult( } func TestMigrate(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Test failed on panic: %v", r) + } + }() + //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") host := "127.0.0.1" port := "27017" driverUrl := "mongodb://" + host + ":" + port - //gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) - d := &mongodb.MongoDbGoMethodsDriver{} + d0 := driver.GetDriver("mongodb") + d, ok := d0.(*mongodb.MongoDbGoMethodsDriver) + if !ok { + t.Fatal("MongoDbGoMethodsDriver has not registered") + } if err := d.Initialize(driverUrl); err != nil { t.Fatal(err) } // Reset DB + d.Session.DB(DB_NAME).C(mongodb.MIGRATE_C).DropCollection() d.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() d.Session.DB(DB_NAME).C(USERS_C).DropCollection() @@ -108,7 +119,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up `), @@ -137,7 +147,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V002_organizations_rename_location_field_to_headquarters_up V002_change_user_cleo_to_cleopatra_up `), @@ -166,7 +175,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - MyMgoMethodsReceiver V002_change_user_cleo_to_cleopatra_down V002_organizations_rename_location_field_to_headquarters_down `), @@ -195,7 +203,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - MyMgoMethodsReceiver V001_init_users_down V001_init_organizations_down `), @@ -216,7 +223,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up v001_non_existing_method_up @@ -229,26 +235,6 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, - { - name: "v0 -> v1: not defined message receiver", - file: file.File{ - Path: "/foobar", - FileName: "001_foobar.up.gm", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations_up - V001_init_users_up - `), - }, - expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, - Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{gomethods.UnregisteredMethodsReceiverError("V001_init_organizations_up")}, - }, - }, } for _, m := range migrations { From 834a096bf8877084689440d6d5b433e0c9b2bac1 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 14:13:53 +0300 Subject: [PATCH 07/27] Added explicit migration method signature validation --- driver/gomethods/gomethods_migrator.go | 14 +++--- driver/gomethods/gomethods_migrator_test.go | 9 ++-- driver/gomethods/mongodb/mongodb.go | 15 ++++++- .../mongodb_example/methods_receiver.go | 9 ++++ .../gomethods/mongodb_example/mongodb_test.go | 44 ++++++++++++++++++- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 70d57c3..a690035 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -17,7 +17,7 @@ func (e MissingMethodError) Error() string { return "Non existing migrate method type WrongMethodSignatureError string func (e WrongMethodSignatureError) Error() string { - return fmt.Sprintf("Method %s has wrong signature", e) + return fmt.Sprintf("Method %s has wrong signature", string(e)) } type MethodInvocationFailedError struct { @@ -30,7 +30,7 @@ func (e *MethodInvocationFailedError) Error() string { } type MigrationMethodInvoker interface { - IsValid(methodName string) bool + Validate(methodName string) error Invoke(methodName string) error } @@ -66,8 +66,10 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { // on failure, try to rollback methods in this migration for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) - if rollbackToMethodName == "" || - !m.MethodInvoker.IsValid(rollbackToMethodName) { + if rollbackToMethodName == "" { + continue + } + if err := m.MethodInvoker.Validate(rollbackToMethodName); err != nil { continue } @@ -140,8 +142,8 @@ func (m *Migrator) getMigrationMethods(f file.File) (methods []string, err error } methodName := line - if !m.MethodInvoker.IsValid(methodName) { - return nil, MissingMethodError(methodName) + if err := m.MethodInvoker.Validate(methodName); err != nil { + return nil, err } methods = append(methods, methodName) diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index db2a817..9dafb8a 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -14,12 +14,12 @@ type FakeGoMethodsInvoker struct { InvokedMethods []string } -func (invoker *FakeGoMethodsInvoker) IsValid(methodName string) bool { +func (invoker *FakeGoMethodsInvoker) Validate(methodName string) error { if methodName == "V001_some_non_existing_method_up" { - return false + return MissingMethodError(methodName) } - return true + return nil } func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { @@ -30,9 +30,8 @@ func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { MethodName: methodName, Err: SomeError{}, } - } else { - return nil } + return nil } type SomeError struct{} diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go index acbc256..f07b38b 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/gomethods/mongodb/mongodb.go @@ -141,8 +141,19 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) IsValid(methodName string) bool { - return reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName).IsValid() +func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { + method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) + if !method.IsValid() { + return gomethods.MissingMethodError(methodName) + } + + methodTemplate := func(*mgo.Session) error { return nil } + + if method.Type() != reflect.TypeOf(methodTemplate) { + return gomethods.WrongMethodSignatureError(methodName) + } + + return nil } func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go index 6749ae7..ae96b71 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -137,3 +137,12 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session * return c.Update(colQuerier, change) } + +// Wrong signature methods for testing +func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_up(s string) error { + return nil +} + +func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { + return true, nil +} diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go index fc21c9c..18862af 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -215,7 +215,7 @@ func TestMigrate(t *testing.T) { }, }, { - name: "v0 -> v1: with error", + name: "v0 -> v1: missing method aborts migration", file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", @@ -235,6 +235,48 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, + { + name: "v0 -> v1: wrong signature method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_method_with_wrong_signature_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.WrongMethodSignatureError("V001_method_with_wrong_signature_up")}, + }, + }, + { + name: "v1 -> v0: wrong signature method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_method_with_wrong_signature_down + V001_init_organizations_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.WrongMethodSignatureError("V001_method_with_wrong_signature_down")}, + }, + }, } for _, m := range migrations { From e27b9c2a7b27ddb37b630039af7f9e5978f50ccc Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 17:15:17 +0300 Subject: [PATCH 08/27] - Refactored project structure to be consistent with original project - Added docker support --- docker-compose.yml | 7 +++- .../example}/mongodb_test.go | 20 +++++----- .../example/sample_mongdb_migrator.go} | 37 ++++++++++--------- .../gomethods/gomethods_migrator.go | 7 ---- .../gomethods/gomethods_migrator_test.go | 26 +------------ .../gomethods/gomethods_registry.go | 0 driver/{gomethods => }/mongodb/mongodb.go | 26 ++++++------- 7 files changed, 48 insertions(+), 75 deletions(-) rename driver/{gomethods/mongodb_example => mongodb/example}/mongodb_test.go (95%) rename driver/{gomethods/mongodb_example/methods_receiver.go => mongodb/example/sample_mongdb_migrator.go} (66%) rename driver/{ => mongodb}/gomethods/gomethods_migrator.go (96%) rename driver/{ => mongodb}/gomethods/gomethods_migrator_test.go (92%) rename driver/{ => mongodb}/gomethods/gomethods_registry.go (100%) rename driver/{gomethods => }/mongodb/mongodb.go (80%) diff --git a/docker-compose.yml b/docker-compose.yml index cac3abc..4968409 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ go: &go image: golang - working_dir: /go/src/github.com/mattes/migrate + working_dir: /go/src/github.com/dimag-jfrog/migrate volumes: - $GOPATH:/go go-test: @@ -10,6 +10,7 @@ go-test: - postgres - mysql - cassandra + - mongo go-build: <<: *go command: sh -c 'go get -v && go build -ldflags ''-s'' -o migrater' @@ -21,6 +22,8 @@ mysql: image: mysql environment: MYSQL_DATABASE: migratetest - MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' cassandra: image: cassandra:2.2 +mongo: + image: mongo diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go similarity index 95% rename from driver/gomethods/mongodb_example/mongodb_test.go rename to driver/mongodb/example/mongodb_test.go index 18862af..c8013d4 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -1,4 +1,4 @@ -package mongodb_example +package example import ( "testing" @@ -7,9 +7,10 @@ import ( "github.com/dimag-jfrog/migrate/migrate/direction" "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" pipep "github.com/dimag-jfrog/migrate/pipe" + "os" "reflect" "time" ) @@ -24,7 +25,7 @@ type ExpectedMigrationResult struct { func RunMigrationAndAssertResult( t *testing.T, title string, - d *mongodb.MongoDbGoMethodsDriver, + d *mongodb.Driver, file file.File, expected *ExpectedMigrationResult) { @@ -70,7 +71,7 @@ func RunMigrationAndAssertResult( t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) } - t.Logf("Migration '%s': PASSED", title) + // t.Logf("Migration '%s': PASSED", title) } func TestMigrate(t *testing.T) { @@ -80,14 +81,13 @@ func TestMigrate(t *testing.T) { } }() - //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") - //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") - host := "127.0.0.1" - port := "27017" + host := os.Getenv("MONGO_PORT_27017_TCP_ADDR") + port := os.Getenv("MONGO_PORT_27017_TCP_PORT") + driverUrl := "mongodb://" + host + ":" + port d0 := driver.GetDriver("mongodb") - d, ok := d0.(*mongodb.MongoDbGoMethodsDriver) + d, ok := d0.(*mongodb.Driver) if !ok { t.Fatal("MongoDbGoMethodsDriver has not registered") } diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/mongodb/example/sample_mongdb_migrator.go similarity index 66% rename from driver/gomethods/mongodb_example/methods_receiver.go rename to driver/mongodb/example/sample_mongdb_migrator.go index ae96b71..8750969 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -1,25 +1,26 @@ -package mongodb_example +package example import ( - "github.com/dimag-jfrog/migrate/driver/gomethods" - _ "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + _ "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" + + "github.com/dimag-jfrog/migrate/driver/mongodb" ) -type MyMgoMethodsReceiver struct { +type SampleMongoDbMigrator struct { } -func (r *MyMgoMethodsReceiver) DbName() string { +func (r *SampleMongoDbMigrator) DbName() string { return DB_NAME } -var _ mongodb.MethodsReceiver = (*MyMgoMethodsReceiver)(nil) +var _ mongodb.MethodsReceiver = (*SampleMongoDbMigrator)(nil) func init() { - gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMgoMethodsReceiver{}) + gomethods.RegisterMethodsReceiverForDriver("mongodb", &SampleMongoDbMigrator{}) } // Here goes the specific mongodb golang methods driver logic @@ -62,7 +63,7 @@ var UserIds []bson.ObjectId = []bson.ObjectId{ bson.NewObjectId(), } -func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_organizations_up(session *mgo.Session) error { date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") @@ -82,11 +83,11 @@ func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) return nil } -func (r *MyMgoMethodsReceiver) V001_init_organizations_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_organizations_down(session *mgo.Session) error { return session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() } -func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_users_up(session *mgo.Session) error { users := []User{ {Id: UserIds[0], Name: "Alex"}, {Id: UserIds[1], Name: "Beatrice"}, @@ -102,25 +103,25 @@ func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { return nil } -func (r *MyMgoMethodsReceiver) V001_init_users_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_users_down(session *mgo.Session) error { return session.DB(DB_NAME).C(USERS_C).DropCollection() } -func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { c := session.DB(DB_NAME).C(ORGANIZATIONS_C) _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) return err } -func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { c := session.DB(DB_NAME).C(ORGANIZATIONS_C) _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) return err } -func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { c := session.DB(DB_NAME).C(USERS_C) colQuerier := bson.M{"name": "Cleo"} @@ -129,7 +130,7 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mg return c.Update(colQuerier, change) } -func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { c := session.DB(DB_NAME).C(USERS_C) colQuerier := bson.M{"name": "Cleopatra"} @@ -139,10 +140,10 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session * } // Wrong signature methods for testing -func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_up(s string) error { +func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_up(s string) error { return nil } -func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { +func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { return true, nil } diff --git a/driver/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go similarity index 96% rename from driver/gomethods/gomethods_migrator.go rename to driver/mongodb/gomethods/gomethods_migrator.go index a690035..4f99afe 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -87,13 +87,6 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { return nil } -func reverseInPlace(a []string) { - for i := 0; i < len(a)/2; i++ { - j := len(a) - i - 1 - a[i], a[j] = a[j], a[i] - } -} - func getRollbackToMethod(methodName string) string { if strings.HasSuffix(methodName, "_up") { return strings.TrimSuffix(methodName, "_up") + "_down" diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go similarity index 92% rename from driver/gomethods/gomethods_migrator_test.go rename to driver/mongodb/gomethods/gomethods_migrator_test.go index 9dafb8a..fe6865e 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -220,7 +220,7 @@ func TestMigrate(t *testing.T) { } if !failed { - t.Logf("case '%s': PASSED", c.name) + //t.Logf("case '%s': PASSED", c.name) } } } @@ -245,27 +245,3 @@ func TestGetRollbackToMethod(t *testing.T) { } } } - -func TestReverseInPlace(t *testing.T) { - methods := []string{ - "method1_down", - "method2_down", - "method3_down", - "method4_down", - "method5_down", - } - - expectedReversedMethods := []string{ - "method5_down", - "method4_down", - "method3_down", - "method2_down", - "method1_down", - } - - reverseInPlace(methods) - - if !reflect.DeepEqual(methods, expectedReversedMethods) { - t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) - } -} diff --git a/driver/gomethods/gomethods_registry.go b/driver/mongodb/gomethods/gomethods_registry.go similarity index 100% rename from driver/gomethods/gomethods_registry.go rename to driver/mongodb/gomethods/gomethods_registry.go diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/mongodb/mongodb.go similarity index 80% rename from driver/gomethods/mongodb/mongodb.go rename to driver/mongodb/mongodb.go index f07b38b..ac27872 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -3,7 +3,7 @@ package mongodb import ( "errors" "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" "gopkg.in/mgo.v2" @@ -27,24 +27,24 @@ func (e WrongMethodsReceiverTypeError) Error() string { const MIGRATE_C = "db_migrations" const DRIVER_NAME = "gomethods.mongodb" -type MongoDbGoMethodsDriver struct { +type Driver struct { Session *mgo.Session methodsReceiver MethodsReceiver migrator gomethods.Migrator } -var _ gomethods.GoMethodsDriver = (*MongoDbGoMethodsDriver)(nil) +var _ gomethods.GoMethodsDriver = (*Driver)(nil) type MethodsReceiver interface { DbName() string } -func (d *MongoDbGoMethodsDriver) MethodsReceiver() interface{} { +func (d *Driver) MethodsReceiver() interface{} { return d.methodsReceiver } -func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { +func (d *Driver) SetMethodsReceiver(r interface{}) error { r1, ok := r.(MethodsReceiver) if !ok { return WrongMethodsReceiverTypeError(DRIVER_NAME) @@ -55,7 +55,7 @@ func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { } func init() { - driver.RegisterDriver("mongodb", &MongoDbGoMethodsDriver{}) + driver.RegisterDriver("mongodb", &Driver{}) } type DbMigration struct { @@ -63,7 +63,7 @@ type DbMigration struct { Version uint64 `bson:"version"` } -func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { +func (driver *Driver) Initialize(url string) error { if driver.methodsReceiver == nil { return UnregisteredMethodsReceiverError(DRIVER_NAME) } @@ -85,18 +85,18 @@ func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { return nil } -func (driver *MongoDbGoMethodsDriver) Close() error { +func (driver *Driver) Close() error { if driver.Session != nil { driver.Session.Close() } return nil } -func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { +func (driver *Driver) FilenameExtension() string { return "mgo" } -func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { +func (driver *Driver) Version() (uint64, error) { var latestMigration DbMigration c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) @@ -111,7 +111,7 @@ func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { return latestMigration.Version, nil } } -func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { +func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { defer close(pipe) pipe <- f @@ -141,7 +141,7 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { +func (driver *Driver) Validate(methodName string) error { method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) if !method.IsValid() { return gomethods.MissingMethodError(methodName) @@ -156,7 +156,7 @@ func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { return nil } -func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { +func (driver *Driver) Invoke(methodName string) error { name := methodName migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { From e9ede5506aca6384312712f24c03bdf9c7bef385 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 20:39:41 +0300 Subject: [PATCH 09/27] Added README.md --- driver/mongodb/README.md | 104 ++++++++++++++++++ .../mongodb/example/sample_mongdb_migrator.go | 2 +- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 driver/mongodb/README.md diff --git a/driver/mongodb/README.md b/driver/mongodb/README.md new file mode 100644 index 0000000..fae3ffe --- /dev/null +++ b/driver/mongodb/README.md @@ -0,0 +1,104 @@ +# MongoDB Driver + +* Runs pre-registered Golang methods that receive a single `*mgo.Session` parameter and return `error` on failure. +* Stores migration version details in collection ``db_migrations``. + This collection will be auto-generated. +* Migrations do not run in transactions, there are no built-in transactions in MongoDB. + That means that if a migration fails, it will not be rolled back. +* There is no out-of-the-box support for command-line interface via terminal. + +## Usage in Go + +```go +import "github.com/dimag-jfrog/migrate/migrate" + +// Import your migration methods package so that they are registered and available for the MongoDB driver. +// There is no need to import the MongoDB driver explicitly, as it should already be imported by your migration methods package. +import _ "my_mongo_db_migrator" + +// use synchronous versions of migration functions ... +allErrors, ok := migrate.UpSync("mongodb://host:port", "./path") +if !ok { + fmt.Println("Oh no ...") + // do sth with allErrors slice +} + +// use the asynchronous version of migration functions ... +pipe := migrate.NewPipe() +go migrate.Up(pipe, "mongodb://host:port", "./path") +// pipe is basically just a channel +// write your own channel listener. see writePipe() in main.go as an example. +``` + +## Migration files format + +The migration files should have an ".mgo" extension and contain a list of registered methods names. + +Migration methods should satisfy the following: +* They should be exported (their name should start with a capital letter) +* Their type should be `func (*mgo.Session) error` + +Recommended (but not required) naming conventions for migration methods: +* Prefix with V : for example V001 for version 1. +* Suffix with "_up" or "_down" for up and down migrations correspondingly. + +001_first_release.up.mgo +``` +V001_some_migration_operation_up +V001_some_other_operation_up +... +``` + +001_first_release.down.mgo +``` +V001_some_other_operation_down +V001_some_migration_operation_down +... +``` + + +## Methods registration + +For a detailed example see: [sample_mongodb_migrator.go](https://github.com/dimag-jfrog/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) + +```go +package my_mongo_db_migrator + +import ( + "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "gopkg.in/mgo.v2" +) + +// common boilerplate +type MyMongoDbMigrator struct { +} + +func (r *MyMongoDbMigrator) DbName() string { + return "" +} + +var _ mongodb.MethodsReceiver = (*MyMongoDbMigrator)(nil) + +func init() { + gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMongoDbMigrator{}) +} + + +// Here goes the application-specific migration logic +func (r *MyMongoDbMigrator) V001_some_migration_operation_up(session *mgo.Session) error { + // do something + return nil +} + +func (r *MyMongoDbMigrator) V001_some_migration_operation_down(session *mgo.Session) error { + // revert some_migration_operation_up from above + return nil +} + +``` + +## Authors + +* Demitry Gershovich, https://github.com/dimag-jfrog + diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index 8750969..0831777 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -23,7 +23,7 @@ func init() { gomethods.RegisterMethodsReceiverForDriver("mongodb", &SampleMongoDbMigrator{}) } -// Here goes the specific mongodb golang methods driver logic +// Here goes the specific mongodb golang methods migration logic const ( DB_NAME = "test" From 9bd4c8c83c054c22bc3fa1a5832ff5e55627cdc9 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 21:25:36 +0300 Subject: [PATCH 10/27] Checking non-exported migration method --- driver/mongodb/example/mongodb_test.go | 21 +++++++++++++++++++ .../mongodb/example/sample_mongdb_migrator.go | 4 ++++ .../mongodb/gomethods/gomethods_migrator.go | 6 ++++++ driver/mongodb/mongodb.go | 10 ++++++--- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index c8013d4..33cecb9 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -235,6 +235,27 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, + { + name: "v0 -> v1: not exported method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + v001_not_exported_method_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.MethodNotExportedError("v001_not_exported_method_up")}, + }, + }, { name: "v0 -> v1: wrong signature method aborts migration", file: file.File{ diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index 0831777..f251e5c 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -140,6 +140,10 @@ func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_down(session } // Wrong signature methods for testing +func (r *SampleMongoDbMigrator) v001_not_exported_method_up(session *mgo.Session) error { + return nil +} + func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_up(s string) error { return nil } diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 4f99afe..225f757 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -20,6 +20,12 @@ func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", string(e)) } +type MethodNotExportedError string + +func (e MethodNotExportedError) Error() string { + return fmt.Sprintf("Method %s is not exported", string(e)) +} + type MethodInvocationFailedError struct { MethodName string Err error diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index ac27872..d077e98 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -142,14 +142,18 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { } func (driver *Driver) Validate(methodName string) error { - method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) - if !method.IsValid() { + methodWithReceiver, ok := reflect.TypeOf(driver.methodsReceiver).MethodByName(methodName) + if !ok { return gomethods.MissingMethodError(methodName) } + if methodWithReceiver.PkgPath != "" { + return gomethods.MethodNotExportedError(methodName) + } + methodFunc := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) methodTemplate := func(*mgo.Session) error { return nil } - if method.Type() != reflect.TypeOf(methodTemplate) { + if methodFunc.Type() != reflect.TypeOf(methodTemplate) { return gomethods.WrongMethodSignatureError(methodName) } From 32cb8251102e93df2642e0c752757c9b8d2351a6 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 21:33:36 +0300 Subject: [PATCH 11/27] specifying version of mongo --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4968409..a39cf29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,4 +26,4 @@ mysql: cassandra: image: cassandra:2.2 mongo: - image: mongo + image: mongo:3.2.6 From 8865262a03a38b97b04d85b1e922f7847c133424 Mon Sep 17 00:00:00 2001 From: dimag Date: Sat, 7 Jan 2017 16:08:09 +0200 Subject: [PATCH 12/27] - Fixed test with not-exported method identified as missing: unified the 2 errors into one - Fixed error message formatting - Ensuring uniqueness of the version field (added unique index) --- driver/mongodb/example/mongodb_test.go | 4 ++-- driver/mongodb/gomethods/gomethods_migrator.go | 16 ++++++---------- .../gomethods/gomethods_migrator_test.go | 4 ++-- driver/mongodb/mongodb.go | 17 +++++++++++++---- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index 33cecb9..c4371bf 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -232,7 +232,7 @@ func TestMigrate(t *testing.T) { Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, Users: []User{}, - Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, + Errors: []error{gomethods.MethodNotFoundError("v001_non_existing_method_up")}, }, }, { @@ -253,7 +253,7 @@ func TestMigrate(t *testing.T) { Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, Users: []User{}, - Errors: []error{gomethods.MethodNotExportedError("v001_not_exported_method_up")}, + Errors: []error{gomethods.MethodNotFoundError("v001_not_exported_method_up")}, }, }, { diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 225f757..96fc1d5 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -10,20 +10,16 @@ import ( "strings" ) -type MissingMethodError string +type MethodNotFoundError string -func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } +func (e MethodNotFoundError) Error() string { + return fmt.Sprintf("Method '%s' was not found. It is either not existing or has not been exported (starts with lowercase).", string(e)) +} type WrongMethodSignatureError string func (e WrongMethodSignatureError) Error() string { - return fmt.Sprintf("Method %s has wrong signature", string(e)) -} - -type MethodNotExportedError string - -func (e MethodNotExportedError) Error() string { - return fmt.Sprintf("Method %s is not exported", string(e)) + return fmt.Sprintf("Method '%s' has wrong signature", string(e)) } type MethodInvocationFailedError struct { @@ -32,7 +28,7 @@ type MethodInvocationFailedError struct { } func (e *MethodInvocationFailedError) Error() string { - return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) + return fmt.Sprintf("Method '%s' returned an error: %v", e.MethodName, e.Err) } type MigrationMethodInvoker interface { diff --git a/driver/mongodb/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go index fe6865e..298f803 100644 --- a/driver/mongodb/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -16,7 +16,7 @@ type FakeGoMethodsInvoker struct { func (invoker *FakeGoMethodsInvoker) Validate(methodName string) error { if methodName == "V001_some_non_existing_method_up" { - return MissingMethodError(methodName) + return MethodNotFoundError(methodName) } return nil @@ -93,7 +93,7 @@ func TestMigrate(t *testing.T) { `), }, expectedInvokedMethods: []string{}, - expectedErrors: []error{MissingMethodError("V001_some_non_existing_method_up")}, + expectedErrors: []error{MethodNotFoundError("V001_some_non_existing_method_up")}, }, { name: "up migration: failing method stops execution", diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index d077e98..026e686 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -59,7 +59,7 @@ func init() { } type DbMigration struct { - Id bson.ObjectId `bson:"_id,omitempty"` + Id bson.ObjectId `bson:"_id"` Version uint64 `bson:"version"` } @@ -79,6 +79,15 @@ func (driver *Driver) Initialize(url string) error { } session.SetMode(mgo.Monotonic, true) + c := session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) + err = c.EnsureIndex(mgo.Index{ + Key: []string{"version"}, + Unique: true, + }) + if err != nil { + return err + } + driver.Session = session driver.migrator = gomethods.Migrator{MethodInvoker: driver} @@ -144,10 +153,10 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { func (driver *Driver) Validate(methodName string) error { methodWithReceiver, ok := reflect.TypeOf(driver.methodsReceiver).MethodByName(methodName) if !ok { - return gomethods.MissingMethodError(methodName) + return gomethods.MethodNotFoundError(methodName) } if methodWithReceiver.PkgPath != "" { - return gomethods.MethodNotExportedError(methodName) + return gomethods.MethodNotFoundError(methodName) } methodFunc := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) @@ -164,7 +173,7 @@ func (driver *Driver) Invoke(methodName string) error { name := methodName migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { - return gomethods.MissingMethodError(methodName) + return gomethods.MethodNotFoundError(methodName) } retValues := migrateMethod.Call([]reflect.Value{reflect.ValueOf(driver.Session)}) From dec52a778fdf5cca204dd4d1b7c4446d54b903b3 Mon Sep 17 00:00:00 2001 From: dimag Date: Sat, 7 Jan 2017 16:12:48 +0200 Subject: [PATCH 13/27] - Using double quotes for parameter in docker-compose - to be more aligned with master repository --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a39cf29..80406d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ mysql: image: mysql environment: MYSQL_DATABASE: migratetest - MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" cassandra: image: cassandra:2.2 mongo: From 34a3e58ab9f18f3d978fac52573bc27d3642aace Mon Sep 17 00:00:00 2001 From: dimag Date: Sat, 7 Jan 2017 16:16:00 +0200 Subject: [PATCH 14/27] - Reverted repository from dimag-jfrog to mattes to prepare for pull request --- docker-compose.yml | 2 +- driver/bash/bash.go | 4 ++-- driver/cassandra/cassandra.go | 6 +++--- driver/cassandra/cassandra_test.go | 6 +++--- driver/driver.go | 2 +- driver/mongodb/README.md | 8 ++++---- driver/mongodb/example/mongodb_test.go | 12 +++++------ .../mongodb/example/sample_mongdb_migrator.go | 6 +++--- .../mongodb/gomethods/gomethods_migrator.go | 4 ++-- .../gomethods/gomethods_migrator_test.go | 6 +++--- .../mongodb/gomethods/gomethods_registry.go | 2 +- driver/mongodb/mongodb.go | 8 ++++---- driver/mysql/mysql.go | 6 +++--- driver/mysql/mysql_test.go | 6 +++--- driver/postgres/postgres.go | 6 +++--- driver/postgres/postgres_test.go | 6 +++--- driver/sqlite3/sqlite3.go | 6 +++--- driver/sqlite3/sqlite3_test.go | 6 +++--- file/file.go | 2 +- file/file_test.go | 2 +- main.go | 20 +++++++++---------- migrate/migrate.go | 8 ++++---- migrate/migrate_test.go | 4 ++-- 23 files changed, 69 insertions(+), 69 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 80406d6..1c87174 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ go: &go image: golang - working_dir: /go/src/github.com/dimag-jfrog/migrate + working_dir: /go/src/github.com/mattes/migrate volumes: - $GOPATH:/go go-test: diff --git a/driver/bash/bash.go b/driver/bash/bash.go index 97ba91f..031f9bb 100644 --- a/driver/bash/bash.go +++ b/driver/bash/bash.go @@ -2,8 +2,8 @@ package bash import ( - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" ) type Driver struct { diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index ffc76ad..e242de2 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -9,9 +9,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index 7d36389..ea73794 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -7,9 +7,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) func TestMigrate(t *testing.T) { diff --git a/driver/driver.go b/driver/driver.go index 5c2f6fc..e4ecb78 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -5,7 +5,7 @@ import ( "fmt" neturl "net/url" // alias to allow `url string` func signature in New - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/file" ) // Driver is the interface type that needs to implemented by all drivers. diff --git a/driver/mongodb/README.md b/driver/mongodb/README.md index fae3ffe..d7d8351 100644 --- a/driver/mongodb/README.md +++ b/driver/mongodb/README.md @@ -10,7 +10,7 @@ ## Usage in Go ```go -import "github.com/dimag-jfrog/migrate/migrate" +import "github.com/mattes/migrate/migrate" // Import your migration methods package so that they are registered and available for the MongoDB driver. // There is no need to import the MongoDB driver explicitly, as it should already be imported by your migration methods package. @@ -59,14 +59,14 @@ V001_some_migration_operation_down ## Methods registration -For a detailed example see: [sample_mongodb_migrator.go](https://github.com/dimag-jfrog/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) +For a detailed example see: [sample_mongodb_migrator.go](https://github.com/mattes/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) ```go package my_mongo_db_migrator import ( - "github.com/dimag-jfrog/migrate/driver/mongodb" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" ) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index c4371bf..86c37c0 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -3,13 +3,13 @@ package example import ( "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/mongodb" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb/gomethods" + pipep "github.com/mattes/migrate/pipe" "os" "reflect" "time" diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index f251e5c..1ab1440 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -1,13 +1,13 @@ package example import ( - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - _ "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/driver/mongodb/gomethods" + _ "github.com/mattes/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb" ) type SampleMongoDbMigrator struct { diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 96fc1d5..97057ab 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -3,8 +3,8 @@ package gomethods import ( "bufio" "fmt" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" "os" "path" "strings" diff --git a/driver/mongodb/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go index 298f803..d94506e 100644 --- a/driver/mongodb/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -4,10 +4,10 @@ import ( "reflect" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + pipep "github.com/mattes/migrate/pipe" ) type FakeGoMethodsInvoker struct { diff --git a/driver/mongodb/gomethods/gomethods_registry.go b/driver/mongodb/gomethods/gomethods_registry.go index 5ceb128..418256f 100644 --- a/driver/mongodb/gomethods/gomethods_registry.go +++ b/driver/mongodb/gomethods/gomethods_registry.go @@ -2,7 +2,7 @@ package gomethods import ( "fmt" - "github.com/dimag-jfrog/migrate/driver" + "github.com/mattes/migrate/driver" "sync" ) diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index 026e686..fcfae50 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -2,10 +2,10 @@ package mongodb import ( "errors" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "reflect" diff --git a/driver/mysql/mysql.go b/driver/mysql/mysql.go index cf186f8..2bbbc13 100644 --- a/driver/mysql/mysql.go +++ b/driver/mysql/mysql.go @@ -12,9 +12,9 @@ import ( "strings" "github.com/go-sql-driver/mysql" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/mysql/mysql_test.go b/driver/mysql/mysql_test.go index 9e08af9..51cf552 100644 --- a/driver/mysql/mysql_test.go +++ b/driver/mysql/mysql_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/postgres/postgres.go b/driver/postgres/postgres.go index dbbe4a2..502c7d1 100644 --- a/driver/postgres/postgres.go +++ b/driver/postgres/postgres.go @@ -8,9 +8,9 @@ import ( "strconv" "github.com/lib/pq" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go index 552f23d..ec91b35 100644 --- a/driver/postgres/postgres_test.go +++ b/driver/postgres/postgres_test.go @@ -5,9 +5,9 @@ import ( "os" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/sqlite3/sqlite3.go b/driver/sqlite3/sqlite3.go index 65e6a2a..a92b14d 100644 --- a/driver/sqlite3/sqlite3.go +++ b/driver/sqlite3/sqlite3.go @@ -7,9 +7,9 @@ import ( "fmt" "strings" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" "github.com/mattn/go-sqlite3" ) diff --git a/driver/sqlite3/sqlite3_test.go b/driver/sqlite3/sqlite3_test.go index 29a501f..8d51a7d 100644 --- a/driver/sqlite3/sqlite3_test.go +++ b/driver/sqlite3/sqlite3_test.go @@ -4,9 +4,9 @@ import ( "database/sql" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate() diff --git a/file/file.go b/file/file.go index 14617ba..54992a4 100644 --- a/file/file.go +++ b/file/file.go @@ -5,7 +5,7 @@ import ( "bytes" "errors" "fmt" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/migrate/direction" "go/token" "io/ioutil" "path" diff --git a/file/file_test.go b/file/file_test.go index c6bd0b3..b9ddeab 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -1,7 +1,7 @@ package file import ( - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/migrate/direction" "io/ioutil" "os" "path" diff --git a/main.go b/main.go index 51d68a8..b6da438 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ // Package main is the CLI. // You can use the CLI via Terminal. -// import "github.com/dimag-jfrog/migrate/migrate" for usage within Go. +// import "github.com/mattes/migrate/migrate" for usage within Go. package main import ( @@ -11,15 +11,15 @@ import ( "time" "github.com/fatih/color" - _ "github.com/dimag-jfrog/migrate/driver/bash" - _ "github.com/dimag-jfrog/migrate/driver/cassandra" - _ "github.com/dimag-jfrog/migrate/driver/mysql" - _ "github.com/dimag-jfrog/migrate/driver/postgres" - _ "github.com/dimag-jfrog/migrate/driver/sqlite3" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + _ "github.com/mattes/migrate/driver/bash" + _ "github.com/mattes/migrate/driver/cassandra" + _ "github.com/mattes/migrate/driver/mysql" + _ "github.com/mattes/migrate/driver/postgres" + _ "github.com/mattes/migrate/driver/sqlite3" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) var url = flag.String("url", os.Getenv("MIGRATE_URL"), "") diff --git a/migrate/migrate.go b/migrate/migrate.go index 54cc42b..b452aee 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -11,10 +11,10 @@ import ( "strconv" "strings" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // Up applies all available migrations diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 7909826..7d2a622 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -6,8 +6,8 @@ import ( "testing" // Ensure imports for each driver we wish to test - _ "github.com/dimag-jfrog/migrate/driver/postgres" - _ "github.com/dimag-jfrog/migrate/driver/sqlite3" + _ "github.com/mattes/migrate/driver/postgres" + _ "github.com/mattes/migrate/driver/sqlite3" ) // Add Driver URLs here to test basic Up, Down, .. functions. From 175550643cfb8b35c8784452bf21a93a5b0fce85 Mon Sep 17 00:00:00 2001 From: dimag Date: Thu, 4 Aug 2016 15:52:31 +0300 Subject: [PATCH 15/27] changed import paths to point to dimag-jfrog --- driver/bash/bash.go | 4 ++-- driver/cassandra/cassandra.go | 6 +++--- driver/cassandra/cassandra_test.go | 6 +++--- driver/driver.go | 2 +- driver/mysql/mysql.go | 6 +++--- driver/mysql/mysql_test.go | 6 +++--- driver/postgres/postgres.go | 6 +++--- driver/postgres/postgres_test.go | 6 +++--- driver/sqlite3/sqlite3.go | 6 +++--- driver/sqlite3/sqlite3_test.go | 6 +++--- file/file.go | 2 +- file/file_test.go | 2 +- main.go | 2 +- migrate/migrate.go | 8 ++++---- 14 files changed, 34 insertions(+), 34 deletions(-) diff --git a/driver/bash/bash.go b/driver/bash/bash.go index 031f9bb..97ba91f 100644 --- a/driver/bash/bash.go +++ b/driver/bash/bash.go @@ -2,8 +2,8 @@ package bash import ( - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" ) type Driver struct { diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index eb66174..cef8ac5 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -9,9 +9,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index 202d5e7..bc024c3 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -7,9 +7,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) func TestMigrate(t *testing.T) { diff --git a/driver/driver.go b/driver/driver.go index e4ecb78..5c2f6fc 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -5,7 +5,7 @@ import ( "fmt" neturl "net/url" // alias to allow `url string` func signature in New - "github.com/mattes/migrate/file" + "github.com/dimag-jfrog/migrate/file" ) // Driver is the interface type that needs to implemented by all drivers. diff --git a/driver/mysql/mysql.go b/driver/mysql/mysql.go index 2bbbc13..cf186f8 100644 --- a/driver/mysql/mysql.go +++ b/driver/mysql/mysql.go @@ -12,9 +12,9 @@ import ( "strings" "github.com/go-sql-driver/mysql" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/mysql/mysql_test.go b/driver/mysql/mysql_test.go index ec19271..511b1b1 100644 --- a/driver/mysql/mysql_test.go +++ b/driver/mysql/mysql_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/postgres/postgres.go b/driver/postgres/postgres.go index 951b005..f5d307b 100644 --- a/driver/postgres/postgres.go +++ b/driver/postgres/postgres.go @@ -8,9 +8,9 @@ import ( "strconv" "github.com/lib/pq" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go index 7c5c0c3..7b2c665 100644 --- a/driver/postgres/postgres_test.go +++ b/driver/postgres/postgres_test.go @@ -5,9 +5,9 @@ import ( "os" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/sqlite3/sqlite3.go b/driver/sqlite3/sqlite3.go index a92b14d..65e6a2a 100644 --- a/driver/sqlite3/sqlite3.go +++ b/driver/sqlite3/sqlite3.go @@ -7,9 +7,9 @@ import ( "fmt" "strings" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" "github.com/mattn/go-sqlite3" ) diff --git a/driver/sqlite3/sqlite3_test.go b/driver/sqlite3/sqlite3_test.go index 7f50095..6abcf0d 100644 --- a/driver/sqlite3/sqlite3_test.go +++ b/driver/sqlite3/sqlite3_test.go @@ -4,9 +4,9 @@ import ( "database/sql" "testing" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate() diff --git a/file/file.go b/file/file.go index 54992a4..14617ba 100644 --- a/file/file.go +++ b/file/file.go @@ -5,7 +5,7 @@ import ( "bytes" "errors" "fmt" - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/migrate/direction" "go/token" "io/ioutil" "path" diff --git a/file/file_test.go b/file/file_test.go index b9ddeab..c6bd0b3 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -1,7 +1,7 @@ package file import ( - "github.com/mattes/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/migrate/direction" "io/ioutil" "os" "path" diff --git a/main.go b/main.go index ac842c7..802bd9d 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ // Package main is the CLI. // You can use the CLI via Terminal. -// import "github.com/mattes/migrate/migrate" for usage within Go. +// import "github.com/dimag-jfrog/migrate/migrate" for usage within Go. package main import ( diff --git a/migrate/migrate.go b/migrate/migrate.go index 0c976be..c5696b5 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -12,10 +12,10 @@ import ( "strings" "time" - "github.com/mattes/migrate/driver" - "github.com/mattes/migrate/file" - "github.com/mattes/migrate/migrate/direction" - pipep "github.com/mattes/migrate/pipe" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + pipep "github.com/dimag-jfrog/migrate/pipe" ) // Up applies all available migrations From 1838852d4d0fed1b9615555c06477c0e0dd97c58 Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 16:29:25 +0300 Subject: [PATCH 16/27] Changes that interfere with the Migrate open source code: - Added go methods migrator, mongo db template: different from the usual driver model. - Added support for bidirectional files (for go methods), appending _up or _down upon context - Added DriverWithFilnameParser for providing custom filename parser functionality that knows to parse bi-directional file names. --- .gitignore | 11 +- driver/driver.go | 9 + driver/gomethods/gomethods_migrator.go | 176 +++++++++ driver/gomethods/gomethods_migrator_test.go | 342 ++++++++++++++++++ driver/gomethods/mongodb/mongodb_template.go | 111 ++++++ driver/gomethods/usage_examples/mongodb.go | 121 +++++++ .../gomethods/usage_examples/mongodb_test.go | 123 +++++++ file/file.go | 106 +++--- file/file_test.go | 50 +-- file/filename_parser.go | 85 +++++ file/filename_parser_test.go | 85 +++++ migrate/direction/direction.go | 1 + 12 files changed, 1118 insertions(+), 102 deletions(-) create mode 100644 driver/gomethods/gomethods_migrator.go create mode 100644 driver/gomethods/gomethods_migrator_test.go create mode 100644 driver/gomethods/mongodb/mongodb_template.go create mode 100644 driver/gomethods/usage_examples/mongodb.go create mode 100644 driver/gomethods/usage_examples/mongodb_test.go create mode 100644 file/filename_parser.go create mode 100644 file/filename_parser_test.go diff --git a/.gitignore b/.gitignore index ee2f41f..9bd9f47 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,11 @@ .DS_Store -test.db \ No newline at end of file +test.db +*.iml +.DS_Store +.idea +*.zip +*.tar.* +*.nuget +*.jar +*.war + diff --git a/driver/driver.go b/driver/driver.go index 5c2f6fc..a668d52 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -34,6 +34,15 @@ type Driver interface { Version() (uint64, error) } +// Driver that has some custom migration file format +// and wants to use a different parsing strategy. +type DriverWithFilenameParser interface { + + Driver + + FilenameParser() file.FilenameParser +} + // New returns Driver and calls Initialize on it func New(url string) (Driver, error) { u, err := neturl.Parse(url) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go new file mode 100644 index 0000000..68b9ff5 --- /dev/null +++ b/driver/gomethods/gomethods_migrator.go @@ -0,0 +1,176 @@ +package gomethods + +import ( + //"bytes" + "reflect" + "fmt" + "strings" + "os" + "path" + "bufio" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" +) + + +type MissingMethodError string +func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } + + +type WrongMethodSignatureError string +func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", e) } + +type MethodInvocationFailedError struct { + MethodName string + Err error +} + +func (e *MethodInvocationFailedError) Error() string { + return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) +} + + +type Migrator struct { + Driver driver.DriverWithFilenameParser + RollbackOnFailure bool +} + +func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { + methods, err := m.getMigrationMethods(f) + if err != nil { + pipe <- err + return err + } + + for i, methodName := range methods { + pipe <- methodName + err := m.Invoke(methodName) + if err != nil { + pipe <- err + if !m.RollbackOnFailure { + return err + } + + // on failure, try to rollback methods in this migration + for j := i-1; j >= 0; j-- { + rollbackToMethodName := getRollbackToMethod(methods[j]) + pipe <- rollbackToMethodName + err = m.Invoke(rollbackToMethodName) + if err != nil { + pipe <- err + break + } + } + return err + } + } + + return nil +} + +func (m *Migrator) IsValid(methodName string) bool { + return reflect.ValueOf(m.Driver).MethodByName(methodName).IsValid() +} + +func (m *Migrator) Invoke(methodName string) error { + name := methodName + migrateMethod := reflect.ValueOf(m.Driver).MethodByName(name) + if !migrateMethod.IsValid() { + return MissingMethodError(methodName) + } + + retValues := migrateMethod.Call([]reflect.Value{}) + if len(retValues) != 1 { + return WrongMethodSignatureError(name) + } + + if !retValues[0].IsNil() { + err, ok := retValues[0].Interface().(error) + if !ok { + return WrongMethodSignatureError(name) + } + return &MethodInvocationFailedError{ MethodName:name, Err:err} + } + + return nil +} + +func reverseInPlace(a []string) { + for i := 0; i < len(a)/2; i++ { + j := len(a) - i - 1 + a[i], a[j] = a[j], a[i] + } +} + +func getRollbackToMethod(methodName string) string { + if strings.HasSuffix(methodName, "_up") { + return strings.TrimSuffix(methodName, "_up") + "_down" + } else { + return strings.TrimSuffix(methodName, "_down") + "_up" + } +} + +func getFileLines(file file.File) ([]string, error) { + if len(file.Content) == 0 { + lines := make([]string, 0) + file, err := os.Open(path.Join(file.Path, file.FileName)) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, nil + } else { + //n := bytes.IndexByte(file.Content, 0) + //n := bytes.Index(file.Content, []byte{0}) + //s := string(file.Content[:n]) + s := string(file.Content) + return strings.Split(s, "\n"), nil + } +} + +func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { + var lines, methods []string + lines, err := getFileLines(f) + if err != nil { + return nil, err + } + + for _, line := range lines { + operationName := strings.TrimSpace(line) + + if operationName == "" || strings.HasPrefix(operationName, "--") { + // an empty line or a comment, ignore + continue + } + + upMethodName := operationName + "_up" + downMethodName := operationName + "_down" + + if !m.IsValid(upMethodName) { + return nil, MissingMethodError(upMethodName) + } + if !m.IsValid(downMethodName) { + return nil, MissingMethodError(downMethodName) + } + + if f.Direction == direction.Up { + methods = append(methods, upMethodName) + } else { + methods = append(methods, downMethodName) + } + } + + _,_,fileType,_ := m.Driver.FilenameParser().Parse(f.FileName) + if fileType == direction.Both && f.Direction == direction.Down { + reverseInPlace(methods) + } + return methods, nil + +} diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go new file mode 100644 index 0000000..58fd4f2 --- /dev/null +++ b/driver/gomethods/gomethods_migrator_test.go @@ -0,0 +1,342 @@ +package gomethods + +import ( + "reflect" + "testing" + + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + + pipep "github.com/dimag-jfrog/migrate/pipe" +) + +type FakeGoMethodsDriver struct { + InvokedMethods []string + Migrator Migrator +} + +func (driver *FakeGoMethodsDriver) Initialize(url string) error { + return nil +} + +func (driver *FakeGoMethodsDriver) Close() error { + return nil +} + +func (driver *FakeGoMethodsDriver) FilenameParser() file.FilenameParser { + return file.UpDownAndBothFilenameParser{ FilenameExtension: driver.FilenameExtension() } +} + +func (driver *FakeGoMethodsDriver) FilenameExtension() string { + return "gm" +} + +func (driver *FakeGoMethodsDriver) Version() (uint64, error) { + return uint64(0), nil +} + +func (driver *FakeGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + return +} + +func (driver *FakeGoMethodsDriver) V001_init_organizations_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_up") + return nil +} + +func (driver *FakeGoMethodsDriver) V001_init_organizations_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_down") + return nil + +} + +func (driver *FakeGoMethodsDriver) V001_init_users_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_up") + return nil +} + +func (driver *FakeGoMethodsDriver) V001_init_users_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_down") + return nil +} + +type SomeError struct{} +func (e SomeError) Error() string { return "Some error happened" } + +func (driver *FakeGoMethodsDriver) V001_some_failing_method_up() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_up") + return SomeError{} +} + +func (driver *FakeGoMethodsDriver) V001_some_failing_method_down() error { + driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_down") + return SomeError{} +} + +func TestMigrate(t *testing.T) { + cases := []struct { + name string + file file.File + expectedInvokedMethods []string + expectedErrors []error + expectRollback bool + }{ + { + name: "up migration, both directions-file: invokes up methods in order", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, + expectedErrors: []error{}, + }, + { + name: "down migration, both-directions-file: reverts direction of invoked down methods", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, + expectedErrors: []error{}, + }, + { + name: "up migration, up direction-file: invokes up methods in order", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, + expectedErrors: []error{}, + }, + { + name: "down migration, down directions-file: keeps order of invoked down methods", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_init_users + `), + }, + expectedInvokedMethods: []string{"V001_init_organizations_down", "V001_init_users_down"}, + expectedErrors: []error{}, + }, + { + name: "up migration: non-existing method causes migration not to execute", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_some_non_existing_method + `), + }, + expectedInvokedMethods: []string{}, + expectedErrors: []error{ MissingMethodError("V001_some_non_existing_method_up") }, + }, + { + name: "up migration: failing method stops execution", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_organizations_up", + "V001_some_failing_method_up", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_up", + Err: SomeError{}, + }}, + }, + { + name: "down migration, both-directions-file: failing method stops migration", + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_users_down", + "V001_some_failing_method_down", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_down", + Err: SomeError{}, + }}, + }, + { + name: "up migration: failing method causes rollback in rollback mode", + expectRollback: true, + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_some_failing_method + `), + }, + expectedInvokedMethods: []string{ + "V001_init_organizations_up", + "V001_init_users_up", + "V001_some_failing_method_up", + "V001_init_users_down", + "V001_init_organizations_down", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_up", + Err: SomeError{}, + }}, + }, + { + name: "down migration, both-directions-file: failing method causes rollback in rollback mode", + expectRollback: true, + file: file.File { + Path: "/foobar", + FileName: "001_foobar.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_organizations + V001_some_failing_method + V001_init_users + `), + }, + expectedInvokedMethods: []string{ + "V001_init_users_down", + "V001_some_failing_method_down", + "V001_init_users_up", + }, + expectedErrors: []error{ &MethodInvocationFailedError{ + MethodName: "V001_some_failing_method_down", + Err: SomeError{}, + }}, + }, + + } + + for _, c := range cases { + migrator := Migrator{} + d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} + migrator.Driver = d + migrator.RollbackOnFailure = c.expectRollback + + pipe := pipep.New() + go func() { + migrator.Migrate(c.file, pipe) + close(pipe) + }() + errs := pipep.ReadErrors(pipe) + + var failed bool + if !reflect.DeepEqual(d.InvokedMethods, c.expectedInvokedMethods) { + failed = true + t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, d.InvokedMethods) + } + if !reflect.DeepEqual(errs, c.expectedErrors) { + failed = true + t.Errorf("case '%s': FAILED\nexpected errors %v\nbut got %v", c.name, c.expectedErrors, errs) + + } + if !failed { + t.Logf("case '%s': PASSED", c.name) + } + } +} + + + +func TestGetRollbackToMethod(t *testing.T) { + cases := []struct { + method string + expectedRollbackMethod string + }{ + {"some_method_up", "some_method_down"}, + {"some_method_down", "some_method_up"}, + {"up_down_up", "up_down_down"}, + {"down_up", "down_down"}, + {"down_down", "down_up"}, + } + + for _, c := range cases { + actualRollbackMethod := getRollbackToMethod(c.method) + if actualRollbackMethod != c.expectedRollbackMethod { + t.Errorf("Expected rollback method to be %s but got %s", c.expectedRollbackMethod, actualRollbackMethod) + } + } +} + +func TestReverseInPlace(t *testing.T) { + methods := []string { + "method1_down", + "method2_down", + "method3_down", + "method4_down", + "method5_down", + } + + expectedReversedMethods := []string { + "method5_down", + "method4_down", + "method3_down", + "method2_down", + "method1_down", + } + + reverseInPlace(methods) + + if !reflect.DeepEqual(methods, expectedReversedMethods) { + t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) + } +} + diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go new file mode 100644 index 0000000..48817cf --- /dev/null +++ b/driver/gomethods/mongodb/mongodb_template.go @@ -0,0 +1,111 @@ +package mongodb + +import ( + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "errors" + "strings" + "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/driver/gomethods" +) + +const MIGRATE_C = "db_migrations" + + +// This is not a real driver since the Initialize method requires a gomethods.Migrator +// The real driver will contain the DriverTemplate and implement all the custom migration Golang methods +// See example in usage_examples for details +type DriverTemplate struct { + Session *mgo.Session + DbName string + + migrator gomethods.Migrator +} + + +type DbMigration struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Version uint64 `bson:"version"` +} + +func (driver *DriverTemplate) Initialize(url, dbName string, migrator gomethods.Migrator) error { + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) + if len(urlWithoutScheme) != 2 { + return errors.New("invalid mongodb:// scheme") + } + + session, err := mgo.Dial(url) + if err != nil { + return err + } + session.SetMode(mgo.Monotonic, true) + + driver.Session = session + driver.DbName = dbName + driver.migrator = migrator + + return nil +} + +func (driver *DriverTemplate) Close() error { + if driver.Session != nil { + driver.Session.Close() + } + return nil +} + +func (driver *DriverTemplate) FilenameParser() file.FilenameParser { + return file.UpDownAndBothFilenameParser{FilenameExtension: driver.FilenameExtension()} +} + +func (driver *DriverTemplate) FilenameExtension() string { + return "mgo" +} + + +func (driver *DriverTemplate) Version() (uint64, error) { + var latestMigration DbMigration + c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + + err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) + + switch { + case err == mgo.ErrNotFound: + return 0, nil + case err != nil: + return 0, err + default: + return latestMigration.Version, nil + } +} +func (driver *DriverTemplate) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + err := driver.migrator.Migrate(f, pipe) + if err != nil { + return + } + + migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + if f.Direction == direction.Up { + id := bson.NewObjectId() + dbMigration := DbMigration{Id: id, Version: f.Version} + + err := migrate_c.Insert(dbMigration) + if err != nil { + pipe <- err + return + } + + } else if f.Direction == direction.Down { + err := migrate_c.Remove(bson.M{"version": f.Version}) + if err != nil { + pipe <- err + return + } + } +} diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go new file mode 100644 index 0000000..be0ccb2 --- /dev/null +++ b/driver/gomethods/usage_examples/mongodb.go @@ -0,0 +1,121 @@ +package usage_examples + +import ( + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "gopkg.in/mgo.v2/bson" + "time" +) + +// This boilerplate part is necessary and the same +// regardless of the specific mongodb golang methods driver + +type GoMethodsMongoDbDriver struct { + mongodb.DriverTemplate +} + +func (d *GoMethodsMongoDbDriver) Initialize(url string) error { + return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{Driver: d}) +} + +func init() { + driver.RegisterDriver("mongodb", &GoMethodsMongoDbDriver{mongodb.DriverTemplate{}}) +} + + + +// Here goes the specific mongodb golang methods driver logic + +const DB_NAME = "test" +const SHORT_DATE_LAYOUT = "2000-Jan-01" +const USERS_C = "users" +const ORGANIZATIONS_C = "organizations" + +type Organization struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Location string `bson:"location"` + DateFounded time.Time `bson:"date_founded"` +} + +type User struct { + Id bson.ObjectId `bson:"_id"` + Name string `bson:"name"` +} + +func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + orgs := []Organization{ + {Id: bson.NewObjectId(), Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: bson.NewObjectId(), Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: bson.NewObjectId(), Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + } + + for _, org := range orgs { + err := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) + if err != nil { + return err + } + } + return nil +} + +func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { + return m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() +} + +func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { + users := []User{ + {Id: bson.NewObjectId(), Name: "Alex"}, + {Id: bson.NewObjectId(), Name: "Beatrice"}, + {Id: bson.NewObjectId(), Name: "Cleo"}, + } + + for _, user := range users { + err := m.Session.DB(DB_NAME).C(USERS_C).Insert(user) + if err != nil { + return err + } + } + return nil +} + +func (m *GoMethodsMongoDbDriver) V001_init_users_down() error { + return m.Session.DB(DB_NAME).C(USERS_C).DropCollection() +} + +func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_up() error { + c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) + return err +} + +func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_down() error { + c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) + return err +} + +func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_up() error { + c := m.Session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleo"} + change := bson.M{"$set": bson.M{"name": "Cleopatra"}} + + return c.Update(colQuerier, change) +} + +func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_down() error { + c := m.Session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleopatra"} + change := bson.M{"$set": bson.M{"name": "Cleo",}} + + return c.Update(colQuerier, change) +} \ No newline at end of file diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go new file mode 100644 index 0000000..ec1a85d --- /dev/null +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -0,0 +1,123 @@ +package usage_examples + +import ( + "testing" + + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + + pipep "github.com/dimag-jfrog/migrate/pipe" +) + + + +func TestMigrate(t *testing.T) { + //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") + //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") + host := "127.0.0.1" + port := "27017" + driverUrl := "mongodb://" + host + ":" + port + + d := &GoMethodsMongoDbDriver{} + if err := d.Initialize(driverUrl); err != nil { + t.Fatal(err) + } + + content1 := []byte(` + V001_init_organizations + V001_init_users + `) + content2 := []byte(` + V002_organizations_rename_location_field_to_headquarters + V002_change_user_cleo_to_cleopatra + `) + + files := []file.File{ + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: content1, + }, + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: content1, + }, + { + Path: "/foobar", + FileName: "002_foobar.mgo", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: content2, + }, + { + Path: "/foobar", + FileName: "002_foobar.mgo", + Version: 2, + Name: "foobar", + Direction: direction.Down, + Content: content2, + }, + { + Path: "/foobar", + FileName: "001_foobar.mgo", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations + V001_init_users + V001_non_existing_operation + `), + }, + } + + var pipe chan interface{} + var errs []error + + pipe = pipep.New() + go d.Migrate(files[0], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + pipe = pipep.New() + go d.Migrate(files[2], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + pipe = pipep.New() + go d.Migrate(files[3], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) > 0 { + t.Fatal(errs) + } + + //pipe = pipep.New() + //go d.Migrate(files[1], pipe) + //errs = pipep.ReadErrors(pipe) + //if len(errs) > 0 { + // t.Fatal(errs) + //} + + pipe = pipep.New() + go d.Migrate(files[4], pipe) + errs = pipep.ReadErrors(pipe) + if len(errs) == 0 { + t.Error("Expected test case to fail") + } + + if err := d.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/file/file.go b/file/file.go index 14617ba..794e951 100644 --- a/file/file.go +++ b/file/file.go @@ -9,20 +9,11 @@ import ( "go/token" "io/ioutil" "path" - "regexp" "sort" "strconv" "strings" ) -var filenameRegex = `^([0-9]+)_(.*)\.(up|down)\.%s$` - -// FilenameRegex builds regular expression stmt with given -// filename extension from driver. -func FilenameRegex(filenameExtension string) *regexp.Regexp { - return regexp.MustCompile(fmt.Sprintf(filenameRegex, filenameExtension)) -} - // File represents one file on disk. // Example: 001_initial_plan_to_do_sth.up.sql type File struct { @@ -149,8 +140,17 @@ func (mf *MigrationFiles) From(version uint64, relativeN int) (Files, error) { return files, nil } + +func ReadMigrationFiles(path, filenameExtension string) (MigrationFiles, error){ + return doReadMigrationFiles(path, DefaultFilenameParser{FilenameExtension: filenameExtension}) +} + +func ReadMigrationFilesWithFilenameParser(path string, filenameParser FilenameParser) (MigrationFiles, error){ + return doReadMigrationFiles(path, filenameParser) +} + // ReadMigrationFiles reads all migration files from a given path -func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files MigrationFiles, err error) { +func doReadMigrationFiles(path string, filenameParser FilenameParser) (files MigrationFiles, err error) { // find all migration files in path ioFiles, err := ioutil.ReadDir(path) if err != nil { @@ -165,7 +165,7 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat tmpFiles := make([]*tmpFile, 0) tmpFileMap := map[uint64]map[direction.Direction]tmpFile{} for _, file := range ioFiles { - version, name, d, err := parseFilenameSchema(file.Name(), filenameRegex) + version, name, d, err := filenameParser.Parse(file.Name()) if err == nil { if _, ok := tmpFileMap[version]; !ok { tmpFileMap[version] = map[direction.Direction]tmpFile{} @@ -210,33 +210,56 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat Direction: direction.Down, } lookFordirection = direction.Up + case direction.Both: + migrationFile.UpFile = &File{ + Path: path, + FileName: file.filename, + Version: file.version, + Name: file.name, + Content: nil, + Direction: direction.Up, + } + migrationFile.DownFile = &File{ + Path: path, + FileName: file.filename, + Version: file.version, + Name: file.name, + Content: nil, + Direction: direction.Down, + } default: return nil, errors.New("Unsupported direction.Direction Type") } for _, file2 := range tmpFiles { - if file2.version == file.version && file2.d == lookFordirection { - switch lookFordirection { - case direction.Up: - migrationFile.UpFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Up, + if file2.version == file.version { + if file.d == direction.Both { + if file.d != file2.d { + return nil, errors.New("Incompatible direction.Direction types") } - case direction.Down: - migrationFile.DownFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Down, + } else if file2.d == lookFordirection { + switch lookFordirection { + case direction.Up: + migrationFile.UpFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Up, + } + case direction.Down: + migrationFile.DownFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Down, + } } + break } - break } } @@ -249,29 +272,6 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat return newFiles, nil } -// parseFilenameSchema parses the filename -func parseFilenameSchema(filename string, filenameRegex *regexp.Regexp) (version uint64, name string, d direction.Direction, err error) { - matches := filenameRegex.FindStringSubmatch(filename) - if len(matches) != 4 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - - if matches[3] == "up" { - d = direction.Up - } else if matches[3] == "down" { - d = direction.Down - } else { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) - } - - return version, matches[2], d, nil -} - // Len is the number of elements in the collection. // Required by Sort Interface{} func (mf MigrationFiles) Len() int { diff --git a/file/file_test.go b/file/file_test.go index c6bd0b3..648f554 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -8,52 +8,6 @@ import ( "testing" ) -func TestParseFilenameSchema(t *testing.T) { - var tests = []struct { - filename string - filenameExtension string - expectVersion uint64 - expectName string - expectDirection direction.Direction - expectErr bool - }{ - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - version, name, migrate, err := parseFilenameSchema(test.filename, FilenameRegex(test.filenameExtension)) - if test.expectErr && err == nil { - t.Fatal("Expected error, but got none.", test) - } - if !test.expectErr && err != nil { - t.Fatal("Did not expect error, but got one:", err, test) - } - if err == nil { - if version != test.expectVersion { - t.Error("Wrong version number", test) - } - if name != test.expectName { - t.Error("wrong name", test) - } - if migrate != test.expectDirection { - t.Error("wrong migrate", test) - } - } - } -} - func TestFiles(t *testing.T) { tmpdir, err := ioutil.TempDir("/tmp", "TestLookForMigrationFilesInSearchPath") if err != nil { @@ -77,7 +31,7 @@ func TestFiles(t *testing.T) { ioutil.WriteFile(path.Join(tmpdir, "401_migrationfile.down.sql"), []byte("test"), 0755) - files, err := ReadMigrationFiles(tmpdir, FilenameRegex("sql")) + files, err := ReadMigrationFiles(tmpdir, "sql") if err != nil { t.Fatal(err) } @@ -223,7 +177,7 @@ func TestDuplicateFiles(t *testing.T) { t.Fatal(err) } - _, err = ReadMigrationFiles(root, FilenameRegex("sql")) + _, err = ReadMigrationFiles(root, "sql") if err == nil { t.Fatal("Expected duplicate migration file error") } diff --git a/file/filename_parser.go b/file/filename_parser.go new file mode 100644 index 0000000..6fb8521 --- /dev/null +++ b/file/filename_parser.go @@ -0,0 +1,85 @@ +package file + +import ( + "errors" + "github.com/dimag-jfrog/migrate/migrate/direction" + "regexp" + "fmt" + "strconv" + "strings" +) + + +type FilenameParser interface { + Parse (filename string) (version uint64, name string, d direction.Direction, err error) +} + + +var defaultFilenameRegexTemplate = `^([0-9]+)_(.*)\.(up|down)\.%s$` + +func parseDefaultFilenameSchema(filename, filenameRegex string) (version uint64, name string, d direction.Direction, err error) { + regexp := regexp.MustCompile(filenameRegex) + matches := regexp.FindStringSubmatch(filename) + if len(matches) != 4 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + + name = matches[2] + + if matches[3] == "up" { + d = direction.Up + } else if matches[3] == "down" { + d = direction.Down + } else { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) + } + + return version, name, d, nil +} + +type DefaultFilenameParser struct { + FilenameExtension string +} + +func (parser DefaultFilenameParser) Parse (filename string) (version uint64, name string, d direction.Direction, err error) { + filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) + return parseDefaultFilenameSchema(filename, filenameRegex) +} + + +type UpDownAndBothFilenameParser struct { + FilenameExtension string +} + +func (parser UpDownAndBothFilenameParser) Parse(filename string) (version uint64, name string, d direction.Direction, err error) { + ext := parser.FilenameExtension + if !strings.HasSuffix(filename, ext) { + return 0, "", 0, errors.New("Filename ") + } + + var matches []string + if strings.HasSuffix(filename, ".up." + ext) || strings.HasSuffix(filename, ".down." + ext) { + filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) + return parseDefaultFilenameSchema(filename, filenameRegex) + } + + regex := regexp.MustCompile(fmt.Sprintf(`^([0-9]+)_(.*)\.%s$`, ext)) + matches = regex.FindStringSubmatch(filename) + if len(matches) != 3 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + name = matches[2] + d = direction.Both + + return version, name, d, nil +} diff --git a/file/filename_parser_test.go b/file/filename_parser_test.go new file mode 100644 index 0000000..ea9135d --- /dev/null +++ b/file/filename_parser_test.go @@ -0,0 +1,85 @@ +package file + +import ( + "github.com/dimag-jfrog/migrate/migrate/direction" + "testing" +) + + +type ParsingTest struct { + filename string + filenameExtension string + expectVersion uint64 + expectName string + expectDirection direction.Direction + expectErr bool +} + +func testParser(t *testing.T, parser FilenameParser, test *ParsingTest) { + version, name, migrate, err := parser.Parse(test.filename) + if test.expectErr && err == nil { + t.Fatal("Expected error, but got none.", test) + } + if !test.expectErr && err != nil { + t.Fatal("Did not expect error, but got one:", err, test) + } + if err == nil { + if version != test.expectVersion { + t.Error("Wrong version number", test) + } + if name != test.expectName { + t.Error("wrong name", test) + } + if migrate != test.expectDirection { + t.Error("wrong migrate", test) + } + } +} + +func TestParseDefaultFilenameSchema(t *testing.T) { + var tests = []ParsingTest { + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + parser := DefaultFilenameParser{FilenameExtension:test.filenameExtension} + testParser(t, &parser, &test) + } +} + +func TestParseUpDownAndBothFilenameSchema(t *testing.T) { + var tests = []ParsingTest { + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 100, "test_file", direction.Both, false}, + {"001_test_file.mgo", "mgo", 1, "test_file", direction.Both, false}, + {"-1_test_file.mgo", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + parser := UpDownAndBothFilenameParser{FilenameExtension:test.filenameExtension} + testParser(t, &parser, &test) + } +} diff --git a/migrate/direction/direction.go b/migrate/direction/direction.go index ed5e5ec..3ebc7b8 100644 --- a/migrate/direction/direction.go +++ b/migrate/direction/direction.go @@ -6,4 +6,5 @@ type Direction int const ( Up Direction = +1 Down = -1 + Both = 0 ) From 223908f9d2383265fc5059e7ad4095c1efabce8c Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 21:07:37 +0300 Subject: [PATCH 17/27] - Reset all changes to the upstream branch - Changed logic not to use custom filename parser: -Supporting up and down files only, no both direction files -Using method names as is - Added complete test to the mongo db migration scenarios --- .gitignore | 11 +- driver/driver.go | 9 - driver/gomethods/gomethods_migrator.go | 35 +-- driver/gomethods/gomethods_migrator_test.go | 95 ++----- driver/gomethods/mongodb/mongodb_template.go | 4 - driver/gomethods/usage_examples/mongodb.go | 31 +- .../gomethods/usage_examples/mongodb_test.go | 268 ++++++++++++------ file/file.go | 106 +++---- file/file_test.go | 50 +++- file/filename_parser.go | 85 ------ file/filename_parser_test.go | 85 ------ migrate/direction/direction.go | 1 - 12 files changed, 359 insertions(+), 421 deletions(-) delete mode 100644 file/filename_parser.go delete mode 100644 file/filename_parser_test.go diff --git a/.gitignore b/.gitignore index 9bd9f47..ee2f41f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,2 @@ .DS_Store -test.db -*.iml -.DS_Store -.idea -*.zip -*.tar.* -*.nuget -*.jar -*.war - +test.db \ No newline at end of file diff --git a/driver/driver.go b/driver/driver.go index a668d52..5c2f6fc 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -34,15 +34,6 @@ type Driver interface { Version() (uint64, error) } -// Driver that has some custom migration file format -// and wants to use a different parsing strategy. -type DriverWithFilenameParser interface { - - Driver - - FilenameParser() file.FilenameParser -} - // New returns Driver and calls Initialize on it func New(url string) (Driver, error) { u, err := neturl.Parse(url) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 68b9ff5..d695342 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -10,7 +10,6 @@ import ( "bufio" "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" ) @@ -32,7 +31,7 @@ func (e *MethodInvocationFailedError) Error() string { type Migrator struct { - Driver driver.DriverWithFilenameParser + Driver driver.Driver RollbackOnFailure bool } @@ -55,6 +54,10 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { // on failure, try to rollback methods in this migration for j := i-1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) + if rollbackToMethodName == "" || !m.IsValid(rollbackToMethodName) { + continue + } + pipe <- rollbackToMethodName err = m.Invoke(rollbackToMethodName) if err != nil { @@ -106,8 +109,10 @@ func reverseInPlace(a []string) { func getRollbackToMethod(methodName string) string { if strings.HasSuffix(methodName, "_up") { return strings.TrimSuffix(methodName, "_up") + "_down" - } else { + } else if strings.HasSuffix(methodName, "_down") { return strings.TrimSuffix(methodName, "_down") + "_up" + } else { + return "" } } @@ -143,34 +148,20 @@ func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { } for _, line := range lines { - operationName := strings.TrimSpace(line) + methodName := strings.TrimSpace(line) - if operationName == "" || strings.HasPrefix(operationName, "--") { + if methodName == "" || strings.HasPrefix(methodName, "--") { // an empty line or a comment, ignore continue } - upMethodName := operationName + "_up" - downMethodName := operationName + "_down" - - if !m.IsValid(upMethodName) { - return nil, MissingMethodError(upMethodName) - } - if !m.IsValid(downMethodName) { - return nil, MissingMethodError(downMethodName) + if !m.IsValid(methodName) { + return nil, MissingMethodError(methodName) } - if f.Direction == direction.Up { - methods = append(methods, upMethodName) - } else { - methods = append(methods, downMethodName) - } + methods = append(methods, methodName) } - _,_,fileType,_ := m.Driver.FilenameParser().Parse(f.FileName) - if fileType == direction.Both && f.Direction == direction.Down { - reverseInPlace(methods) - } return methods, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index 58fd4f2..1a5452f 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -23,10 +23,6 @@ func (driver *FakeGoMethodsDriver) Close() error { return nil } -func (driver *FakeGoMethodsDriver) FilenameParser() file.FilenameParser { - return file.UpDownAndBothFilenameParser{ FilenameExtension: driver.FilenameExtension() } -} - func (driver *FakeGoMethodsDriver) FilenameExtension() string { return "gm" } @@ -84,39 +80,7 @@ func TestMigrate(t *testing.T) { expectRollback bool }{ { - name: "up migration, both directions-file: invokes up methods in order", - file: file.File { - Path: "/foobar", - FileName: "001_foobar.gm", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations - V001_init_users - `), - }, - expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, - expectedErrors: []error{}, - }, - { - name: "down migration, both-directions-file: reverts direction of invoked down methods", - file: file.File { - Path: "/foobar", - FileName: "001_foobar.gm", - Version: 1, - Name: "foobar", - Direction: direction.Down, - Content: []byte(` - V001_init_organizations - V001_init_users - `), - }, - expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, - expectedErrors: []error{}, - }, - { - name: "up migration, up direction-file: invokes up methods in order", + name: "up migration invokes up methods", file: file.File { Path: "/foobar", FileName: "001_foobar.up.gm", @@ -124,15 +88,15 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users + V001_init_organizations_up + V001_init_users_up `), }, expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, expectedErrors: []error{}, }, { - name: "down migration, down directions-file: keeps order of invoked down methods", + name: "down migration invoked down methods", file: file.File { Path: "/foobar", FileName: "001_foobar.down.gm", @@ -140,25 +104,25 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_init_users + V001_init_users_down + V001_init_organizations_down `), }, - expectedInvokedMethods: []string{"V001_init_organizations_down", "V001_init_users_down"}, + expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, expectedErrors: []error{}, }, { name: "up migration: non-existing method causes migration not to execute", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users - V001_some_non_existing_method + V001_init_organizations_up + V001_init_users_up + V001_some_non_existing_method_up `), }, expectedInvokedMethods: []string{}, @@ -168,14 +132,14 @@ func TestMigrate(t *testing.T) { name: "up migration: failing method stops execution", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_organizations_up + V001_some_failing_method_up + V001_init_users_up `), }, expectedInvokedMethods: []string{ @@ -188,17 +152,17 @@ func TestMigrate(t *testing.T) { }}, }, { - name: "down migration, both-directions-file: failing method stops migration", + name: "down migration: failing method stops migration", file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_users_down + V001_some_failing_method_down + V001_init_organizations_down `), }, expectedInvokedMethods: []string{ @@ -215,14 +179,14 @@ func TestMigrate(t *testing.T) { expectRollback: true, file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` - V001_init_organizations - V001_init_users - V001_some_failing_method + V001_init_organizations_up + V001_init_users_up + V001_some_failing_method_up `), }, expectedInvokedMethods: []string{ @@ -238,18 +202,18 @@ func TestMigrate(t *testing.T) { }}, }, { - name: "down migration, both-directions-file: failing method causes rollback in rollback mode", + name: "down migration: failing method causes rollback in rollback mode", expectRollback: true, file: file.File { Path: "/foobar", - FileName: "001_foobar.gm", + FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` - V001_init_organizations - V001_some_failing_method - V001_init_users + V001_init_users_down + V001_some_failing_method_down + V001_init_organizations_down `), }, expectedInvokedMethods: []string{ @@ -306,6 +270,7 @@ func TestGetRollbackToMethod(t *testing.T) { {"up_down_up", "up_down_down"}, {"down_up", "down_down"}, {"down_down", "down_up"}, + {"some_method", ""}, } for _, c := range cases { diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go index 48817cf..1edbe78 100644 --- a/driver/gomethods/mongodb/mongodb_template.go +++ b/driver/gomethods/mongodb/mongodb_template.go @@ -55,10 +55,6 @@ func (driver *DriverTemplate) Close() error { return nil } -func (driver *DriverTemplate) FilenameParser() file.FilenameParser { - return file.UpDownAndBothFilenameParser{FilenameExtension: driver.FilenameExtension()} -} - func (driver *DriverTemplate) FilenameExtension() string { return "mgo" } diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go index be0ccb2..ea7a680 100644 --- a/driver/gomethods/usage_examples/mongodb.go +++ b/driver/gomethods/usage_examples/mongodb.go @@ -39,20 +39,39 @@ type Organization struct { DateFounded time.Time `bson:"date_founded"` } +type Organization_v2 struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Headquarters string `bson:"headquarters"` + DateFounded time.Time `bson:"date_founded"` +} + type User struct { Id bson.ObjectId `bson:"_id"` Name string `bson:"name"` } +var OrganizationIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +var UserIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") orgs := []Organization{ - {Id: bson.NewObjectId(), Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: bson.NewObjectId(), Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: bson.NewObjectId(), Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, } for _, org := range orgs { @@ -70,9 +89,9 @@ func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { users := []User{ - {Id: bson.NewObjectId(), Name: "Alex"}, - {Id: bson.NewObjectId(), Name: "Beatrice"}, - {Id: bson.NewObjectId(), Name: "Cleo"}, + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, } for _, user := range users { diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go index ec1a85d..033876c 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -7,9 +7,69 @@ import ( "github.com/dimag-jfrog/migrate/migrate/direction" pipep "github.com/dimag-jfrog/migrate/pipe" + "reflect" + "time" + "github.com/dimag-jfrog/migrate/driver/gomethods" ) +type ExpectedMigrationResult struct { + Organizations []Organization + Organizations_v2 []Organization_v2 + Users []User + Errors []error +} + +func RunMigrationAndAssertResult( + t *testing.T, + title string, + d *GoMethodsMongoDbDriver, + file file.File, + expected *ExpectedMigrationResult) { + + actualOrganizations := []Organization{} + actualOrganizations_v2 := []Organization_v2{} + actualUsers := []User{} + var err error + var pipe chan interface{} + var errs []error + + pipe = pipep.New() + go d.Migrate(file, pipe) + errs = pipep.ReadErrors(pipe) + + session := d.Session + if len(expected.Organizations) > 0 { + err = session.DB(DB_NAME).C(ORGANIZATIONS_C).Find(nil).All(&actualOrganizations) + } else { + err = session.DB(DB_NAME).C(ORGANIZATIONS_C).Find(nil).All(&actualOrganizations_v2) + } + if err != nil { + t.Fatal("Failed to query Organizations collection") + } + + err = session.DB(DB_NAME).C(USERS_C).Find(nil).All(&actualUsers) + if err != nil { + t.Fatal("Failed to query Users collection") + } + + if !reflect.DeepEqual(expected.Organizations, actualOrganizations) { + t.Fatalf("Migration '%s': FAILED\nexpected organizations %v\nbut got %v", title, expected.Organizations, actualOrganizations) + } + + if !reflect.DeepEqual(expected.Organizations_v2, actualOrganizations_v2) { + t.Fatalf("Migration '%s': FAILED\nexpected organizations v2 %v\nbut got %v", title, expected.Organizations_v2, actualOrganizations_v2) + } + + if !reflect.DeepEqual(expected.Users, actualUsers) { + t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) + + } + if !reflect.DeepEqual(expected.Errors, errs) { + t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) + } +} + func TestMigrate(t *testing.T) { //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") @@ -23,98 +83,148 @@ func TestMigrate(t *testing.T) { t.Fatal(err) } - content1 := []byte(` - V001_init_organizations - V001_init_users - `) - content2 := []byte(` - V002_organizations_rename_location_field_to_headquarters - V002_change_user_cleo_to_cleopatra - `) + // Reset DB + d.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() + d.Session.DB(DB_NAME).C(USERS_C).DropCollection() - files := []file.File{ + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + migrations := []struct { + name string + file file.File + expectedResult ExpectedMigrationResult + }{ { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: content1, + name: "v0 -> v1", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + }, + Organizations_v2: []Organization_v2{}, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Down, - Content: content1, + name: "v1 -> v2", + file: file.File{ + Path: "/foobar", + FileName: "002_foobar.up.gm", + Version: 2, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V002_organizations_rename_location_field_to_headquarters_up + V002_change_user_cleo_to_cleopatra_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{ + {Id: OrganizationIds[0], Name: "Amazon", Headquarters:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Headquarters:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Headquarters:"Santa Clara", DateFounded: date3}, + }, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleopatra"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "002_foobar.mgo", - Version: 2, - Name: "foobar", - Direction: direction.Up, - Content: content2, + name: "v2 -> v1", + file: file.File{ + Path: "/foobar", + FileName: "002_foobar.down.gm", + Version: 2, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V002_change_user_cleo_to_cleopatra_down + V002_organizations_rename_location_field_to_headquarters_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + }, + Organizations_v2: []Organization_v2{}, + Users: []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + }, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "002_foobar.mgo", - Version: 2, - Name: "foobar", - Direction: direction.Down, - Content: content2, + name: "v1 -> v0", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_init_organizations_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{}, + }, }, { - Path: "/foobar", - FileName: "001_foobar.mgo", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations - V001_init_users - V001_non_existing_operation - `), + name: "v0 -> v1: with error", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + v001_non_existing_method_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{ gomethods.MissingMethodError("v001_non_existing_method_up") }, + }, }, } - var pipe chan interface{} - var errs []error - - pipe = pipep.New() - go d.Migrate(files[0], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - pipe = pipep.New() - go d.Migrate(files[2], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - pipe = pipep.New() - go d.Migrate(files[3], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) > 0 { - t.Fatal(errs) - } - - //pipe = pipep.New() - //go d.Migrate(files[1], pipe) - //errs = pipep.ReadErrors(pipe) - //if len(errs) > 0 { - // t.Fatal(errs) - //} - - pipe = pipep.New() - go d.Migrate(files[4], pipe) - errs = pipep.ReadErrors(pipe) - if len(errs) == 0 { - t.Error("Expected test case to fail") + for _, m := range migrations { + RunMigrationAndAssertResult(t, m.name, d, m.file, &m.expectedResult) } if err := d.Close(); err != nil { diff --git a/file/file.go b/file/file.go index 794e951..14617ba 100644 --- a/file/file.go +++ b/file/file.go @@ -9,11 +9,20 @@ import ( "go/token" "io/ioutil" "path" + "regexp" "sort" "strconv" "strings" ) +var filenameRegex = `^([0-9]+)_(.*)\.(up|down)\.%s$` + +// FilenameRegex builds regular expression stmt with given +// filename extension from driver. +func FilenameRegex(filenameExtension string) *regexp.Regexp { + return regexp.MustCompile(fmt.Sprintf(filenameRegex, filenameExtension)) +} + // File represents one file on disk. // Example: 001_initial_plan_to_do_sth.up.sql type File struct { @@ -140,17 +149,8 @@ func (mf *MigrationFiles) From(version uint64, relativeN int) (Files, error) { return files, nil } - -func ReadMigrationFiles(path, filenameExtension string) (MigrationFiles, error){ - return doReadMigrationFiles(path, DefaultFilenameParser{FilenameExtension: filenameExtension}) -} - -func ReadMigrationFilesWithFilenameParser(path string, filenameParser FilenameParser) (MigrationFiles, error){ - return doReadMigrationFiles(path, filenameParser) -} - // ReadMigrationFiles reads all migration files from a given path -func doReadMigrationFiles(path string, filenameParser FilenameParser) (files MigrationFiles, err error) { +func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files MigrationFiles, err error) { // find all migration files in path ioFiles, err := ioutil.ReadDir(path) if err != nil { @@ -165,7 +165,7 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig tmpFiles := make([]*tmpFile, 0) tmpFileMap := map[uint64]map[direction.Direction]tmpFile{} for _, file := range ioFiles { - version, name, d, err := filenameParser.Parse(file.Name()) + version, name, d, err := parseFilenameSchema(file.Name(), filenameRegex) if err == nil { if _, ok := tmpFileMap[version]; !ok { tmpFileMap[version] = map[direction.Direction]tmpFile{} @@ -210,56 +210,33 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig Direction: direction.Down, } lookFordirection = direction.Up - case direction.Both: - migrationFile.UpFile = &File{ - Path: path, - FileName: file.filename, - Version: file.version, - Name: file.name, - Content: nil, - Direction: direction.Up, - } - migrationFile.DownFile = &File{ - Path: path, - FileName: file.filename, - Version: file.version, - Name: file.name, - Content: nil, - Direction: direction.Down, - } default: return nil, errors.New("Unsupported direction.Direction Type") } for _, file2 := range tmpFiles { - if file2.version == file.version { - if file.d == direction.Both { - if file.d != file2.d { - return nil, errors.New("Incompatible direction.Direction types") + if file2.version == file.version && file2.d == lookFordirection { + switch lookFordirection { + case direction.Up: + migrationFile.UpFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Up, } - } else if file2.d == lookFordirection { - switch lookFordirection { - case direction.Up: - migrationFile.UpFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Up, - } - case direction.Down: - migrationFile.DownFile = &File{ - Path: path, - FileName: file2.filename, - Version: file.version, - Name: file2.name, - Content: nil, - Direction: direction.Down, - } + case direction.Down: + migrationFile.DownFile = &File{ + Path: path, + FileName: file2.filename, + Version: file.version, + Name: file2.name, + Content: nil, + Direction: direction.Down, } - break } + break } } @@ -272,6 +249,29 @@ func doReadMigrationFiles(path string, filenameParser FilenameParser) (files Mig return newFiles, nil } +// parseFilenameSchema parses the filename +func parseFilenameSchema(filename string, filenameRegex *regexp.Regexp) (version uint64, name string, d direction.Direction, err error) { + matches := filenameRegex.FindStringSubmatch(filename) + if len(matches) != 4 { + return 0, "", 0, errors.New("Unable to parse filename schema") + } + + version, err = strconv.ParseUint(matches[1], 10, 0) + if err != nil { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) + } + + if matches[3] == "up" { + d = direction.Up + } else if matches[3] == "down" { + d = direction.Down + } else { + return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) + } + + return version, matches[2], d, nil +} + // Len is the number of elements in the collection. // Required by Sort Interface{} func (mf MigrationFiles) Len() int { diff --git a/file/file_test.go b/file/file_test.go index 648f554..c6bd0b3 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -8,6 +8,52 @@ import ( "testing" ) +func TestParseFilenameSchema(t *testing.T) { + var tests = []struct { + filename string + filenameExtension string + expectVersion uint64 + expectName string + expectDirection direction.Direction + expectErr bool + }{ + {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, + {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, + {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, + {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"test_file.down.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file.down", "sql", 0, "", direction.Up, true}, + {"100_test_file.sql", "sql", 0, "", direction.Up, true}, + {"100_test_file", "sql", 0, "", direction.Up, true}, + {"test_file", "sql", 0, "", direction.Up, true}, + {"100", "sql", 0, "", direction.Up, true}, + {".sql", "sql", 0, "", direction.Up, true}, + {"up.sql", "sql", 0, "", direction.Up, true}, + {"down.sql", "sql", 0, "", direction.Up, true}, + } + + for _, test := range tests { + version, name, migrate, err := parseFilenameSchema(test.filename, FilenameRegex(test.filenameExtension)) + if test.expectErr && err == nil { + t.Fatal("Expected error, but got none.", test) + } + if !test.expectErr && err != nil { + t.Fatal("Did not expect error, but got one:", err, test) + } + if err == nil { + if version != test.expectVersion { + t.Error("Wrong version number", test) + } + if name != test.expectName { + t.Error("wrong name", test) + } + if migrate != test.expectDirection { + t.Error("wrong migrate", test) + } + } + } +} + func TestFiles(t *testing.T) { tmpdir, err := ioutil.TempDir("/tmp", "TestLookForMigrationFilesInSearchPath") if err != nil { @@ -31,7 +77,7 @@ func TestFiles(t *testing.T) { ioutil.WriteFile(path.Join(tmpdir, "401_migrationfile.down.sql"), []byte("test"), 0755) - files, err := ReadMigrationFiles(tmpdir, "sql") + files, err := ReadMigrationFiles(tmpdir, FilenameRegex("sql")) if err != nil { t.Fatal(err) } @@ -177,7 +223,7 @@ func TestDuplicateFiles(t *testing.T) { t.Fatal(err) } - _, err = ReadMigrationFiles(root, "sql") + _, err = ReadMigrationFiles(root, FilenameRegex("sql")) if err == nil { t.Fatal("Expected duplicate migration file error") } diff --git a/file/filename_parser.go b/file/filename_parser.go deleted file mode 100644 index 6fb8521..0000000 --- a/file/filename_parser.go +++ /dev/null @@ -1,85 +0,0 @@ -package file - -import ( - "errors" - "github.com/dimag-jfrog/migrate/migrate/direction" - "regexp" - "fmt" - "strconv" - "strings" -) - - -type FilenameParser interface { - Parse (filename string) (version uint64, name string, d direction.Direction, err error) -} - - -var defaultFilenameRegexTemplate = `^([0-9]+)_(.*)\.(up|down)\.%s$` - -func parseDefaultFilenameSchema(filename, filenameRegex string) (version uint64, name string, d direction.Direction, err error) { - regexp := regexp.MustCompile(filenameRegex) - matches := regexp.FindStringSubmatch(filename) - if len(matches) != 4 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - - name = matches[2] - - if matches[3] == "up" { - d = direction.Up - } else if matches[3] == "down" { - d = direction.Down - } else { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3])) - } - - return version, name, d, nil -} - -type DefaultFilenameParser struct { - FilenameExtension string -} - -func (parser DefaultFilenameParser) Parse (filename string) (version uint64, name string, d direction.Direction, err error) { - filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) - return parseDefaultFilenameSchema(filename, filenameRegex) -} - - -type UpDownAndBothFilenameParser struct { - FilenameExtension string -} - -func (parser UpDownAndBothFilenameParser) Parse(filename string) (version uint64, name string, d direction.Direction, err error) { - ext := parser.FilenameExtension - if !strings.HasSuffix(filename, ext) { - return 0, "", 0, errors.New("Filename ") - } - - var matches []string - if strings.HasSuffix(filename, ".up." + ext) || strings.HasSuffix(filename, ".down." + ext) { - filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension) - return parseDefaultFilenameSchema(filename, filenameRegex) - } - - regex := regexp.MustCompile(fmt.Sprintf(`^([0-9]+)_(.*)\.%s$`, ext)) - matches = regex.FindStringSubmatch(filename) - if len(matches) != 3 { - return 0, "", 0, errors.New("Unable to parse filename schema") - } - - version, err = strconv.ParseUint(matches[1], 10, 0) - if err != nil { - return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0])) - } - name = matches[2] - d = direction.Both - - return version, name, d, nil -} diff --git a/file/filename_parser_test.go b/file/filename_parser_test.go deleted file mode 100644 index ea9135d..0000000 --- a/file/filename_parser_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package file - -import ( - "github.com/dimag-jfrog/migrate/migrate/direction" - "testing" -) - - -type ParsingTest struct { - filename string - filenameExtension string - expectVersion uint64 - expectName string - expectDirection direction.Direction - expectErr bool -} - -func testParser(t *testing.T, parser FilenameParser, test *ParsingTest) { - version, name, migrate, err := parser.Parse(test.filename) - if test.expectErr && err == nil { - t.Fatal("Expected error, but got none.", test) - } - if !test.expectErr && err != nil { - t.Fatal("Did not expect error, but got one:", err, test) - } - if err == nil { - if version != test.expectVersion { - t.Error("Wrong version number", test) - } - if name != test.expectName { - t.Error("wrong name", test) - } - if migrate != test.expectDirection { - t.Error("wrong migrate", test) - } - } -} - -func TestParseDefaultFilenameSchema(t *testing.T) { - var tests = []ParsingTest { - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - parser := DefaultFilenameParser{FilenameExtension:test.filenameExtension} - testParser(t, &parser, &test) - } -} - -func TestParseUpDownAndBothFilenameSchema(t *testing.T) { - var tests = []ParsingTest { - {"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false}, - {"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false}, - {"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false}, - {"-1_test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"test_file.down.sql", "sql", 0, "", direction.Up, true}, - {"100_test_file.down", "sql", 0, "", direction.Up, true}, - {"100_test_file.sql", "sql", 100, "test_file", direction.Both, false}, - {"001_test_file.mgo", "mgo", 1, "test_file", direction.Both, false}, - {"-1_test_file.mgo", "sql", 0, "", direction.Up, true}, - {"100_test_file", "sql", 0, "", direction.Up, true}, - {"test_file", "sql", 0, "", direction.Up, true}, - {"100", "sql", 0, "", direction.Up, true}, - {".sql", "sql", 0, "", direction.Up, true}, - {"up.sql", "sql", 0, "", direction.Up, true}, - {"down.sql", "sql", 0, "", direction.Up, true}, - } - - for _, test := range tests { - parser := UpDownAndBothFilenameParser{FilenameExtension:test.filenameExtension} - testParser(t, &parser, &test) - } -} diff --git a/migrate/direction/direction.go b/migrate/direction/direction.go index 3ebc7b8..ed5e5ec 100644 --- a/migrate/direction/direction.go +++ b/migrate/direction/direction.go @@ -6,5 +6,4 @@ type Direction int const ( Up Direction = +1 Down = -1 - Both = 0 ) From 70b23c3f849675b85913a021ad1ee196b5a88d30 Mon Sep 17 00:00:00 2001 From: dimag Date: Mon, 8 Aug 2016 21:34:52 +0300 Subject: [PATCH 18/27] Relaxed constratint that method receiver has to be of Driver interface. --- driver/gomethods/gomethods_migrator.go | 7 +++---- driver/gomethods/gomethods_migrator_test.go | 2 +- driver/gomethods/usage_examples/mongodb.go | 2 +- driver/gomethods/usage_examples/mongodb_test.go | 1 + 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index d695342..0ff848a 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -8,7 +8,6 @@ import ( "os" "path" "bufio" - "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" ) @@ -31,7 +30,7 @@ func (e *MethodInvocationFailedError) Error() string { type Migrator struct { - Driver driver.Driver + MigrationMethodsReceiver interface{} RollbackOnFailure bool } @@ -73,12 +72,12 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { } func (m *Migrator) IsValid(methodName string) bool { - return reflect.ValueOf(m.Driver).MethodByName(methodName).IsValid() + return reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(methodName).IsValid() } func (m *Migrator) Invoke(methodName string) error { name := methodName - migrateMethod := reflect.ValueOf(m.Driver).MethodByName(name) + migrateMethod := reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { return MissingMethodError(methodName) } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index 1a5452f..a50d6c1 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -232,7 +232,7 @@ func TestMigrate(t *testing.T) { for _, c := range cases { migrator := Migrator{} d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} - migrator.Driver = d + migrator.MigrationMethodsReceiver = d migrator.RollbackOnFailure = c.expectRollback pipe := pipep.New() diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go index ea7a680..e7f30c0 100644 --- a/driver/gomethods/usage_examples/mongodb.go +++ b/driver/gomethods/usage_examples/mongodb.go @@ -16,7 +16,7 @@ type GoMethodsMongoDbDriver struct { } func (d *GoMethodsMongoDbDriver) Initialize(url string) error { - return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{Driver: d}) + return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{MigrationMethodsReceiver: d}) } func init() { diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/usage_examples/mongodb_test.go index 033876c..a57c51d 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/usage_examples/mongodb_test.go @@ -68,6 +68,7 @@ func RunMigrationAndAssertResult( if !reflect.DeepEqual(expected.Errors, errs) { t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) } + t.Logf("Migration '%s': PASSED", title) } From fe48ddfdcc0870bac4616e7be43f6fa6adc8cda8 Mon Sep 17 00:00:00 2001 From: dimag Date: Tue, 9 Aug 2016 16:47:34 +0300 Subject: [PATCH 19/27] - redefined mongo db driver to be autonomous and not depending on specific methods - defined methods receivers registration by name and change the migration files format to include them - added extensive testing --- driver/gomethods/gomethods_migrator.go | 105 ++++++------- driver/gomethods/gomethods_migrator_test.go | 144 +++++++----------- driver/gomethods/gomethods_registry.go | 30 ++++ driver/gomethods/mongodb/mongodb.go | 134 ++++++++++++++++ driver/gomethods/mongodb/mongodb_template.go | 107 ------------- .../mongodb_example/methods_receiver.go | 129 ++++++++++++++++ .../mongodb_test.go | 85 +++++++---- driver/gomethods/usage_examples/mongodb.go | 140 ----------------- 8 files changed, 462 insertions(+), 412 deletions(-) create mode 100644 driver/gomethods/gomethods_registry.go create mode 100644 driver/gomethods/mongodb/mongodb.go delete mode 100644 driver/gomethods/mongodb/mongodb_template.go create mode 100644 driver/gomethods/mongodb_example/methods_receiver.go rename driver/gomethods/{usage_examples => mongodb_example}/mongodb_test.go (71%) delete mode 100644 driver/gomethods/usage_examples/mongodb.go diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 0ff848a..2ee0192 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -2,40 +2,57 @@ package gomethods import ( //"bytes" - "reflect" + "bufio" + "database/sql/driver" "fmt" - "strings" + "github.com/dimag-jfrog/migrate/file" "os" "path" - "bufio" - "github.com/dimag-jfrog/migrate/file" + "strings" ) +type UnregisteredMethodsReceiverError string + +func (e UnregisteredMethodsReceiverError) Error() string { + return "Unregistered methods receiver: " + string(e) +} type MissingMethodError string -func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } +func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } type WrongMethodSignatureError string -func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", e) } + +func (e WrongMethodSignatureError) Error() string { + return fmt.Sprintf("Method %s has wrong signature", e) +} type MethodInvocationFailedError struct { MethodName string - Err error + Err error } func (e *MethodInvocationFailedError) Error() string { return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) } +type MigrationMethodInvoker interface { + IsValid(methodName string, methodReceiver interface{}) bool + Invoke(methodName string, methodReceiver interface{}) error +} + +type GoMethodsDriver interface { + driver.Driver + MigrationMethodInvoker +} type Migrator struct { - MigrationMethodsReceiver interface{} RollbackOnFailure bool + MethodInvoker MigrationMethodInvoker } func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { - methods, err := m.getMigrationMethods(f) + methods, methodsReceiver, err := m.getMigrationMethods(f) if err != nil { pipe <- err return err @@ -43,7 +60,7 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for i, methodName := range methods { pipe <- methodName - err := m.Invoke(methodName) + err := m.MethodInvoker.Invoke(methodName, methodsReceiver) if err != nil { pipe <- err if !m.RollbackOnFailure { @@ -51,14 +68,15 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { } // on failure, try to rollback methods in this migration - for j := i-1; j >= 0; j-- { + for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) - if rollbackToMethodName == "" || !m.IsValid(rollbackToMethodName) { + if rollbackToMethodName == "" || + !m.MethodInvoker.IsValid(rollbackToMethodName, methodsReceiver) { continue } pipe <- rollbackToMethodName - err = m.Invoke(rollbackToMethodName) + err = m.MethodInvoker.Invoke(rollbackToMethodName, methodsReceiver) if err != nil { pipe <- err break @@ -71,33 +89,6 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { return nil } -func (m *Migrator) IsValid(methodName string) bool { - return reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(methodName).IsValid() -} - -func (m *Migrator) Invoke(methodName string) error { - name := methodName - migrateMethod := reflect.ValueOf(m.MigrationMethodsReceiver).MethodByName(name) - if !migrateMethod.IsValid() { - return MissingMethodError(methodName) - } - - retValues := migrateMethod.Call([]reflect.Value{}) - if len(retValues) != 1 { - return WrongMethodSignatureError(name) - } - - if !retValues[0].IsNil() { - err, ok := retValues[0].Interface().(error) - if !ok { - return WrongMethodSignatureError(name) - } - return &MethodInvocationFailedError{ MethodName:name, Err:err} - } - - return nil -} - func reverseInPlace(a []string) { for i := 0; i < len(a)/2; i++ { j := len(a) - i - 1 @@ -139,28 +130,40 @@ func getFileLines(file file.File) ([]string, error) { } } -func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) { - var lines, methods []string - lines, err := getFileLines(f) +func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsReceiver interface{}, err error) { + var lines []string + + lines, err = getFileLines(f) if err != nil { - return nil, err + return nil, nil, err } for _, line := range lines { - methodName := strings.TrimSpace(line) + line := strings.TrimSpace(line) - if methodName == "" || strings.HasPrefix(methodName, "--") { + if line == "" || strings.HasPrefix(line, "--") { // an empty line or a comment, ignore continue } - if !m.IsValid(methodName) { - return nil, MissingMethodError(methodName) - } + if methodsReceiver == nil { + receiverName := line + methodsReceiver = GetMethodsReceiver(receiverName) + if methodsReceiver == nil { + return nil, nil, UnregisteredMethodsReceiverError(receiverName) + } + continue - methods = append(methods, methodName) + } else { + methodName := line + if !m.MethodInvoker.IsValid(methodName, methodsReceiver) { + return nil, nil, MissingMethodError(methodName) + } + + methods = append(methods, methodName) + } } - return methods, nil + return methods, methodsReceiver, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index a50d6c1..eb33103 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -10,133 +10,105 @@ import ( pipep "github.com/dimag-jfrog/migrate/pipe" ) -type FakeGoMethodsDriver struct { +type FakeGoMethodsInvoker struct { InvokedMethods []string - Migrator Migrator } -func (driver *FakeGoMethodsDriver) Initialize(url string) error { - return nil +func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver interface{}) bool { + if methodName == "V001_some_non_existing_method_up" { + return false + } + + return true } -func (driver *FakeGoMethodsDriver) Close() error { - return nil -} +func (invoker *FakeGoMethodsInvoker) Invoke(methodName string, methodReceiver interface{}) error { + invoker.InvokedMethods = append(invoker.InvokedMethods, methodName) -func (driver *FakeGoMethodsDriver) FilenameExtension() string { - return "gm" -} - -func (driver *FakeGoMethodsDriver) Version() (uint64, error) { - return uint64(0), nil -} - -func (driver *FakeGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { - defer close(pipe) - pipe <- f - return -} - -func (driver *FakeGoMethodsDriver) V001_init_organizations_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_up") - return nil -} - -func (driver *FakeGoMethodsDriver) V001_init_organizations_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_down") - return nil - -} - -func (driver *FakeGoMethodsDriver) V001_init_users_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_up") - return nil -} - -func (driver *FakeGoMethodsDriver) V001_init_users_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_down") - return nil + if methodName == "V001_some_failing_method_up" || methodName == "V001_some_failing_method_down" { + return &MethodInvocationFailedError{ + MethodName: methodName, + Err: SomeError{}, + } + } else { + return nil + } } type SomeError struct{} -func (e SomeError) Error() string { return "Some error happened" } -func (driver *FakeGoMethodsDriver) V001_some_failing_method_up() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_up") - return SomeError{} -} - -func (driver *FakeGoMethodsDriver) V001_some_failing_method_down() error { - driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_down") - return SomeError{} -} +func (e SomeError) Error() string { return "Some error happened" } func TestMigrate(t *testing.T) { cases := []struct { - name string - file file.File + name string + file file.File expectedInvokedMethods []string - expectedErrors []error - expectRollback bool + expectedErrors []error + expectRollback bool }{ { name: "up migration invokes up methods", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up `), }, expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"}, - expectedErrors: []error{}, + expectedErrors: []error{}, }, { name: "down migration invoked down methods", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_init_organizations_down `), }, expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"}, - expectedErrors: []error{}, + expectedErrors: []error{}, }, { name: "up migration: non-existing method causes migration not to execute", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_non_existing_method_up `), }, expectedInvokedMethods: []string{}, - expectedErrors: []error{ MissingMethodError("V001_some_non_existing_method_up") }, + expectedErrors: []error{MissingMethodError("V001_some_non_existing_method_up")}, }, { name: "up migration: failing method stops execution", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_some_failing_method_up V001_init_users_up @@ -146,20 +118,21 @@ func TestMigrate(t *testing.T) { "V001_init_organizations_up", "V001_some_failing_method_up", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_up", - Err: SomeError{}, + Err: SomeError{}, }}, }, { name: "down migration: failing method stops migration", - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -169,21 +142,22 @@ func TestMigrate(t *testing.T) { "V001_init_users_down", "V001_some_failing_method_down", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_down", - Err: SomeError{}, + Err: SomeError{}, }}, }, { - name: "up migration: failing method causes rollback in rollback mode", + name: "up migration: failing method causes rollback in rollback mode", expectRollback: true, - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", Version: 1, Name: "foobar", Direction: direction.Up, Content: []byte(` + FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_failing_method_up @@ -196,21 +170,22 @@ func TestMigrate(t *testing.T) { "V001_init_users_down", "V001_init_organizations_down", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_up", - Err: SomeError{}, + Err: SomeError{}, }}, }, { - name: "down migration: failing method causes rollback in rollback mode", + name: "down migration: failing method causes rollback in rollback mode", expectRollback: true, - file: file.File { + file: file.File{ Path: "/foobar", FileName: "001_foobar.down.gm", Version: 1, Name: "foobar", Direction: direction.Down, Content: []byte(` + FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -221,18 +196,20 @@ func TestMigrate(t *testing.T) { "V001_some_failing_method_down", "V001_init_users_up", }, - expectedErrors: []error{ &MethodInvocationFailedError{ + expectedErrors: []error{&MethodInvocationFailedError{ MethodName: "V001_some_failing_method_down", - Err: SomeError{}, + Err: SomeError{}, }}, }, - } + RegisterMethodsReceiver("FakeMethodsReceiver", "") + for _, c := range cases { migrator := Migrator{} - d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}} - migrator.MigrationMethodsReceiver = d + fakeInvoker := &FakeGoMethodsInvoker{InvokedMethods: []string{}} + + migrator.MethodInvoker = fakeInvoker migrator.RollbackOnFailure = c.expectRollback pipe := pipep.New() @@ -243,9 +220,9 @@ func TestMigrate(t *testing.T) { errs := pipep.ReadErrors(pipe) var failed bool - if !reflect.DeepEqual(d.InvokedMethods, c.expectedInvokedMethods) { + if !reflect.DeepEqual(fakeInvoker.InvokedMethods, c.expectedInvokedMethods) { failed = true - t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, d.InvokedMethods) + t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, fakeInvoker.InvokedMethods) } if !reflect.DeepEqual(errs, c.expectedErrors) { failed = true @@ -258,11 +235,9 @@ func TestMigrate(t *testing.T) { } } - - func TestGetRollbackToMethod(t *testing.T) { cases := []struct { - method string + method string expectedRollbackMethod string }{ {"some_method_up", "some_method_down"}, @@ -282,7 +257,7 @@ func TestGetRollbackToMethod(t *testing.T) { } func TestReverseInPlace(t *testing.T) { - methods := []string { + methods := []string{ "method1_down", "method2_down", "method3_down", @@ -290,7 +265,7 @@ func TestReverseInPlace(t *testing.T) { "method5_down", } - expectedReversedMethods := []string { + expectedReversedMethods := []string{ "method5_down", "method4_down", "method3_down", @@ -304,4 +279,3 @@ func TestReverseInPlace(t *testing.T) { t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) } } - diff --git a/driver/gomethods/gomethods_registry.go b/driver/gomethods/gomethods_registry.go new file mode 100644 index 0000000..78ac84c --- /dev/null +++ b/driver/gomethods/gomethods_registry.go @@ -0,0 +1,30 @@ +package gomethods + +import ( + "sync" +) + +var methodsReceiversMu sync.Mutex +var methodsReceivers = make(map[string]interface{}) + +// Registers a methods receiver object so it can be created from its name. Users of gomethods migration drivers should +// call this method to register objects with their migration methods before executing the migration +func RegisterMethodsReceiver(name string, receiver interface{}) { + methodsReceiversMu.Lock() + defer methodsReceiversMu.Unlock() + if receiver == nil { + panic("Go methods: Register receiver object is nil") + } + if _, dup := methodsReceivers[name]; dup { + panic("Go methods: Register called twice for receiver " + name) + } + methodsReceivers[name] = receiver +} + +// Retrieves a registered driver by name +func GetMethodsReceiver(name string) interface{} { + methodsReceiversMu.Lock() + defer methodsReceiversMu.Unlock() + receiver := methodsReceivers[name] + return receiver +} diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go new file mode 100644 index 0000000..0acad17 --- /dev/null +++ b/driver/gomethods/mongodb/mongodb.go @@ -0,0 +1,134 @@ +package mongodb + +import ( + "errors" + "github.com/dimag-jfrog/migrate/driver" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/file" + "github.com/dimag-jfrog/migrate/migrate/direction" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "reflect" + "strings" +) + +const MIGRATE_DB = "db_migrations" +const MIGRATE_C = "db_migrations" + +type MongoDbGoMethodsDriver struct { + Session *mgo.Session + DbName string + + migrator gomethods.Migrator +} + +func init() { + driver.RegisterDriver("mongodb", &MongoDbGoMethodsDriver{}) +} + +type DbMigration struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Version uint64 `bson:"version"` +} + +func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) + if len(urlWithoutScheme) != 2 { + return errors.New("invalid mongodb:// scheme") + } + + session, err := mgo.Dial(url) + if err != nil { + return err + } + session.SetMode(mgo.Monotonic, true) + + driver.Session = session + driver.DbName = MIGRATE_DB + driver.migrator = gomethods.Migrator{MethodInvoker: driver} + + return nil +} + +func (driver *MongoDbGoMethodsDriver) Close() error { + if driver.Session != nil { + driver.Session.Close() + } + return nil +} + +func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { + return "mgo" +} + +func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { + var latestMigration DbMigration + c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) + + switch { + case err == mgo.ErrNotFound: + return 0, nil + case err != nil: + return 0, err + default: + return latestMigration.Version, nil + } +} +func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { + defer close(pipe) + pipe <- f + + err := driver.migrator.Migrate(f, pipe) + if err != nil { + return + } + + migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + + if f.Direction == direction.Up { + id := bson.NewObjectId() + dbMigration := DbMigration{Id: id, Version: f.Version} + + err := migrate_c.Insert(dbMigration) + if err != nil { + pipe <- err + return + } + + } else if f.Direction == direction.Down { + err := migrate_c.Remove(bson.M{"version": f.Version}) + if err != nil { + pipe <- err + return + } + } +} + +func (driver *MongoDbGoMethodsDriver) IsValid(methodName string, methodsReceiver interface{}) bool { + return reflect.ValueOf(methodsReceiver).MethodByName(methodName).IsValid() +} + +func (driver *MongoDbGoMethodsDriver) Invoke(methodName string, methodsReceiver interface{}) error { + name := methodName + migrateMethod := reflect.ValueOf(methodsReceiver).MethodByName(name) + if !migrateMethod.IsValid() { + return gomethods.MissingMethodError(methodName) + } + + retValues := migrateMethod.Call([]reflect.Value{reflect.ValueOf(driver.Session)}) + if len(retValues) != 1 { + return gomethods.WrongMethodSignatureError(name) + } + + if !retValues[0].IsNil() { + err, ok := retValues[0].Interface().(error) + if !ok { + return gomethods.WrongMethodSignatureError(name) + } + return &gomethods.MethodInvocationFailedError{MethodName: name, Err: err} + } + + return nil +} diff --git a/driver/gomethods/mongodb/mongodb_template.go b/driver/gomethods/mongodb/mongodb_template.go deleted file mode 100644 index 1edbe78..0000000 --- a/driver/gomethods/mongodb/mongodb_template.go +++ /dev/null @@ -1,107 +0,0 @@ -package mongodb - -import ( - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "errors" - "strings" - "github.com/dimag-jfrog/migrate/migrate/direction" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/driver/gomethods" -) - -const MIGRATE_C = "db_migrations" - - -// This is not a real driver since the Initialize method requires a gomethods.Migrator -// The real driver will contain the DriverTemplate and implement all the custom migration Golang methods -// See example in usage_examples for details -type DriverTemplate struct { - Session *mgo.Session - DbName string - - migrator gomethods.Migrator -} - - -type DbMigration struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Version uint64 `bson:"version"` -} - -func (driver *DriverTemplate) Initialize(url, dbName string, migrator gomethods.Migrator) error { - urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) - if len(urlWithoutScheme) != 2 { - return errors.New("invalid mongodb:// scheme") - } - - session, err := mgo.Dial(url) - if err != nil { - return err - } - session.SetMode(mgo.Monotonic, true) - - driver.Session = session - driver.DbName = dbName - driver.migrator = migrator - - return nil -} - -func (driver *DriverTemplate) Close() error { - if driver.Session != nil { - driver.Session.Close() - } - return nil -} - -func (driver *DriverTemplate) FilenameExtension() string { - return "mgo" -} - - -func (driver *DriverTemplate) Version() (uint64, error) { - var latestMigration DbMigration - c := driver.Session.DB(driver.DbName).C(MIGRATE_C) - - - err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) - - switch { - case err == mgo.ErrNotFound: - return 0, nil - case err != nil: - return 0, err - default: - return latestMigration.Version, nil - } -} -func (driver *DriverTemplate) Migrate(f file.File, pipe chan interface{}) { - defer close(pipe) - pipe <- f - - err := driver.migrator.Migrate(f, pipe) - if err != nil { - return - } - - migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) - - if f.Direction == direction.Up { - id := bson.NewObjectId() - dbMigration := DbMigration{Id: id, Version: f.Version} - - err := migrate_c.Insert(dbMigration) - if err != nil { - pipe <- err - return - } - - } else if f.Direction == direction.Down { - err := migrate_c.Remove(bson.M{"version": f.Version}) - if err != nil { - pipe <- err - return - } - } -} diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go new file mode 100644 index 0000000..24433e9 --- /dev/null +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -0,0 +1,129 @@ +package mongodb_example + +import ( + "github.com/dimag-jfrog/migrate/driver/gomethods" + "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" + "time" +) + +type MyMgoMethodsReceiver struct { +} + +func init() { + gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) +} + +// Here goes the specific mongodb golang methods driver logic + +const DB_NAME = "test" +const SHORT_DATE_LAYOUT = "2000-Jan-01" +const USERS_C = "users" +const ORGANIZATIONS_C = "organizations" + +type Organization struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Location string `bson:"location"` + DateFounded time.Time `bson:"date_founded"` +} + +type Organization_v2 struct { + Id bson.ObjectId `bson:"_id,omitempty"` + Name string `bson:"name"` + Headquarters string `bson:"headquarters"` + DateFounded time.Time `bson:"date_founded"` +} + +type User struct { + Id bson.ObjectId `bson:"_id"` + Name string `bson:"name"` +} + +var OrganizationIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +var UserIds []bson.ObjectId = []bson.ObjectId{ + bson.NewObjectId(), + bson.NewObjectId(), + bson.NewObjectId(), +} + +func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) error { + date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") + date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") + date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") + + orgs := []Organization{ + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, + } + + for _, org := range orgs { + err := session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) + if err != nil { + return err + } + } + return nil +} + +func (r *MyMgoMethodsReceiver) V001_init_organizations_down(session *mgo.Session) error { + return session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() +} + +func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { + users := []User{ + {Id: UserIds[0], Name: "Alex"}, + {Id: UserIds[1], Name: "Beatrice"}, + {Id: UserIds[2], Name: "Cleo"}, + } + + for _, user := range users { + err := session.DB(DB_NAME).C(USERS_C).Insert(user) + if err != nil { + return err + } + } + return nil +} + +func (r *MyMgoMethodsReceiver) V001_init_users_down(session *mgo.Session) error { + return session.DB(DB_NAME).C(USERS_C).DropCollection() +} + +func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { + c := session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) + return err +} + +func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { + c := session.DB(DB_NAME).C(ORGANIZATIONS_C) + + _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) + return err +} + +func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { + c := session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleo"} + change := bson.M{"$set": bson.M{"name": "Cleopatra"}} + + return c.Update(colQuerier, change) +} + +func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { + c := session.DB(DB_NAME).C(USERS_C) + + colQuerier := bson.M{"name": "Cleopatra"} + change := bson.M{"$set": bson.M{"name": "Cleo"}} + + return c.Update(colQuerier, change) +} diff --git a/driver/gomethods/usage_examples/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go similarity index 71% rename from driver/gomethods/usage_examples/mongodb_test.go rename to driver/gomethods/mongodb_example/mongodb_test.go index a57c51d..2ef2797 100644 --- a/driver/gomethods/usage_examples/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -1,4 +1,4 @@ -package usage_examples +package mongodb_example import ( "testing" @@ -6,24 +6,24 @@ import ( "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" pipep "github.com/dimag-jfrog/migrate/pipe" "reflect" "time" - "github.com/dimag-jfrog/migrate/driver/gomethods" ) - type ExpectedMigrationResult struct { - Organizations []Organization + Organizations []Organization Organizations_v2 []Organization_v2 - Users []User - Errors []error + Users []User + Errors []error } func RunMigrationAndAssertResult( t *testing.T, title string, - d *GoMethodsMongoDbDriver, + d *mongodb.MongoDbGoMethodsDriver, file file.File, expected *ExpectedMigrationResult) { @@ -53,6 +53,10 @@ func RunMigrationAndAssertResult( t.Fatal("Failed to query Users collection") } + if !reflect.DeepEqual(expected.Errors, errs) { + t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) + } + if !reflect.DeepEqual(expected.Organizations, actualOrganizations) { t.Fatalf("Migration '%s': FAILED\nexpected organizations %v\nbut got %v", title, expected.Organizations, actualOrganizations) } @@ -65,13 +69,9 @@ func RunMigrationAndAssertResult( t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) } - if !reflect.DeepEqual(expected.Errors, errs) { - t.Fatalf("Migration '%s': FAILED\nexpected errors %v\nbut got %v", title, expected.Errors, errs) - } t.Logf("Migration '%s': PASSED", title) } - func TestMigrate(t *testing.T) { //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") @@ -79,7 +79,9 @@ func TestMigrate(t *testing.T) { port := "27017" driverUrl := "mongodb://" + host + ":" + port - d := &GoMethodsMongoDbDriver{} + //gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) + d := &mongodb.MongoDbGoMethodsDriver{} + if err := d.Initialize(driverUrl); err != nil { t.Fatal(err) } @@ -93,8 +95,8 @@ func TestMigrate(t *testing.T) { date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") migrations := []struct { - name string - file file.File + name string + file file.File expectedResult ExpectedMigrationResult }{ { @@ -106,15 +108,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up `), }, expectedResult: ExpectedMigrationResult{ Organizations: []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, }, Organizations_v2: []Organization_v2{}, Users: []User{ @@ -134,6 +137,7 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V002_organizations_rename_location_field_to_headquarters_up V002_change_user_cleo_to_cleopatra_up `), @@ -141,9 +145,9 @@ func TestMigrate(t *testing.T) { expectedResult: ExpectedMigrationResult{ Organizations: []Organization{}, Organizations_v2: []Organization_v2{ - {Id: OrganizationIds[0], Name: "Amazon", Headquarters:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Headquarters:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Headquarters:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Headquarters: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Headquarters: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Headquarters: "Santa Clara", DateFounded: date3}, }, Users: []User{ {Id: UserIds[0], Name: "Alex"}, @@ -162,15 +166,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` + MyMgoMethodsReceiver V002_change_user_cleo_to_cleopatra_down V002_organizations_rename_location_field_to_headquarters_down `), }, expectedResult: ExpectedMigrationResult{ Organizations: []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, + {Id: OrganizationIds[0], Name: "Amazon", Location: "Seattle", DateFounded: date1}, + {Id: OrganizationIds[1], Name: "Google", Location: "Mountain View", DateFounded: date2}, + {Id: OrganizationIds[2], Name: "JFrog", Location: "Santa Clara", DateFounded: date3}, }, Organizations_v2: []Organization_v2{}, Users: []User{ @@ -190,15 +195,16 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` + MyMgoMethodsReceiver V001_init_users_down V001_init_organizations_down `), }, expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, + Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{}, + Users: []User{}, + Errors: []error{}, }, }, { @@ -210,16 +216,37 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` + MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up v001_non_existing_method_up `), }, expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, + Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{ gomethods.MissingMethodError("v001_non_existing_method_up") }, + Users: []User{}, + Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, + }, + }, + { + name: "v0 -> v1: not defined message receiver", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.UnregisteredMethodsReceiverError("V001_init_organizations_up")}, }, }, } diff --git a/driver/gomethods/usage_examples/mongodb.go b/driver/gomethods/usage_examples/mongodb.go deleted file mode 100644 index e7f30c0..0000000 --- a/driver/gomethods/usage_examples/mongodb.go +++ /dev/null @@ -1,140 +0,0 @@ -package usage_examples - -import ( - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" - "gopkg.in/mgo.v2/bson" - "time" -) - -// This boilerplate part is necessary and the same -// regardless of the specific mongodb golang methods driver - -type GoMethodsMongoDbDriver struct { - mongodb.DriverTemplate -} - -func (d *GoMethodsMongoDbDriver) Initialize(url string) error { - return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{MigrationMethodsReceiver: d}) -} - -func init() { - driver.RegisterDriver("mongodb", &GoMethodsMongoDbDriver{mongodb.DriverTemplate{}}) -} - - - -// Here goes the specific mongodb golang methods driver logic - -const DB_NAME = "test" -const SHORT_DATE_LAYOUT = "2000-Jan-01" -const USERS_C = "users" -const ORGANIZATIONS_C = "organizations" - -type Organization struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Name string `bson:"name"` - Location string `bson:"location"` - DateFounded time.Time `bson:"date_founded"` -} - -type Organization_v2 struct { - Id bson.ObjectId `bson:"_id,omitempty"` - Name string `bson:"name"` - Headquarters string `bson:"headquarters"` - DateFounded time.Time `bson:"date_founded"` -} - -type User struct { - Id bson.ObjectId `bson:"_id"` - Name string `bson:"name"` -} - -var OrganizationIds []bson.ObjectId = []bson.ObjectId{ - bson.NewObjectId(), - bson.NewObjectId(), - bson.NewObjectId(), -} - -var UserIds []bson.ObjectId = []bson.ObjectId{ - bson.NewObjectId(), - bson.NewObjectId(), - bson.NewObjectId(), -} - -func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error { - date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") - date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") - date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") - - orgs := []Organization{ - {Id: OrganizationIds[0], Name: "Amazon", Location:"Seattle", DateFounded: date1}, - {Id: OrganizationIds[1], Name: "Google", Location:"Mountain View", DateFounded: date2}, - {Id: OrganizationIds[2], Name: "JFrog", Location:"Santa Clara", DateFounded: date3}, - } - - for _, org := range orgs { - err := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org) - if err != nil { - return err - } - } - return nil -} - -func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error { - return m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() -} - -func (m *GoMethodsMongoDbDriver) V001_init_users_up() error { - users := []User{ - {Id: UserIds[0], Name: "Alex"}, - {Id: UserIds[1], Name: "Beatrice"}, - {Id: UserIds[2], Name: "Cleo"}, - } - - for _, user := range users { - err := m.Session.DB(DB_NAME).C(USERS_C).Insert(user) - if err != nil { - return err - } - } - return nil -} - -func (m *GoMethodsMongoDbDriver) V001_init_users_down() error { - return m.Session.DB(DB_NAME).C(USERS_C).DropCollection() -} - -func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_up() error { - c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) - - _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) - return err -} - -func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_down() error { - c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C) - - _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) - return err -} - -func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_up() error { - c := m.Session.DB(DB_NAME).C(USERS_C) - - colQuerier := bson.M{"name": "Cleo"} - change := bson.M{"$set": bson.M{"name": "Cleopatra"}} - - return c.Update(colQuerier, change) -} - -func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_down() error { - c := m.Session.DB(DB_NAME).C(USERS_C) - - colQuerier := bson.M{"name": "Cleopatra"} - change := bson.M{"$set": bson.M{"name": "Cleo",}} - - return c.Update(colQuerier, change) -} \ No newline at end of file From b7b75f3c59f33360084268055df8d54b83de9cdc Mon Sep 17 00:00:00 2001 From: dimag Date: Tue, 9 Aug 2016 21:16:21 +0300 Subject: [PATCH 20/27] - registering method receiver directly for the driver - currently each driver contains only a single method receiver: - enforcing method receiver pre-registration on go methods driver initialization - Method receiver name can be removed from files format - passing the DbName parameter inside the method receiver for the go methods driver --- driver/gomethods/gomethods_migrator.go | 53 ++++++------------ driver/gomethods/gomethods_migrator_test.go | 13 +---- driver/gomethods/gomethods_registry.go | 39 ++++++++----- driver/gomethods/mongodb/mongodb.go | 55 +++++++++++++++---- .../mongodb_example/methods_receiver.go | 20 +++++-- .../gomethods/mongodb_example/mongodb_test.go | 40 +++++--------- 6 files changed, 117 insertions(+), 103 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 2ee0192..70d57c3 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -1,22 +1,15 @@ package gomethods import ( - //"bytes" "bufio" - "database/sql/driver" "fmt" + "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/file" "os" "path" "strings" ) -type UnregisteredMethodsReceiverError string - -func (e UnregisteredMethodsReceiverError) Error() string { - return "Unregistered methods receiver: " + string(e) -} - type MissingMethodError string func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } @@ -37,13 +30,16 @@ func (e *MethodInvocationFailedError) Error() string { } type MigrationMethodInvoker interface { - IsValid(methodName string, methodReceiver interface{}) bool - Invoke(methodName string, methodReceiver interface{}) error + IsValid(methodName string) bool + Invoke(methodName string) error } type GoMethodsDriver interface { driver.Driver + MigrationMethodInvoker + MethodsReceiver() interface{} + SetMethodsReceiver(r interface{}) error } type Migrator struct { @@ -52,7 +48,7 @@ type Migrator struct { } func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { - methods, methodsReceiver, err := m.getMigrationMethods(f) + methods, err := m.getMigrationMethods(f) if err != nil { pipe <- err return err @@ -60,7 +56,7 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for i, methodName := range methods { pipe <- methodName - err := m.MethodInvoker.Invoke(methodName, methodsReceiver) + err := m.MethodInvoker.Invoke(methodName) if err != nil { pipe <- err if !m.RollbackOnFailure { @@ -71,12 +67,12 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) if rollbackToMethodName == "" || - !m.MethodInvoker.IsValid(rollbackToMethodName, methodsReceiver) { + !m.MethodInvoker.IsValid(rollbackToMethodName) { continue } pipe <- rollbackToMethodName - err = m.MethodInvoker.Invoke(rollbackToMethodName, methodsReceiver) + err = m.MethodInvoker.Invoke(rollbackToMethodName) if err != nil { pipe <- err break @@ -122,20 +118,17 @@ func getFileLines(file file.File) ([]string, error) { } return lines, nil } else { - //n := bytes.IndexByte(file.Content, 0) - //n := bytes.Index(file.Content, []byte{0}) - //s := string(file.Content[:n]) s := string(file.Content) return strings.Split(s, "\n"), nil } } -func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsReceiver interface{}, err error) { +func (m *Migrator) getMigrationMethods(f file.File) (methods []string, err error) { var lines []string lines, err = getFileLines(f) if err != nil { - return nil, nil, err + return nil, err } for _, line := range lines { @@ -146,24 +139,14 @@ func (m *Migrator) getMigrationMethods(f file.File) (methods []string, methodsRe continue } - if methodsReceiver == nil { - receiverName := line - methodsReceiver = GetMethodsReceiver(receiverName) - if methodsReceiver == nil { - return nil, nil, UnregisteredMethodsReceiverError(receiverName) - } - continue - - } else { - methodName := line - if !m.MethodInvoker.IsValid(methodName, methodsReceiver) { - return nil, nil, MissingMethodError(methodName) - } - - methods = append(methods, methodName) + methodName := line + if !m.MethodInvoker.IsValid(methodName) { + return nil, MissingMethodError(methodName) } + + methods = append(methods, methodName) } - return methods, methodsReceiver, nil + return methods, nil } diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index eb33103..db2a817 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -14,7 +14,7 @@ type FakeGoMethodsInvoker struct { InvokedMethods []string } -func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver interface{}) bool { +func (invoker *FakeGoMethodsInvoker) IsValid(methodName string) bool { if methodName == "V001_some_non_existing_method_up" { return false } @@ -22,7 +22,7 @@ func (invoker *FakeGoMethodsInvoker) IsValid(methodName string, methodReceiver i return true } -func (invoker *FakeGoMethodsInvoker) Invoke(methodName string, methodReceiver interface{}) error { +func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { invoker.InvokedMethods = append(invoker.InvokedMethods, methodName) if methodName == "V001_some_failing_method_up" || methodName == "V001_some_failing_method_down" { @@ -56,7 +56,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up `), @@ -73,7 +72,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_init_organizations_down `), @@ -90,7 +88,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_non_existing_method_up @@ -108,7 +105,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_some_failing_method_up V001_init_users_up @@ -132,7 +128,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -157,7 +152,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - FakeMethodsReceiver V001_init_organizations_up V001_init_users_up V001_some_failing_method_up @@ -185,7 +179,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - FakeMethodsReceiver V001_init_users_down V001_some_failing_method_down V001_init_organizations_down @@ -203,8 +196,6 @@ func TestMigrate(t *testing.T) { }, } - RegisterMethodsReceiver("FakeMethodsReceiver", "") - for _, c := range cases { migrator := Migrator{} fakeInvoker := &FakeGoMethodsInvoker{InvokedMethods: []string{}} diff --git a/driver/gomethods/gomethods_registry.go b/driver/gomethods/gomethods_registry.go index 78ac84c..5ceb128 100644 --- a/driver/gomethods/gomethods_registry.go +++ b/driver/gomethods/gomethods_registry.go @@ -1,30 +1,39 @@ package gomethods import ( + "fmt" + "github.com/dimag-jfrog/migrate/driver" "sync" ) var methodsReceiversMu sync.Mutex -var methodsReceivers = make(map[string]interface{}) -// Registers a methods receiver object so it can be created from its name. Users of gomethods migration drivers should -// call this method to register objects with their migration methods before executing the migration -func RegisterMethodsReceiver(name string, receiver interface{}) { +// Registers a methods receiver for go methods driver +// Users of gomethods migration drivers should call this method +// to register objects with their migration methods before executing the migration +func RegisterMethodsReceiverForDriver(driverName string, receiver interface{}) { methodsReceiversMu.Lock() defer methodsReceiversMu.Unlock() if receiver == nil { panic("Go methods: Register receiver object is nil") } - if _, dup := methodsReceivers[name]; dup { - panic("Go methods: Register called twice for receiver " + name) - } - methodsReceivers[name] = receiver -} -// Retrieves a registered driver by name -func GetMethodsReceiver(name string) interface{} { - methodsReceiversMu.Lock() - defer methodsReceiversMu.Unlock() - receiver := methodsReceivers[name] - return receiver + driver := driver.GetDriver(driverName) + if driver == nil { + panic("Go methods: Trying to register receiver for not registered driver " + driverName) + } + + methodsDriver, ok := driver.(GoMethodsDriver) + if !ok { + panic("Go methods: Trying to register receiver for non go methods driver " + driverName) + } + + if methodsDriver.MethodsReceiver() != nil { + panic("Go methods: Methods receiver already registered for driver " + driverName) + } + + if err := methodsDriver.SetMethodsReceiver(receiver); err != nil { + panic(fmt.Sprintf("Go methods: Failed to set methods receiver for driver %s\nError: %v", + driverName, err)) + } } diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go index 0acad17..acbc256 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/gomethods/mongodb/mongodb.go @@ -12,14 +12,46 @@ import ( "strings" ) -const MIGRATE_DB = "db_migrations" +type UnregisteredMethodsReceiverError string + +func (e UnregisteredMethodsReceiverError) Error() string { + return "Unregistered methods receiver for driver: " + string(e) +} + +type WrongMethodsReceiverTypeError string + +func (e WrongMethodsReceiverTypeError) Error() string { + return "Wrong methods receiver type for driver: " + string(e) +} + const MIGRATE_C = "db_migrations" +const DRIVER_NAME = "gomethods.mongodb" type MongoDbGoMethodsDriver struct { Session *mgo.Session - DbName string - migrator gomethods.Migrator + methodsReceiver MethodsReceiver + migrator gomethods.Migrator +} + +var _ gomethods.GoMethodsDriver = (*MongoDbGoMethodsDriver)(nil) + +type MethodsReceiver interface { + DbName() string +} + +func (d *MongoDbGoMethodsDriver) MethodsReceiver() interface{} { + return d.methodsReceiver +} + +func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { + r1, ok := r.(MethodsReceiver) + if !ok { + return WrongMethodsReceiverTypeError(DRIVER_NAME) + } + + d.methodsReceiver = r1 + return nil } func init() { @@ -32,6 +64,10 @@ type DbMigration struct { } func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { + if driver.methodsReceiver == nil { + return UnregisteredMethodsReceiverError(DRIVER_NAME) + } + urlWithoutScheme := strings.SplitN(url, "mongodb://", 2) if len(urlWithoutScheme) != 2 { return errors.New("invalid mongodb:// scheme") @@ -44,7 +80,6 @@ func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { session.SetMode(mgo.Monotonic, true) driver.Session = session - driver.DbName = MIGRATE_DB driver.migrator = gomethods.Migrator{MethodInvoker: driver} return nil @@ -63,7 +98,7 @@ func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { var latestMigration DbMigration - c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) err := c.Find(bson.M{}).Sort("-version").One(&latestMigration) @@ -85,7 +120,7 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} return } - migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C) + migrate_c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) if f.Direction == direction.Up { id := bson.NewObjectId() @@ -106,13 +141,13 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) IsValid(methodName string, methodsReceiver interface{}) bool { - return reflect.ValueOf(methodsReceiver).MethodByName(methodName).IsValid() +func (driver *MongoDbGoMethodsDriver) IsValid(methodName string) bool { + return reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName).IsValid() } -func (driver *MongoDbGoMethodsDriver) Invoke(methodName string, methodsReceiver interface{}) error { +func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { name := methodName - migrateMethod := reflect.ValueOf(methodsReceiver).MethodByName(name) + migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { return gomethods.MissingMethodError(methodName) } diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go index 24433e9..6749ae7 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -2,6 +2,8 @@ package mongodb_example import ( "github.com/dimag-jfrog/migrate/driver/gomethods" + _ "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" @@ -10,16 +12,24 @@ import ( type MyMgoMethodsReceiver struct { } +func (r *MyMgoMethodsReceiver) DbName() string { + return DB_NAME +} + +var _ mongodb.MethodsReceiver = (*MyMgoMethodsReceiver)(nil) + func init() { - gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) + gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMgoMethodsReceiver{}) } // Here goes the specific mongodb golang methods driver logic -const DB_NAME = "test" -const SHORT_DATE_LAYOUT = "2000-Jan-01" -const USERS_C = "users" -const ORGANIZATIONS_C = "organizations" +const ( + DB_NAME = "test" + SHORT_DATE_LAYOUT = "2000-Jan-01" + USERS_C = "users" + ORGANIZATIONS_C = "organizations" +) type Organization struct { Id bson.ObjectId `bson:"_id,omitempty"` diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go index 2ef2797..fc21c9c 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -6,6 +6,7 @@ import ( "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/dimag-jfrog/migrate/driver" "github.com/dimag-jfrog/migrate/driver/gomethods" "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" pipep "github.com/dimag-jfrog/migrate/pipe" @@ -73,20 +74,30 @@ func RunMigrationAndAssertResult( } func TestMigrate(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Test failed on panic: %v", r) + } + }() + //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") host := "127.0.0.1" port := "27017" driverUrl := "mongodb://" + host + ":" + port - //gomethods.RegisterMethodsReceiver("MyMgoMethodsReceiver", &MyMgoMethodsReceiver{}) - d := &mongodb.MongoDbGoMethodsDriver{} + d0 := driver.GetDriver("mongodb") + d, ok := d0.(*mongodb.MongoDbGoMethodsDriver) + if !ok { + t.Fatal("MongoDbGoMethodsDriver has not registered") + } if err := d.Initialize(driverUrl); err != nil { t.Fatal(err) } // Reset DB + d.Session.DB(DB_NAME).C(mongodb.MIGRATE_C).DropCollection() d.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() d.Session.DB(DB_NAME).C(USERS_C).DropCollection() @@ -108,7 +119,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up `), @@ -137,7 +147,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V002_organizations_rename_location_field_to_headquarters_up V002_change_user_cleo_to_cleopatra_up `), @@ -166,7 +175,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - MyMgoMethodsReceiver V002_change_user_cleo_to_cleopatra_down V002_organizations_rename_location_field_to_headquarters_down `), @@ -195,7 +203,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Down, Content: []byte(` - MyMgoMethodsReceiver V001_init_users_down V001_init_organizations_down `), @@ -216,7 +223,6 @@ func TestMigrate(t *testing.T) { Name: "foobar", Direction: direction.Up, Content: []byte(` - MyMgoMethodsReceiver V001_init_organizations_up V001_init_users_up v001_non_existing_method_up @@ -229,26 +235,6 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, - { - name: "v0 -> v1: not defined message receiver", - file: file.File{ - Path: "/foobar", - FileName: "001_foobar.up.gm", - Version: 1, - Name: "foobar", - Direction: direction.Up, - Content: []byte(` - V001_init_organizations_up - V001_init_users_up - `), - }, - expectedResult: ExpectedMigrationResult{ - Organizations: []Organization{}, - Organizations_v2: []Organization_v2{}, - Users: []User{}, - Errors: []error{gomethods.UnregisteredMethodsReceiverError("V001_init_organizations_up")}, - }, - }, } for _, m := range migrations { From 0eecb742305510ed0cb116003e9cda4cf6af207d Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 14:13:53 +0300 Subject: [PATCH 21/27] Added explicit migration method signature validation --- driver/gomethods/gomethods_migrator.go | 14 +++--- driver/gomethods/gomethods_migrator_test.go | 9 ++-- driver/gomethods/mongodb/mongodb.go | 15 ++++++- .../mongodb_example/methods_receiver.go | 9 ++++ .../gomethods/mongodb_example/mongodb_test.go | 44 ++++++++++++++++++- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/driver/gomethods/gomethods_migrator.go b/driver/gomethods/gomethods_migrator.go index 70d57c3..a690035 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/gomethods/gomethods_migrator.go @@ -17,7 +17,7 @@ func (e MissingMethodError) Error() string { return "Non existing migrate method type WrongMethodSignatureError string func (e WrongMethodSignatureError) Error() string { - return fmt.Sprintf("Method %s has wrong signature", e) + return fmt.Sprintf("Method %s has wrong signature", string(e)) } type MethodInvocationFailedError struct { @@ -30,7 +30,7 @@ func (e *MethodInvocationFailedError) Error() string { } type MigrationMethodInvoker interface { - IsValid(methodName string) bool + Validate(methodName string) error Invoke(methodName string) error } @@ -66,8 +66,10 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { // on failure, try to rollback methods in this migration for j := i - 1; j >= 0; j-- { rollbackToMethodName := getRollbackToMethod(methods[j]) - if rollbackToMethodName == "" || - !m.MethodInvoker.IsValid(rollbackToMethodName) { + if rollbackToMethodName == "" { + continue + } + if err := m.MethodInvoker.Validate(rollbackToMethodName); err != nil { continue } @@ -140,8 +142,8 @@ func (m *Migrator) getMigrationMethods(f file.File) (methods []string, err error } methodName := line - if !m.MethodInvoker.IsValid(methodName) { - return nil, MissingMethodError(methodName) + if err := m.MethodInvoker.Validate(methodName); err != nil { + return nil, err } methods = append(methods, methodName) diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/gomethods/gomethods_migrator_test.go index db2a817..9dafb8a 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/gomethods/gomethods_migrator_test.go @@ -14,12 +14,12 @@ type FakeGoMethodsInvoker struct { InvokedMethods []string } -func (invoker *FakeGoMethodsInvoker) IsValid(methodName string) bool { +func (invoker *FakeGoMethodsInvoker) Validate(methodName string) error { if methodName == "V001_some_non_existing_method_up" { - return false + return MissingMethodError(methodName) } - return true + return nil } func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { @@ -30,9 +30,8 @@ func (invoker *FakeGoMethodsInvoker) Invoke(methodName string) error { MethodName: methodName, Err: SomeError{}, } - } else { - return nil } + return nil } type SomeError struct{} diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/gomethods/mongodb/mongodb.go index acbc256..f07b38b 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/gomethods/mongodb/mongodb.go @@ -141,8 +141,19 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) IsValid(methodName string) bool { - return reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName).IsValid() +func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { + method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) + if !method.IsValid() { + return gomethods.MissingMethodError(methodName) + } + + methodTemplate := func(*mgo.Session) error { return nil } + + if method.Type() != reflect.TypeOf(methodTemplate) { + return gomethods.WrongMethodSignatureError(methodName) + } + + return nil } func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/gomethods/mongodb_example/methods_receiver.go index 6749ae7..ae96b71 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/gomethods/mongodb_example/methods_receiver.go @@ -137,3 +137,12 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session * return c.Update(colQuerier, change) } + +// Wrong signature methods for testing +func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_up(s string) error { + return nil +} + +func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { + return true, nil +} diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/gomethods/mongodb_example/mongodb_test.go index fc21c9c..18862af 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/gomethods/mongodb_example/mongodb_test.go @@ -215,7 +215,7 @@ func TestMigrate(t *testing.T) { }, }, { - name: "v0 -> v1: with error", + name: "v0 -> v1: missing method aborts migration", file: file.File{ Path: "/foobar", FileName: "001_foobar.up.gm", @@ -235,6 +235,48 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, + { + name: "v0 -> v1: wrong signature method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + V001_method_with_wrong_signature_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.WrongMethodSignatureError("V001_method_with_wrong_signature_up")}, + }, + }, + { + name: "v1 -> v0: wrong signature method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.down.gm", + Version: 1, + Name: "foobar", + Direction: direction.Down, + Content: []byte(` + V001_init_users_down + V001_method_with_wrong_signature_down + V001_init_organizations_down + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.WrongMethodSignatureError("V001_method_with_wrong_signature_down")}, + }, + }, } for _, m := range migrations { From 61ecf2c096e2d323f2fde64a544ad2975a78d51f Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 17:15:17 +0300 Subject: [PATCH 22/27] - Refactored project structure to be consistent with original project - Added docker support --- docker-compose.yml | 6 ++- .../example}/mongodb_test.go | 20 +++++----- .../example/sample_mongdb_migrator.go} | 37 ++++++++++--------- .../gomethods/gomethods_migrator.go | 7 ---- .../gomethods/gomethods_migrator_test.go | 26 +------------ .../gomethods/gomethods_registry.go | 0 driver/{gomethods => }/mongodb/mongodb.go | 26 ++++++------- 7 files changed, 48 insertions(+), 74 deletions(-) rename driver/{gomethods/mongodb_example => mongodb/example}/mongodb_test.go (95%) rename driver/{gomethods/mongodb_example/methods_receiver.go => mongodb/example/sample_mongdb_migrator.go} (66%) rename driver/{ => mongodb}/gomethods/gomethods_migrator.go (96%) rename driver/{ => mongodb}/gomethods/gomethods_migrator_test.go (92%) rename driver/{ => mongodb}/gomethods/gomethods_registry.go (100%) rename driver/{gomethods => }/mongodb/mongodb.go (80%) diff --git a/docker-compose.yml b/docker-compose.yml index 2b13d8f..78d4a96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ go: &go image: golang - working_dir: /go/src/github.com/mattes/migrate + working_dir: /go/src/github.com/dimag-jfrog/migrate volumes: - $GOPATH:/go go-test: @@ -11,6 +11,7 @@ go-test: - mysql - cassandra - crate + - mongo go-build: <<: *go command: sh -c 'go get -v && go build -ldflags ''-s'' -o migrater' @@ -27,3 +28,6 @@ cassandra: image: cassandra:2.2 crate: image: crate +mongo: + image: mongo + diff --git a/driver/gomethods/mongodb_example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go similarity index 95% rename from driver/gomethods/mongodb_example/mongodb_test.go rename to driver/mongodb/example/mongodb_test.go index 18862af..c8013d4 100644 --- a/driver/gomethods/mongodb_example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -1,4 +1,4 @@ -package mongodb_example +package example import ( "testing" @@ -7,9 +7,10 @@ import ( "github.com/dimag-jfrog/migrate/migrate/direction" "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" pipep "github.com/dimag-jfrog/migrate/pipe" + "os" "reflect" "time" ) @@ -24,7 +25,7 @@ type ExpectedMigrationResult struct { func RunMigrationAndAssertResult( t *testing.T, title string, - d *mongodb.MongoDbGoMethodsDriver, + d *mongodb.Driver, file file.File, expected *ExpectedMigrationResult) { @@ -70,7 +71,7 @@ func RunMigrationAndAssertResult( t.Fatalf("Migration '%s': FAILED\nexpected users %v\nbut got %v", title, expected.Users, actualUsers) } - t.Logf("Migration '%s': PASSED", title) + // t.Logf("Migration '%s': PASSED", title) } func TestMigrate(t *testing.T) { @@ -80,14 +81,13 @@ func TestMigrate(t *testing.T) { } }() - //host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR") - //port := os.Getenv("MONGODB_PORT_27017_TCP_PORT") - host := "127.0.0.1" - port := "27017" + host := os.Getenv("MONGO_PORT_27017_TCP_ADDR") + port := os.Getenv("MONGO_PORT_27017_TCP_PORT") + driverUrl := "mongodb://" + host + ":" + port d0 := driver.GetDriver("mongodb") - d, ok := d0.(*mongodb.MongoDbGoMethodsDriver) + d, ok := d0.(*mongodb.Driver) if !ok { t.Fatal("MongoDbGoMethodsDriver has not registered") } diff --git a/driver/gomethods/mongodb_example/methods_receiver.go b/driver/mongodb/example/sample_mongdb_migrator.go similarity index 66% rename from driver/gomethods/mongodb_example/methods_receiver.go rename to driver/mongodb/example/sample_mongdb_migrator.go index ae96b71..8750969 100644 --- a/driver/gomethods/mongodb_example/methods_receiver.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -1,25 +1,26 @@ -package mongodb_example +package example import ( - "github.com/dimag-jfrog/migrate/driver/gomethods" - _ "github.com/dimag-jfrog/migrate/driver/gomethods" - "github.com/dimag-jfrog/migrate/driver/gomethods/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + _ "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" + + "github.com/dimag-jfrog/migrate/driver/mongodb" ) -type MyMgoMethodsReceiver struct { +type SampleMongoDbMigrator struct { } -func (r *MyMgoMethodsReceiver) DbName() string { +func (r *SampleMongoDbMigrator) DbName() string { return DB_NAME } -var _ mongodb.MethodsReceiver = (*MyMgoMethodsReceiver)(nil) +var _ mongodb.MethodsReceiver = (*SampleMongoDbMigrator)(nil) func init() { - gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMgoMethodsReceiver{}) + gomethods.RegisterMethodsReceiverForDriver("mongodb", &SampleMongoDbMigrator{}) } // Here goes the specific mongodb golang methods driver logic @@ -62,7 +63,7 @@ var UserIds []bson.ObjectId = []bson.ObjectId{ bson.NewObjectId(), } -func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_organizations_up(session *mgo.Session) error { date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05") date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04") date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28") @@ -82,11 +83,11 @@ func (r *MyMgoMethodsReceiver) V001_init_organizations_up(session *mgo.Session) return nil } -func (r *MyMgoMethodsReceiver) V001_init_organizations_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_organizations_down(session *mgo.Session) error { return session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection() } -func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_users_up(session *mgo.Session) error { users := []User{ {Id: UserIds[0], Name: "Alex"}, {Id: UserIds[1], Name: "Beatrice"}, @@ -102,25 +103,25 @@ func (r *MyMgoMethodsReceiver) V001_init_users_up(session *mgo.Session) error { return nil } -func (r *MyMgoMethodsReceiver) V001_init_users_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V001_init_users_down(session *mgo.Session) error { return session.DB(DB_NAME).C(USERS_C).DropCollection() } -func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_organizations_rename_location_field_to_headquarters_up(session *mgo.Session) error { c := session.DB(DB_NAME).C(ORGANIZATIONS_C) _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}}) return err } -func (r *MyMgoMethodsReceiver) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_organizations_rename_location_field_to_headquarters_down(session *mgo.Session) error { c := session.DB(DB_NAME).C(ORGANIZATIONS_C) _, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}}) return err } -func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_up(session *mgo.Session) error { c := session.DB(DB_NAME).C(USERS_C) colQuerier := bson.M{"name": "Cleo"} @@ -129,7 +130,7 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_up(session *mg return c.Update(colQuerier, change) } -func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { +func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_down(session *mgo.Session) error { c := session.DB(DB_NAME).C(USERS_C) colQuerier := bson.M{"name": "Cleopatra"} @@ -139,10 +140,10 @@ func (r *MyMgoMethodsReceiver) V002_change_user_cleo_to_cleopatra_down(session * } // Wrong signature methods for testing -func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_up(s string) error { +func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_up(s string) error { return nil } -func (r *MyMgoMethodsReceiver) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { +func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_down(session *mgo.Session) (bool, error) { return true, nil } diff --git a/driver/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go similarity index 96% rename from driver/gomethods/gomethods_migrator.go rename to driver/mongodb/gomethods/gomethods_migrator.go index a690035..4f99afe 100644 --- a/driver/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -87,13 +87,6 @@ func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error { return nil } -func reverseInPlace(a []string) { - for i := 0; i < len(a)/2; i++ { - j := len(a) - i - 1 - a[i], a[j] = a[j], a[i] - } -} - func getRollbackToMethod(methodName string) string { if strings.HasSuffix(methodName, "_up") { return strings.TrimSuffix(methodName, "_up") + "_down" diff --git a/driver/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go similarity index 92% rename from driver/gomethods/gomethods_migrator_test.go rename to driver/mongodb/gomethods/gomethods_migrator_test.go index 9dafb8a..fe6865e 100644 --- a/driver/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -220,7 +220,7 @@ func TestMigrate(t *testing.T) { } if !failed { - t.Logf("case '%s': PASSED", c.name) + //t.Logf("case '%s': PASSED", c.name) } } } @@ -245,27 +245,3 @@ func TestGetRollbackToMethod(t *testing.T) { } } } - -func TestReverseInPlace(t *testing.T) { - methods := []string{ - "method1_down", - "method2_down", - "method3_down", - "method4_down", - "method5_down", - } - - expectedReversedMethods := []string{ - "method5_down", - "method4_down", - "method3_down", - "method2_down", - "method1_down", - } - - reverseInPlace(methods) - - if !reflect.DeepEqual(methods, expectedReversedMethods) { - t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods) - } -} diff --git a/driver/gomethods/gomethods_registry.go b/driver/mongodb/gomethods/gomethods_registry.go similarity index 100% rename from driver/gomethods/gomethods_registry.go rename to driver/mongodb/gomethods/gomethods_registry.go diff --git a/driver/gomethods/mongodb/mongodb.go b/driver/mongodb/mongodb.go similarity index 80% rename from driver/gomethods/mongodb/mongodb.go rename to driver/mongodb/mongodb.go index f07b38b..ac27872 100644 --- a/driver/gomethods/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -3,7 +3,7 @@ package mongodb import ( "errors" "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/gomethods" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" "github.com/dimag-jfrog/migrate/file" "github.com/dimag-jfrog/migrate/migrate/direction" "gopkg.in/mgo.v2" @@ -27,24 +27,24 @@ func (e WrongMethodsReceiverTypeError) Error() string { const MIGRATE_C = "db_migrations" const DRIVER_NAME = "gomethods.mongodb" -type MongoDbGoMethodsDriver struct { +type Driver struct { Session *mgo.Session methodsReceiver MethodsReceiver migrator gomethods.Migrator } -var _ gomethods.GoMethodsDriver = (*MongoDbGoMethodsDriver)(nil) +var _ gomethods.GoMethodsDriver = (*Driver)(nil) type MethodsReceiver interface { DbName() string } -func (d *MongoDbGoMethodsDriver) MethodsReceiver() interface{} { +func (d *Driver) MethodsReceiver() interface{} { return d.methodsReceiver } -func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { +func (d *Driver) SetMethodsReceiver(r interface{}) error { r1, ok := r.(MethodsReceiver) if !ok { return WrongMethodsReceiverTypeError(DRIVER_NAME) @@ -55,7 +55,7 @@ func (d *MongoDbGoMethodsDriver) SetMethodsReceiver(r interface{}) error { } func init() { - driver.RegisterDriver("mongodb", &MongoDbGoMethodsDriver{}) + driver.RegisterDriver("mongodb", &Driver{}) } type DbMigration struct { @@ -63,7 +63,7 @@ type DbMigration struct { Version uint64 `bson:"version"` } -func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { +func (driver *Driver) Initialize(url string) error { if driver.methodsReceiver == nil { return UnregisteredMethodsReceiverError(DRIVER_NAME) } @@ -85,18 +85,18 @@ func (driver *MongoDbGoMethodsDriver) Initialize(url string) error { return nil } -func (driver *MongoDbGoMethodsDriver) Close() error { +func (driver *Driver) Close() error { if driver.Session != nil { driver.Session.Close() } return nil } -func (driver *MongoDbGoMethodsDriver) FilenameExtension() string { +func (driver *Driver) FilenameExtension() string { return "mgo" } -func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { +func (driver *Driver) Version() (uint64, error) { var latestMigration DbMigration c := driver.Session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) @@ -111,7 +111,7 @@ func (driver *MongoDbGoMethodsDriver) Version() (uint64, error) { return latestMigration.Version, nil } } -func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) { +func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { defer close(pipe) pipe <- f @@ -141,7 +141,7 @@ func (driver *MongoDbGoMethodsDriver) Migrate(f file.File, pipe chan interface{} } } -func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { +func (driver *Driver) Validate(methodName string) error { method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) if !method.IsValid() { return gomethods.MissingMethodError(methodName) @@ -156,7 +156,7 @@ func (driver *MongoDbGoMethodsDriver) Validate(methodName string) error { return nil } -func (driver *MongoDbGoMethodsDriver) Invoke(methodName string) error { +func (driver *Driver) Invoke(methodName string) error { name := methodName migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { From 583921e8bf0469aef22f42e20c2c2a4b675b2330 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 20:39:41 +0300 Subject: [PATCH 23/27] Added README.md --- driver/mongodb/README.md | 104 ++++++++++++++++++ .../mongodb/example/sample_mongdb_migrator.go | 2 +- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 driver/mongodb/README.md diff --git a/driver/mongodb/README.md b/driver/mongodb/README.md new file mode 100644 index 0000000..fae3ffe --- /dev/null +++ b/driver/mongodb/README.md @@ -0,0 +1,104 @@ +# MongoDB Driver + +* Runs pre-registered Golang methods that receive a single `*mgo.Session` parameter and return `error` on failure. +* Stores migration version details in collection ``db_migrations``. + This collection will be auto-generated. +* Migrations do not run in transactions, there are no built-in transactions in MongoDB. + That means that if a migration fails, it will not be rolled back. +* There is no out-of-the-box support for command-line interface via terminal. + +## Usage in Go + +```go +import "github.com/dimag-jfrog/migrate/migrate" + +// Import your migration methods package so that they are registered and available for the MongoDB driver. +// There is no need to import the MongoDB driver explicitly, as it should already be imported by your migration methods package. +import _ "my_mongo_db_migrator" + +// use synchronous versions of migration functions ... +allErrors, ok := migrate.UpSync("mongodb://host:port", "./path") +if !ok { + fmt.Println("Oh no ...") + // do sth with allErrors slice +} + +// use the asynchronous version of migration functions ... +pipe := migrate.NewPipe() +go migrate.Up(pipe, "mongodb://host:port", "./path") +// pipe is basically just a channel +// write your own channel listener. see writePipe() in main.go as an example. +``` + +## Migration files format + +The migration files should have an ".mgo" extension and contain a list of registered methods names. + +Migration methods should satisfy the following: +* They should be exported (their name should start with a capital letter) +* Their type should be `func (*mgo.Session) error` + +Recommended (but not required) naming conventions for migration methods: +* Prefix with V : for example V001 for version 1. +* Suffix with "_up" or "_down" for up and down migrations correspondingly. + +001_first_release.up.mgo +``` +V001_some_migration_operation_up +V001_some_other_operation_up +... +``` + +001_first_release.down.mgo +``` +V001_some_other_operation_down +V001_some_migration_operation_down +... +``` + + +## Methods registration + +For a detailed example see: [sample_mongodb_migrator.go](https://github.com/dimag-jfrog/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) + +```go +package my_mongo_db_migrator + +import ( + "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "gopkg.in/mgo.v2" +) + +// common boilerplate +type MyMongoDbMigrator struct { +} + +func (r *MyMongoDbMigrator) DbName() string { + return "" +} + +var _ mongodb.MethodsReceiver = (*MyMongoDbMigrator)(nil) + +func init() { + gomethods.RegisterMethodsReceiverForDriver("mongodb", &MyMongoDbMigrator{}) +} + + +// Here goes the application-specific migration logic +func (r *MyMongoDbMigrator) V001_some_migration_operation_up(session *mgo.Session) error { + // do something + return nil +} + +func (r *MyMongoDbMigrator) V001_some_migration_operation_down(session *mgo.Session) error { + // revert some_migration_operation_up from above + return nil +} + +``` + +## Authors + +* Demitry Gershovich, https://github.com/dimag-jfrog + diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index 8750969..0831777 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -23,7 +23,7 @@ func init() { gomethods.RegisterMethodsReceiverForDriver("mongodb", &SampleMongoDbMigrator{}) } -// Here goes the specific mongodb golang methods driver logic +// Here goes the specific mongodb golang methods migration logic const ( DB_NAME = "test" From 6909594c0ba8bf7e82d0a3e54c2295ce68c3ec9f Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 21:25:36 +0300 Subject: [PATCH 24/27] Checking non-exported migration method --- driver/mongodb/example/mongodb_test.go | 21 +++++++++++++++++++ .../mongodb/example/sample_mongdb_migrator.go | 4 ++++ .../mongodb/gomethods/gomethods_migrator.go | 6 ++++++ driver/mongodb/mongodb.go | 10 ++++++--- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index c8013d4..33cecb9 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -235,6 +235,27 @@ func TestMigrate(t *testing.T) { Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, }, }, + { + name: "v0 -> v1: not exported method aborts migration", + file: file.File{ + Path: "/foobar", + FileName: "001_foobar.up.gm", + Version: 1, + Name: "foobar", + Direction: direction.Up, + Content: []byte(` + V001_init_organizations_up + v001_not_exported_method_up + V001_init_users_up + `), + }, + expectedResult: ExpectedMigrationResult{ + Organizations: []Organization{}, + Organizations_v2: []Organization_v2{}, + Users: []User{}, + Errors: []error{gomethods.MethodNotExportedError("v001_not_exported_method_up")}, + }, + }, { name: "v0 -> v1: wrong signature method aborts migration", file: file.File{ diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index 0831777..f251e5c 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -140,6 +140,10 @@ func (r *SampleMongoDbMigrator) V002_change_user_cleo_to_cleopatra_down(session } // Wrong signature methods for testing +func (r *SampleMongoDbMigrator) v001_not_exported_method_up(session *mgo.Session) error { + return nil +} + func (r *SampleMongoDbMigrator) V001_method_with_wrong_signature_up(s string) error { return nil } diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 4f99afe..225f757 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -20,6 +20,12 @@ func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", string(e)) } +type MethodNotExportedError string + +func (e MethodNotExportedError) Error() string { + return fmt.Sprintf("Method %s is not exported", string(e)) +} + type MethodInvocationFailedError struct { MethodName string Err error diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index ac27872..d077e98 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -142,14 +142,18 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { } func (driver *Driver) Validate(methodName string) error { - method := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) - if !method.IsValid() { + methodWithReceiver, ok := reflect.TypeOf(driver.methodsReceiver).MethodByName(methodName) + if !ok { return gomethods.MissingMethodError(methodName) } + if methodWithReceiver.PkgPath != "" { + return gomethods.MethodNotExportedError(methodName) + } + methodFunc := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) methodTemplate := func(*mgo.Session) error { return nil } - if method.Type() != reflect.TypeOf(methodTemplate) { + if methodFunc.Type() != reflect.TypeOf(methodTemplate) { return gomethods.WrongMethodSignatureError(methodName) } From 0f28b01383a0edf51675a9ddc65911ee5de92c68 Mon Sep 17 00:00:00 2001 From: dimag Date: Wed, 10 Aug 2016 21:33:36 +0300 Subject: [PATCH 25/27] specifying version of mongo --- docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 78d4a96..1f619cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,5 +29,4 @@ cassandra: crate: image: crate mongo: - image: mongo - + image: mongo:3.2.6 From 914c38e379795439e4a1fc367f470635af1dff8a Mon Sep 17 00:00:00 2001 From: dimag Date: Sat, 7 Jan 2017 16:08:09 +0200 Subject: [PATCH 26/27] - Fixed test with not-exported method identified as missing: unified the 2 errors into one - Fixed error message formatting - Ensuring uniqueness of the version field (added unique index) --- driver/mongodb/example/mongodb_test.go | 4 ++-- driver/mongodb/gomethods/gomethods_migrator.go | 16 ++++++---------- .../gomethods/gomethods_migrator_test.go | 4 ++-- driver/mongodb/mongodb.go | 17 +++++++++++++---- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index 33cecb9..c4371bf 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -232,7 +232,7 @@ func TestMigrate(t *testing.T) { Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, Users: []User{}, - Errors: []error{gomethods.MissingMethodError("v001_non_existing_method_up")}, + Errors: []error{gomethods.MethodNotFoundError("v001_non_existing_method_up")}, }, }, { @@ -253,7 +253,7 @@ func TestMigrate(t *testing.T) { Organizations: []Organization{}, Organizations_v2: []Organization_v2{}, Users: []User{}, - Errors: []error{gomethods.MethodNotExportedError("v001_not_exported_method_up")}, + Errors: []error{gomethods.MethodNotFoundError("v001_not_exported_method_up")}, }, }, { diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 225f757..96fc1d5 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -10,20 +10,16 @@ import ( "strings" ) -type MissingMethodError string +type MethodNotFoundError string -func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) } +func (e MethodNotFoundError) Error() string { + return fmt.Sprintf("Method '%s' was not found. It is either not existing or has not been exported (starts with lowercase).", string(e)) +} type WrongMethodSignatureError string func (e WrongMethodSignatureError) Error() string { - return fmt.Sprintf("Method %s has wrong signature", string(e)) -} - -type MethodNotExportedError string - -func (e MethodNotExportedError) Error() string { - return fmt.Sprintf("Method %s is not exported", string(e)) + return fmt.Sprintf("Method '%s' has wrong signature", string(e)) } type MethodInvocationFailedError struct { @@ -32,7 +28,7 @@ type MethodInvocationFailedError struct { } func (e *MethodInvocationFailedError) Error() string { - return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error) + return fmt.Sprintf("Method '%s' returned an error: %v", e.MethodName, e.Err) } type MigrationMethodInvoker interface { diff --git a/driver/mongodb/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go index fe6865e..298f803 100644 --- a/driver/mongodb/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -16,7 +16,7 @@ type FakeGoMethodsInvoker struct { func (invoker *FakeGoMethodsInvoker) Validate(methodName string) error { if methodName == "V001_some_non_existing_method_up" { - return MissingMethodError(methodName) + return MethodNotFoundError(methodName) } return nil @@ -93,7 +93,7 @@ func TestMigrate(t *testing.T) { `), }, expectedInvokedMethods: []string{}, - expectedErrors: []error{MissingMethodError("V001_some_non_existing_method_up")}, + expectedErrors: []error{MethodNotFoundError("V001_some_non_existing_method_up")}, }, { name: "up migration: failing method stops execution", diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index d077e98..026e686 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -59,7 +59,7 @@ func init() { } type DbMigration struct { - Id bson.ObjectId `bson:"_id,omitempty"` + Id bson.ObjectId `bson:"_id"` Version uint64 `bson:"version"` } @@ -79,6 +79,15 @@ func (driver *Driver) Initialize(url string) error { } session.SetMode(mgo.Monotonic, true) + c := session.DB(driver.methodsReceiver.DbName()).C(MIGRATE_C) + err = c.EnsureIndex(mgo.Index{ + Key: []string{"version"}, + Unique: true, + }) + if err != nil { + return err + } + driver.Session = session driver.migrator = gomethods.Migrator{MethodInvoker: driver} @@ -144,10 +153,10 @@ func (driver *Driver) Migrate(f file.File, pipe chan interface{}) { func (driver *Driver) Validate(methodName string) error { methodWithReceiver, ok := reflect.TypeOf(driver.methodsReceiver).MethodByName(methodName) if !ok { - return gomethods.MissingMethodError(methodName) + return gomethods.MethodNotFoundError(methodName) } if methodWithReceiver.PkgPath != "" { - return gomethods.MethodNotExportedError(methodName) + return gomethods.MethodNotFoundError(methodName) } methodFunc := reflect.ValueOf(driver.methodsReceiver).MethodByName(methodName) @@ -164,7 +173,7 @@ func (driver *Driver) Invoke(methodName string) error { name := methodName migrateMethod := reflect.ValueOf(driver.methodsReceiver).MethodByName(name) if !migrateMethod.IsValid() { - return gomethods.MissingMethodError(methodName) + return gomethods.MethodNotFoundError(methodName) } retValues := migrateMethod.Call([]reflect.Value{reflect.ValueOf(driver.Session)}) From 70238f448b7ff4b038efda57c33c20d44dd172ba Mon Sep 17 00:00:00 2001 From: dimag Date: Sat, 7 Jan 2017 16:16:00 +0200 Subject: [PATCH 27/27] - Reverted repository from dimag-jfrog to mattes to prepare for pull request --- docker-compose.yml | 2 +- driver/bash/bash.go | 4 ++-- driver/cassandra/cassandra.go | 6 +++--- driver/cassandra/cassandra_test.go | 6 +++--- driver/driver.go | 2 +- driver/mongodb/README.md | 8 ++++---- driver/mongodb/example/mongodb_test.go | 12 ++++++------ driver/mongodb/example/sample_mongdb_migrator.go | 6 +++--- driver/mongodb/gomethods/gomethods_migrator.go | 4 ++-- driver/mongodb/gomethods/gomethods_migrator_test.go | 6 +++--- driver/mongodb/gomethods/gomethods_registry.go | 2 +- driver/mongodb/mongodb.go | 8 ++++---- driver/mysql/mysql.go | 6 +++--- driver/mysql/mysql_test.go | 6 +++--- driver/postgres/postgres.go | 6 +++--- driver/postgres/postgres_test.go | 6 +++--- driver/sqlite3/sqlite3.go | 6 +++--- driver/sqlite3/sqlite3_test.go | 6 +++--- file/file.go | 2 +- file/file_test.go | 2 +- main.go | 2 +- migrate/migrate.go | 8 ++++---- 22 files changed, 58 insertions(+), 58 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1f619cf..2747839 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ go: &go image: golang - working_dir: /go/src/github.com/dimag-jfrog/migrate + working_dir: /go/src/github.com/mattes/migrate volumes: - $GOPATH:/go go-test: diff --git a/driver/bash/bash.go b/driver/bash/bash.go index 97ba91f..031f9bb 100644 --- a/driver/bash/bash.go +++ b/driver/bash/bash.go @@ -2,8 +2,8 @@ package bash import ( - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" ) type Driver struct { diff --git a/driver/cassandra/cassandra.go b/driver/cassandra/cassandra.go index cef8ac5..eb66174 100644 --- a/driver/cassandra/cassandra.go +++ b/driver/cassandra/cassandra.go @@ -9,9 +9,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/cassandra/cassandra_test.go b/driver/cassandra/cassandra_test.go index bc024c3..202d5e7 100644 --- a/driver/cassandra/cassandra_test.go +++ b/driver/cassandra/cassandra_test.go @@ -7,9 +7,9 @@ import ( "time" "github.com/gocql/gocql" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) func TestMigrate(t *testing.T) { diff --git a/driver/driver.go b/driver/driver.go index 5c2f6fc..e4ecb78 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -5,7 +5,7 @@ import ( "fmt" neturl "net/url" // alias to allow `url string` func signature in New - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/file" ) // Driver is the interface type that needs to implemented by all drivers. diff --git a/driver/mongodb/README.md b/driver/mongodb/README.md index fae3ffe..d7d8351 100644 --- a/driver/mongodb/README.md +++ b/driver/mongodb/README.md @@ -10,7 +10,7 @@ ## Usage in Go ```go -import "github.com/dimag-jfrog/migrate/migrate" +import "github.com/mattes/migrate/migrate" // Import your migration methods package so that they are registered and available for the MongoDB driver. // There is no need to import the MongoDB driver explicitly, as it should already be imported by your migration methods package. @@ -59,14 +59,14 @@ V001_some_migration_operation_down ## Methods registration -For a detailed example see: [sample_mongodb_migrator.go](https://github.com/dimag-jfrog/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) +For a detailed example see: [sample_mongodb_migrator.go](https://github.com/mattes/migrate/blob/master/driver/mongodb/example/sample_mongdb_migrator.go) ```go package my_mongo_db_migrator import ( - "github.com/dimag-jfrog/migrate/driver/mongodb" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" ) diff --git a/driver/mongodb/example/mongodb_test.go b/driver/mongodb/example/mongodb_test.go index c4371bf..86c37c0 100644 --- a/driver/mongodb/example/mongodb_test.go +++ b/driver/mongodb/example/mongodb_test.go @@ -3,13 +3,13 @@ package example import ( "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/mongodb" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb/gomethods" + pipep "github.com/mattes/migrate/pipe" "os" "reflect" "time" diff --git a/driver/mongodb/example/sample_mongdb_migrator.go b/driver/mongodb/example/sample_mongdb_migrator.go index f251e5c..1ab1440 100644 --- a/driver/mongodb/example/sample_mongdb_migrator.go +++ b/driver/mongodb/example/sample_mongdb_migrator.go @@ -1,13 +1,13 @@ package example import ( - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - _ "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/driver/mongodb/gomethods" + _ "github.com/mattes/migrate/driver/mongodb/gomethods" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/dimag-jfrog/migrate/driver/mongodb" + "github.com/mattes/migrate/driver/mongodb" ) type SampleMongoDbMigrator struct { diff --git a/driver/mongodb/gomethods/gomethods_migrator.go b/driver/mongodb/gomethods/gomethods_migrator.go index 96fc1d5..97057ab 100644 --- a/driver/mongodb/gomethods/gomethods_migrator.go +++ b/driver/mongodb/gomethods/gomethods_migrator.go @@ -3,8 +3,8 @@ package gomethods import ( "bufio" "fmt" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" "os" "path" "strings" diff --git a/driver/mongodb/gomethods/gomethods_migrator_test.go b/driver/mongodb/gomethods/gomethods_migrator_test.go index 298f803..d94506e 100644 --- a/driver/mongodb/gomethods/gomethods_migrator_test.go +++ b/driver/mongodb/gomethods/gomethods_migrator_test.go @@ -4,10 +4,10 @@ import ( "reflect" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + pipep "github.com/mattes/migrate/pipe" ) type FakeGoMethodsInvoker struct { diff --git a/driver/mongodb/gomethods/gomethods_registry.go b/driver/mongodb/gomethods/gomethods_registry.go index 5ceb128..418256f 100644 --- a/driver/mongodb/gomethods/gomethods_registry.go +++ b/driver/mongodb/gomethods/gomethods_registry.go @@ -2,7 +2,7 @@ package gomethods import ( "fmt" - "github.com/dimag-jfrog/migrate/driver" + "github.com/mattes/migrate/driver" "sync" ) diff --git a/driver/mongodb/mongodb.go b/driver/mongodb/mongodb.go index 026e686..fcfae50 100644 --- a/driver/mongodb/mongodb.go +++ b/driver/mongodb/mongodb.go @@ -2,10 +2,10 @@ package mongodb import ( "errors" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/driver/mongodb/gomethods" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/driver/mongodb/gomethods" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "reflect" diff --git a/driver/mysql/mysql.go b/driver/mysql/mysql.go index cf186f8..2bbbc13 100644 --- a/driver/mysql/mysql.go +++ b/driver/mysql/mysql.go @@ -12,9 +12,9 @@ import ( "strings" "github.com/go-sql-driver/mysql" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/mysql/mysql_test.go b/driver/mysql/mysql_test.go index 511b1b1..ec19271 100644 --- a/driver/mysql/mysql_test.go +++ b/driver/mysql/mysql_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/postgres/postgres.go b/driver/postgres/postgres.go index f5d307b..951b005 100644 --- a/driver/postgres/postgres.go +++ b/driver/postgres/postgres.go @@ -8,9 +8,9 @@ import ( "strconv" "github.com/lib/pq" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" ) type Driver struct { diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go index 7b2c665..7c5c0c3 100644 --- a/driver/postgres/postgres_test.go +++ b/driver/postgres/postgres_test.go @@ -5,9 +5,9 @@ import ( "os" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate(). diff --git a/driver/sqlite3/sqlite3.go b/driver/sqlite3/sqlite3.go index 65e6a2a..a92b14d 100644 --- a/driver/sqlite3/sqlite3.go +++ b/driver/sqlite3/sqlite3.go @@ -7,9 +7,9 @@ import ( "fmt" "strings" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" "github.com/mattn/go-sqlite3" ) diff --git a/driver/sqlite3/sqlite3_test.go b/driver/sqlite3/sqlite3_test.go index 6abcf0d..7f50095 100644 --- a/driver/sqlite3/sqlite3_test.go +++ b/driver/sqlite3/sqlite3_test.go @@ -4,9 +4,9 @@ import ( "database/sql" "testing" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // TestMigrate runs some additional tests on Migrate() diff --git a/file/file.go b/file/file.go index 14617ba..54992a4 100644 --- a/file/file.go +++ b/file/file.go @@ -5,7 +5,7 @@ import ( "bytes" "errors" "fmt" - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/migrate/direction" "go/token" "io/ioutil" "path" diff --git a/file/file_test.go b/file/file_test.go index c6bd0b3..b9ddeab 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -1,7 +1,7 @@ package file import ( - "github.com/dimag-jfrog/migrate/migrate/direction" + "github.com/mattes/migrate/migrate/direction" "io/ioutil" "os" "path" diff --git a/main.go b/main.go index 802bd9d..ac842c7 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ // Package main is the CLI. // You can use the CLI via Terminal. -// import "github.com/dimag-jfrog/migrate/migrate" for usage within Go. +// import "github.com/mattes/migrate/migrate" for usage within Go. package main import ( diff --git a/migrate/migrate.go b/migrate/migrate.go index c5696b5..0c976be 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -12,10 +12,10 @@ import ( "strings" "time" - "github.com/dimag-jfrog/migrate/driver" - "github.com/dimag-jfrog/migrate/file" - "github.com/dimag-jfrog/migrate/migrate/direction" - pipep "github.com/dimag-jfrog/migrate/pipe" + "github.com/mattes/migrate/driver" + "github.com/mattes/migrate/file" + "github.com/mattes/migrate/migrate/direction" + pipep "github.com/mattes/migrate/pipe" ) // Up applies all available migrations