status-im
/
migrate
mirror of https://github.com/status-im/migrate.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Changes that interfere with the Migrate open source code: 

- Added go methods migrator, mongo db template: different from the usual driver
model.
- Added support for bidirectional files (for go methods), appending _up or _down upon context
- Added DriverWithFilnameParser for providing custom filename parser functionality that knows to parse bi-directional file names.
This commit is contained in:
dimag 2016-08-08 16:29:25 +03:00
parent 175550643c
commit 1838852d4d
12 changed files with 1118 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.gitignore vendored
View File

@ -1,2 +1,11 @@
.DS_Store
test.db
test.db
*.iml
.DS_Store
.idea
*.zip
*.tar.*
*.nuget
*.jar
*.war

9
driver/driver.go
View File

@ -34,6 +34,15 @@ type Driver interface {
Version() (uint64, error)
}
// Driver that has some custom migration file format
// and wants to use a different parsing strategy.
type DriverWithFilenameParser interface {
Driver
FilenameParser() file.FilenameParser
}
// New returns Driver and calls Initialize on it
func New(url string) (Driver, error) {
u, err := neturl.Parse(url)

176
driver/gomethods/gomethods_migrator.go Normal file
View File

@ -0,0 +1,176 @@
package gomethods
import (
//"bytes"
"reflect"
"fmt"
"strings"
"os"
"path"
"bufio"
"github.com/dimag-jfrog/migrate/driver"
"github.com/dimag-jfrog/migrate/file"
"github.com/dimag-jfrog/migrate/migrate/direction"
)
type MissingMethodError string
func (e MissingMethodError) Error() string { return "Non existing migrate method: " + string(e) }
type WrongMethodSignatureError string
func (e WrongMethodSignatureError) Error() string { return fmt.Sprintf("Method %s has wrong signature", e) }
type MethodInvocationFailedError struct {
MethodName string
Err error
}
func (e *MethodInvocationFailedError) Error() string {
return fmt.Sprintf("Method %s returned an error: %v", e.MethodName, e.Error)
}
type Migrator struct {
Driver driver.DriverWithFilenameParser
RollbackOnFailure bool
}
func (m *Migrator) Migrate(f file.File, pipe chan interface{}) error {
methods, err := m.getMigrationMethods(f)
if err != nil {
pipe <- err
return err
}
for i, methodName := range methods {
pipe <- methodName
err := m.Invoke(methodName)
if err != nil {
pipe <- err
if !m.RollbackOnFailure {
return err
}
// on failure, try to rollback methods in this migration
for j := i-1; j >= 0; j-- {
rollbackToMethodName := getRollbackToMethod(methods[j])
pipe <- rollbackToMethodName
err = m.Invoke(rollbackToMethodName)
if err != nil {
pipe <- err
break
}
}
return err
}
}
return nil
}
func (m *Migrator) IsValid(methodName string) bool {
return reflect.ValueOf(m.Driver).MethodByName(methodName).IsValid()
}
func (m *Migrator) Invoke(methodName string) error {
name := methodName
migrateMethod := reflect.ValueOf(m.Driver).MethodByName(name)
if !migrateMethod.IsValid() {
return MissingMethodError(methodName)
}
retValues := migrateMethod.Call([]reflect.Value{})
if len(retValues) != 1 {
return WrongMethodSignatureError(name)
}
if !retValues[0].IsNil() {
err, ok := retValues[0].Interface().(error)
if !ok {
return WrongMethodSignatureError(name)
}
return &MethodInvocationFailedError{ MethodName:name, Err:err}
}
return nil
}
func reverseInPlace(a []string) {
for i := 0; i < len(a)/2; i++ {
j := len(a) - i - 1
a[i], a[j] = a[j], a[i]
}
}
func getRollbackToMethod(methodName string) string {
if strings.HasSuffix(methodName, "_up") {
return strings.TrimSuffix(methodName, "_up") + "_down"
} else {
return strings.TrimSuffix(methodName, "_down") + "_up"
}
}
func getFileLines(file file.File) ([]string, error) {
if len(file.Content) == 0 {
lines := make([]string, 0)
file, err := os.Open(path.Join(file.Path, file.FileName))
if err != nil {
return nil, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, nil
} else {
//n := bytes.IndexByte(file.Content, 0)
//n := bytes.Index(file.Content, []byte{0})
//s := string(file.Content[:n])
s := string(file.Content)
return strings.Split(s, "\n"), nil
}
}
func (m *Migrator) getMigrationMethods(f file.File) ([]string, error) {
var lines, methods []string
lines, err := getFileLines(f)
if err != nil {
return nil, err
}
for _, line := range lines {
operationName := strings.TrimSpace(line)
if operationName == "" || strings.HasPrefix(operationName, "--") {
// an empty line or a comment, ignore
continue
}
upMethodName := operationName + "_up"
downMethodName := operationName + "_down"
if !m.IsValid(upMethodName) {
return nil, MissingMethodError(upMethodName)
}
if !m.IsValid(downMethodName) {
return nil, MissingMethodError(downMethodName)
}
if f.Direction == direction.Up {
methods = append(methods, upMethodName)
} else {
methods = append(methods, downMethodName)
}
}
_,_,fileType,_ := m.Driver.FilenameParser().Parse(f.FileName)
if fileType == direction.Both && f.Direction == direction.Down {
reverseInPlace(methods)
}
return methods, nil
}

342
driver/gomethods/gomethods_migrator_test.go Normal file
View File

@ -0,0 +1,342 @@
package gomethods
import (
"reflect"
"testing"
"github.com/dimag-jfrog/migrate/file"
"github.com/dimag-jfrog/migrate/migrate/direction"
pipep "github.com/dimag-jfrog/migrate/pipe"
)
type FakeGoMethodsDriver struct {
InvokedMethods []string
Migrator Migrator
}
func (driver *FakeGoMethodsDriver) Initialize(url string) error {
return nil
}
func (driver *FakeGoMethodsDriver) Close() error {
return nil
}
func (driver *FakeGoMethodsDriver) FilenameParser() file.FilenameParser {
return file.UpDownAndBothFilenameParser{ FilenameExtension: driver.FilenameExtension() }
}
func (driver *FakeGoMethodsDriver) FilenameExtension() string {
return "gm"
}
func (driver *FakeGoMethodsDriver) Version() (uint64, error) {
return uint64(0), nil
}
func (driver *FakeGoMethodsDriver) Migrate(f file.File, pipe chan interface{}) {
defer close(pipe)
pipe <- f
return
}
func (driver *FakeGoMethodsDriver) V001_init_organizations_up() error {
driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_up")
return nil
}
func (driver *FakeGoMethodsDriver) V001_init_organizations_down() error {
driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_organizations_down")
return nil
}
func (driver *FakeGoMethodsDriver) V001_init_users_up() error {
driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_up")
return nil
}
func (driver *FakeGoMethodsDriver) V001_init_users_down() error {
driver.InvokedMethods = append(driver.InvokedMethods, "V001_init_users_down")
return nil
}
type SomeError struct{}
func (e SomeError) Error() string { return "Some error happened" }
func (driver *FakeGoMethodsDriver) V001_some_failing_method_up() error {
driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_up")
return SomeError{}
}
func (driver *FakeGoMethodsDriver) V001_some_failing_method_down() error {
driver.InvokedMethods = append(driver.InvokedMethods, "V001_some_failing_method_down")
return SomeError{}
}
func TestMigrate(t *testing.T) {
cases := []struct {
name string
file file.File
expectedInvokedMethods []string
expectedErrors []error
expectRollback bool
}{
{
name: "up migration, both directions-file: invokes up methods in order",
file: file.File {
Path: "/foobar",
FileName: "001_foobar.gm",
Version: 1,
Name: "foobar",
Direction: direction.Up,
Content: []byte(`
V001_init_organizations
V001_init_users
`),
},
expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"},
expectedErrors: []error{},
},
{
name: "down migration, both-directions-file: reverts direction of invoked down methods",
file: file.File {
Path: "/foobar",
FileName: "001_foobar.gm",
Version: 1,
Name: "foobar",
Direction: direction.Down,
Content: []byte(`
V001_init_organizations
V001_init_users
`),
},
expectedInvokedMethods: []string{"V001_init_users_down", "V001_init_organizations_down"},
expectedErrors: []error{},
},
{
name: "up migration, up direction-file: invokes up methods in order",
file: file.File {
Path: "/foobar",
FileName: "001_foobar.up.gm",
Version: 1,
Name: "foobar",
Direction: direction.Up,
Content: []byte(`
V001_init_organizations
V001_init_users
`),
},
expectedInvokedMethods: []string{"V001_init_organizations_up", "V001_init_users_up"},
expectedErrors: []error{},
},
{
name: "down migration, down directions-file: keeps order of invoked down methods",
file: file.File {
Path: "/foobar",
FileName: "001_foobar.down.gm",
Version: 1,
Name: "foobar",
Direction: direction.Down,
Content: []byte(`
V001_init_organizations
V001_init_users
`),
},
expectedInvokedMethods: []string{"V001_init_organizations_down", "V001_init_users_down"},
expectedErrors: []error{},
},
{
name: "up migration: non-existing method causes migration not to execute",
file: file.File {
Path: "/foobar",
FileName: "001_foobar.gm",
Version: 1,
Name: "foobar",
Direction: direction.Up,
Content: []byte(`
V001_init_organizations
V001_init_users
V001_some_non_existing_method
`),
},
expectedInvokedMethods: []string{},
expectedErrors: []error{ MissingMethodError("V001_some_non_existing_method_up") },
},
{
name: "up migration: failing method stops execution",
file: file.File {
Path: "/foobar",
FileName: "001_foobar.gm",
Version: 1,
Name: "foobar",
Direction: direction.Up,
Content: []byte(`
V001_init_organizations
V001_some_failing_method
V001_init_users
`),
},
expectedInvokedMethods: []string{
"V001_init_organizations_up",
"V001_some_failing_method_up",
},
expectedErrors: []error{ &MethodInvocationFailedError{
MethodName: "V001_some_failing_method_up",
Err: SomeError{},
}},
},
{
name: "down migration, both-directions-file: failing method stops migration",
file: file.File {
Path: "/foobar",
FileName: "001_foobar.gm",
Version: 1,
Name: "foobar",
Direction: direction.Down,
Content: []byte(`
V001_init_organizations
V001_some_failing_method
V001_init_users
`),
},
expectedInvokedMethods: []string{
"V001_init_users_down",
"V001_some_failing_method_down",
},
expectedErrors: []error{ &MethodInvocationFailedError{
MethodName: "V001_some_failing_method_down",
Err: SomeError{},
}},
},
{
name: "up migration: failing method causes rollback in rollback mode",
expectRollback: true,
file: file.File {
Path: "/foobar",
FileName: "001_foobar.gm",
Version: 1,
Name: "foobar",
Direction: direction.Up,
Content: []byte(`
V001_init_organizations
V001_init_users
V001_some_failing_method
`),
},
expectedInvokedMethods: []string{
"V001_init_organizations_up",
"V001_init_users_up",
"V001_some_failing_method_up",
"V001_init_users_down",
"V001_init_organizations_down",
},
expectedErrors: []error{ &MethodInvocationFailedError{
MethodName: "V001_some_failing_method_up",
Err: SomeError{},
}},
},
{
name: "down migration, both-directions-file: failing method causes rollback in rollback mode",
expectRollback: true,
file: file.File {
Path: "/foobar",
FileName: "001_foobar.gm",
Version: 1,
Name: "foobar",
Direction: direction.Down,
Content: []byte(`
V001_init_organizations
V001_some_failing_method
V001_init_users
`),
},
expectedInvokedMethods: []string{
"V001_init_users_down",
"V001_some_failing_method_down",
"V001_init_users_up",
},
expectedErrors: []error{ &MethodInvocationFailedError{
MethodName: "V001_some_failing_method_down",
Err: SomeError{},
}},
},
}
for _, c := range cases {
migrator := Migrator{}
d := &FakeGoMethodsDriver{Migrator: migrator, InvokedMethods:[]string{}}
migrator.Driver = d
migrator.RollbackOnFailure = c.expectRollback
pipe := pipep.New()
go func() {
migrator.Migrate(c.file, pipe)
close(pipe)
}()
errs := pipep.ReadErrors(pipe)
var failed bool
if !reflect.DeepEqual(d.InvokedMethods, c.expectedInvokedMethods) {
failed = true
t.Errorf("case '%s': FAILED\nexpected invoked methods %v\nbut got %v", c.name, c.expectedInvokedMethods, d.InvokedMethods)
}
if !reflect.DeepEqual(errs, c.expectedErrors) {
failed = true
t.Errorf("case '%s': FAILED\nexpected errors %v\nbut got %v", c.name, c.expectedErrors, errs)
}
if !failed {
t.Logf("case '%s': PASSED", c.name)
}
}
}
func TestGetRollbackToMethod(t *testing.T) {
cases := []struct {
method string
expectedRollbackMethod string
}{
{"some_method_up", "some_method_down"},
{"some_method_down", "some_method_up"},
{"up_down_up", "up_down_down"},
{"down_up", "down_down"},
{"down_down", "down_up"},
}
for _, c := range cases {
actualRollbackMethod := getRollbackToMethod(c.method)
if actualRollbackMethod != c.expectedRollbackMethod {
t.Errorf("Expected rollback method to be %s but got %s", c.expectedRollbackMethod, actualRollbackMethod)
}
}
}
func TestReverseInPlace(t *testing.T) {
methods := []string {
"method1_down",
"method2_down",
"method3_down",
"method4_down",
"method5_down",
}
expectedReversedMethods := []string {
"method5_down",
"method4_down",
"method3_down",
"method2_down",
"method1_down",
}
reverseInPlace(methods)
if !reflect.DeepEqual(methods, expectedReversedMethods) {
t.Errorf("Expected reverse methods %v but got %v", expectedReversedMethods, methods)
}
}

111
driver/gomethods/mongodb/mongodb_template.go Normal file
View File

@ -0,0 +1,111 @@
package mongodb
import (
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
"errors"
"strings"
"github.com/dimag-jfrog/migrate/migrate/direction"
"github.com/dimag-jfrog/migrate/file"
"github.com/dimag-jfrog/migrate/driver/gomethods"
)
const MIGRATE_C = "db_migrations"
// This is not a real driver since the Initialize method requires a gomethods.Migrator
// The real driver will contain the DriverTemplate and implement all the custom migration Golang methods
// See example in usage_examples for details
type DriverTemplate struct {
Session *mgo.Session
DbName string
migrator gomethods.Migrator
}
type DbMigration struct {
Id bson.ObjectId `bson:"_id,omitempty"`
Version uint64 `bson:"version"`
}
func (driver *DriverTemplate) Initialize(url, dbName string, migrator gomethods.Migrator) error {
urlWithoutScheme := strings.SplitN(url, "mongodb://", 2)
if len(urlWithoutScheme) != 2 {
return errors.New("invalid mongodb:// scheme")
}
session, err := mgo.Dial(url)
if err != nil {
return err
}
session.SetMode(mgo.Monotonic, true)
driver.Session = session
driver.DbName = dbName
driver.migrator = migrator
return nil
}
func (driver *DriverTemplate) Close() error {
if driver.Session != nil {
driver.Session.Close()
}
return nil
}
func (driver *DriverTemplate) FilenameParser() file.FilenameParser {
return file.UpDownAndBothFilenameParser{FilenameExtension: driver.FilenameExtension()}
}
func (driver *DriverTemplate) FilenameExtension() string {
return "mgo"
}
func (driver *DriverTemplate) Version() (uint64, error) {
var latestMigration DbMigration
c := driver.Session.DB(driver.DbName).C(MIGRATE_C)
err := c.Find(bson.M{}).Sort("-version").One(&latestMigration)
switch {
case err == mgo.ErrNotFound:
return 0, nil
case err != nil:
return 0, err
default:
return latestMigration.Version, nil
}
}
func (driver *DriverTemplate) Migrate(f file.File, pipe chan interface{}) {
defer close(pipe)
pipe <- f
err := driver.migrator.Migrate(f, pipe)
if err != nil {
return
}
migrate_c := driver.Session.DB(driver.DbName).C(MIGRATE_C)
if f.Direction == direction.Up {
id := bson.NewObjectId()
dbMigration := DbMigration{Id: id, Version: f.Version}
err := migrate_c.Insert(dbMigration)
if err != nil {
pipe <- err
return
}
} else if f.Direction == direction.Down {
err := migrate_c.Remove(bson.M{"version": f.Version})
if err != nil {
pipe <- err
return
}
}
}

121
driver/gomethods/usage_examples/mongodb.go Normal file
View File

@ -0,0 +1,121 @@
package usage_examples
import (
"github.com/dimag-jfrog/migrate/driver"
"github.com/dimag-jfrog/migrate/driver/gomethods"
"github.com/dimag-jfrog/migrate/driver/gomethods/mongodb"
"gopkg.in/mgo.v2/bson"
"time"
)
// This boilerplate part is necessary and the same
// regardless of the specific mongodb golang methods driver
type GoMethodsMongoDbDriver struct {
mongodb.DriverTemplate
}
func (d *GoMethodsMongoDbDriver) Initialize(url string) error {
return d.DriverTemplate.Initialize(url, DB_NAME, gomethods.Migrator{Driver: d})
}
func init() {
driver.RegisterDriver("mongodb", &GoMethodsMongoDbDriver{mongodb.DriverTemplate{}})
}
// Here goes the specific mongodb golang methods driver logic
const DB_NAME = "test"
const SHORT_DATE_LAYOUT = "2000-Jan-01"
const USERS_C = "users"
const ORGANIZATIONS_C = "organizations"
type Organization struct {
Id bson.ObjectId `bson:"_id,omitempty"`
Name string `bson:"name"`
Location string `bson:"location"`
DateFounded time.Time `bson:"date_founded"`
}
type User struct {
Id bson.ObjectId `bson:"_id"`
Name string `bson:"name"`
}
func (m *GoMethodsMongoDbDriver) V001_init_organizations_up() error {
date1, _ := time.Parse(SHORT_DATE_LAYOUT, "1994-Jul-05")
date2, _ := time.Parse(SHORT_DATE_LAYOUT, "1998-Sep-04")
date3, _ := time.Parse(SHORT_DATE_LAYOUT, "2008-Apr-28")
orgs := []Organization{
{Id: bson.NewObjectId(), Name: "Amazon", Location:"Seattle", DateFounded: date1},
{Id: bson.NewObjectId(), Name: "Google", Location:"Mountain View", DateFounded: date2},
{Id: bson.NewObjectId(), Name: "JFrog", Location:"Santa Clara", DateFounded: date3},
}
for _, org := range orgs {
err := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).Insert(org)
if err != nil {
return err
}
}
return nil
}
func (m *GoMethodsMongoDbDriver) V001_init_organizations_down() error {
return m.Session.DB(DB_NAME).C(ORGANIZATIONS_C).DropCollection()
}
func (m *GoMethodsMongoDbDriver) V001_init_users_up() error {
users := []User{
{Id: bson.NewObjectId(), Name: "Alex"},
{Id: bson.NewObjectId(), Name: "Beatrice"},
{Id: bson.NewObjectId(), Name: "Cleo"},
}
for _, user := range users {
err := m.Session.DB(DB_NAME).C(USERS_C).Insert(user)
if err != nil {
return err
}
}
return nil
}
func (m *GoMethodsMongoDbDriver) V001_init_users_down() error {
return m.Session.DB(DB_NAME).C(USERS_C).DropCollection()
}
func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_up() error {
c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C)
_, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"location": "headquarters"}})
return err
}
func (m *GoMethodsMongoDbDriver) V002_organizations_rename_location_field_to_headquarters_down() error {
c := m.Session.DB(DB_NAME).C(ORGANIZATIONS_C)
_, err := c.UpdateAll(nil, bson.M{"$rename": bson.M{"headquarters": "location"}})
return err
}
func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_up() error {
c := m.Session.DB(DB_NAME).C(USERS_C)
colQuerier := bson.M{"name": "Cleo"}
change := bson.M{"$set": bson.M{"name": "Cleopatra"}}
return c.Update(colQuerier, change)
}
func (m *GoMethodsMongoDbDriver) V002_change_user_cleo_to_cleopatra_down() error {
c := m.Session.DB(DB_NAME).C(USERS_C)
colQuerier := bson.M{"name": "Cleopatra"}
change := bson.M{"$set": bson.M{"name": "Cleo",}}
return c.Update(colQuerier, change)
}

123
driver/gomethods/usage_examples/mongodb_test.go Normal file
View File

@ -0,0 +1,123 @@
package usage_examples
import (
"testing"
"github.com/dimag-jfrog/migrate/file"
"github.com/dimag-jfrog/migrate/migrate/direction"
pipep "github.com/dimag-jfrog/migrate/pipe"
)
func TestMigrate(t *testing.T) {
//host := os.Getenv("MONGODB_PORT_27017_TCP_ADDR")
//port := os.Getenv("MONGODB_PORT_27017_TCP_PORT")
host := "127.0.0.1"
port := "27017"
driverUrl := "mongodb://" + host + ":" + port
d := &GoMethodsMongoDbDriver{}
if err := d.Initialize(driverUrl); err != nil {
t.Fatal(err)
}
content1 := []byte(`
V001_init_organizations
V001_init_users
`)
content2 := []byte(`
V002_organizations_rename_location_field_to_headquarters
V002_change_user_cleo_to_cleopatra
`)
files := []file.File{
{
Path: "/foobar",
FileName: "001_foobar.mgo",
Version: 1,
Name: "foobar",
Direction: direction.Up,
Content: content1,
},
{
Path: "/foobar",
FileName: "001_foobar.mgo",
Version: 1,
Name: "foobar",
Direction: direction.Down,
Content: content1,
},
{
Path: "/foobar",
FileName: "002_foobar.mgo",
Version: 2,
Name: "foobar",
Direction: direction.Up,
Content: content2,
},
{
Path: "/foobar",
FileName: "002_foobar.mgo",
Version: 2,
Name: "foobar",
Direction: direction.Down,
Content: content2,
},
{
Path: "/foobar",
FileName: "001_foobar.mgo",
Version: 1,
Name: "foobar",
Direction: direction.Up,
Content: []byte(`
V001_init_organizations
V001_init_users
V001_non_existing_operation
`),
},
}
var pipe chan interface{}
var errs []error
pipe = pipep.New()
go d.Migrate(files[0], pipe)
errs = pipep.ReadErrors(pipe)
if len(errs) > 0 {
t.Fatal(errs)
}
pipe = pipep.New()
go d.Migrate(files[2], pipe)
errs = pipep.ReadErrors(pipe)
if len(errs) > 0 {
t.Fatal(errs)
}
pipe = pipep.New()
go d.Migrate(files[3], pipe)
errs = pipep.ReadErrors(pipe)
if len(errs) > 0 {
t.Fatal(errs)
}
//pipe = pipep.New()
//go d.Migrate(files[1], pipe)
//errs = pipep.ReadErrors(pipe)
//if len(errs) > 0 {
// t.Fatal(errs)
//}
pipe = pipep.New()
go d.Migrate(files[4], pipe)
errs = pipep.ReadErrors(pipe)
if len(errs) == 0 {
t.Error("Expected test case to fail")
}
if err := d.Close(); err != nil {
t.Fatal(err)
}
}

106
file/file.go
View File

@ -9,20 +9,11 @@ import (
"go/token"
"io/ioutil"
"path"
"regexp"
"sort"
"strconv"
"strings"
)
var filenameRegex = `^([0-9]+)_(.*)\.(up|down)\.%s$`
// FilenameRegex builds regular expression stmt with given
// filename extension from driver.
func FilenameRegex(filenameExtension string) *regexp.Regexp {
return regexp.MustCompile(fmt.Sprintf(filenameRegex, filenameExtension))
}
// File represents one file on disk.
// Example: 001_initial_plan_to_do_sth.up.sql
type File struct {
@ -149,8 +140,17 @@ func (mf *MigrationFiles) From(version uint64, relativeN int) (Files, error) {
return files, nil
}
func ReadMigrationFiles(path, filenameExtension string) (MigrationFiles, error){
return doReadMigrationFiles(path, DefaultFilenameParser{FilenameExtension: filenameExtension})
}
func ReadMigrationFilesWithFilenameParser(path string, filenameParser FilenameParser) (MigrationFiles, error){
return doReadMigrationFiles(path, filenameParser)
}
// ReadMigrationFiles reads all migration files from a given path
func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files MigrationFiles, err error) {
func doReadMigrationFiles(path string, filenameParser FilenameParser) (files MigrationFiles, err error) {
// find all migration files in path
ioFiles, err := ioutil.ReadDir(path)
if err != nil {
@ -165,7 +165,7 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat
tmpFiles := make([]*tmpFile, 0)
tmpFileMap := map[uint64]map[direction.Direction]tmpFile{}
for _, file := range ioFiles {
version, name, d, err := parseFilenameSchema(file.Name(), filenameRegex)
version, name, d, err := filenameParser.Parse(file.Name())
if err == nil {
if _, ok := tmpFileMap[version]; !ok {
tmpFileMap[version] = map[direction.Direction]tmpFile{}
@ -210,33 +210,56 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat
Direction: direction.Down,
}
lookFordirection = direction.Up
case direction.Both:
migrationFile.UpFile = &File{
Path: path,
FileName: file.filename,
Version: file.version,
Name: file.name,
Content: nil,
Direction: direction.Up,
}
migrationFile.DownFile = &File{
Path: path,
FileName: file.filename,
Version: file.version,
Name: file.name,
Content: nil,
Direction: direction.Down,
}
default:
return nil, errors.New("Unsupported direction.Direction Type")
}
for _, file2 := range tmpFiles {
if file2.version == file.version && file2.d == lookFordirection {
switch lookFordirection {
case direction.Up:
migrationFile.UpFile = &File{
Path: path,
FileName: file2.filename,
Version: file.version,
Name: file2.name,
Content: nil,
Direction: direction.Up,
if file2.version == file.version {
if file.d == direction.Both {
if file.d != file2.d {
return nil, errors.New("Incompatible direction.Direction types")
}
case direction.Down:
migrationFile.DownFile = &File{
Path: path,
FileName: file2.filename,
Version: file.version,
Name: file2.name,
Content: nil,
Direction: direction.Down,
} else if file2.d == lookFordirection {
switch lookFordirection {
case direction.Up:
migrationFile.UpFile = &File{
Path: path,
FileName: file2.filename,
Version: file.version,
Name: file2.name,
Content: nil,
Direction: direction.Up,
}
case direction.Down:
migrationFile.DownFile = &File{
Path: path,
FileName: file2.filename,
Version: file.version,
Name: file2.name,
Content: nil,
Direction: direction.Down,
}
}
break
}
break
}
}
@ -249,29 +272,6 @@ func ReadMigrationFiles(path string, filenameRegex *regexp.Regexp) (files Migrat
return newFiles, nil
}
// parseFilenameSchema parses the filename
func parseFilenameSchema(filename string, filenameRegex *regexp.Regexp) (version uint64, name string, d direction.Direction, err error) {
matches := filenameRegex.FindStringSubmatch(filename)
if len(matches) != 4 {
return 0, "", 0, errors.New("Unable to parse filename schema")
}
version, err = strconv.ParseUint(matches[1], 10, 0)
if err != nil {
return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0]))
}
if matches[3] == "up" {
d = direction.Up
} else if matches[3] == "down" {
d = direction.Down
} else {
return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3]))
}
return version, matches[2], d, nil
}
// Len is the number of elements in the collection.
// Required by Sort Interface{}
func (mf MigrationFiles) Len() int {

50
file/file_test.go
View File

@ -8,52 +8,6 @@ import (
"testing"
)
func TestParseFilenameSchema(t *testing.T) {
var tests = []struct {
filename string
filenameExtension string
expectVersion uint64
expectName string
expectDirection direction.Direction
expectErr bool
}{
{"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false},
{"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false},
{"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false},
{"-1_test_file.down.sql", "sql", 0, "", direction.Up, true},
{"test_file.down.sql", "sql", 0, "", direction.Up, true},
{"100_test_file.down", "sql", 0, "", direction.Up, true},
{"100_test_file.sql", "sql", 0, "", direction.Up, true},
{"100_test_file", "sql", 0, "", direction.Up, true},
{"test_file", "sql", 0, "", direction.Up, true},
{"100", "sql", 0, "", direction.Up, true},
{".sql", "sql", 0, "", direction.Up, true},
{"up.sql", "sql", 0, "", direction.Up, true},
{"down.sql", "sql", 0, "", direction.Up, true},
}
for _, test := range tests {
version, name, migrate, err := parseFilenameSchema(test.filename, FilenameRegex(test.filenameExtension))
if test.expectErr && err == nil {
t.Fatal("Expected error, but got none.", test)
}
if !test.expectErr && err != nil {
t.Fatal("Did not expect error, but got one:", err, test)
}
if err == nil {
if version != test.expectVersion {
t.Error("Wrong version number", test)
}
if name != test.expectName {
t.Error("wrong name", test)
}
if migrate != test.expectDirection {
t.Error("wrong migrate", test)
}
}
}
}
func TestFiles(t *testing.T) {
tmpdir, err := ioutil.TempDir("/tmp", "TestLookForMigrationFilesInSearchPath")
if err != nil {
@ -77,7 +31,7 @@ func TestFiles(t *testing.T) {
ioutil.WriteFile(path.Join(tmpdir, "401_migrationfile.down.sql"), []byte("test"), 0755)
files, err := ReadMigrationFiles(tmpdir, FilenameRegex("sql"))
files, err := ReadMigrationFiles(tmpdir, "sql")
if err != nil {
t.Fatal(err)
}
@ -223,7 +177,7 @@ func TestDuplicateFiles(t *testing.T) {
t.Fatal(err)
}
_, err = ReadMigrationFiles(root, FilenameRegex("sql"))
_, err = ReadMigrationFiles(root, "sql")
if err == nil {
t.Fatal("Expected duplicate migration file error")
}

85
file/filename_parser.go Normal file
View File

@ -0,0 +1,85 @@
package file
import (
"errors"
"github.com/dimag-jfrog/migrate/migrate/direction"
"regexp"
"fmt"
"strconv"
"strings"
)
type FilenameParser interface {
Parse (filename string) (version uint64, name string, d direction.Direction, err error)
}
var defaultFilenameRegexTemplate = `^([0-9]+)_(.*)\.(up|down)\.%s$`
func parseDefaultFilenameSchema(filename, filenameRegex string) (version uint64, name string, d direction.Direction, err error) {
regexp := regexp.MustCompile(filenameRegex)
matches := regexp.FindStringSubmatch(filename)
if len(matches) != 4 {
return 0, "", 0, errors.New("Unable to parse filename schema")
}
version, err = strconv.ParseUint(matches[1], 10, 0)
if err != nil {
return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0]))
}
name = matches[2]
if matches[3] == "up" {
d = direction.Up
} else if matches[3] == "down" {
d = direction.Down
} else {
return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse up|down '%v' in filename schema", matches[3]))
}
return version, name, d, nil
}
type DefaultFilenameParser struct {
FilenameExtension string
}
func (parser DefaultFilenameParser) Parse (filename string) (version uint64, name string, d direction.Direction, err error) {
filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension)
return parseDefaultFilenameSchema(filename, filenameRegex)
}
type UpDownAndBothFilenameParser struct {
FilenameExtension string
}
func (parser UpDownAndBothFilenameParser) Parse(filename string) (version uint64, name string, d direction.Direction, err error) {
ext := parser.FilenameExtension
if !strings.HasSuffix(filename, ext) {
return 0, "", 0, errors.New("Filename ")
}
var matches []string
if strings.HasSuffix(filename, ".up." + ext) || strings.HasSuffix(filename, ".down." + ext) {
filenameRegex := fmt.Sprintf(defaultFilenameRegexTemplate, parser.FilenameExtension)
return parseDefaultFilenameSchema(filename, filenameRegex)
}
regex := regexp.MustCompile(fmt.Sprintf(`^([0-9]+)_(.*)\.%s$`, ext))
matches = regex.FindStringSubmatch(filename)
if len(matches) != 3 {
return 0, "", 0, errors.New("Unable to parse filename schema")
}
version, err = strconv.ParseUint(matches[1], 10, 0)
if err != nil {
return 0, "", 0, errors.New(fmt.Sprintf("Unable to parse version '%v' in filename schema", matches[0]))
}
name = matches[2]
d = direction.Both
return version, name, d, nil
}

85
file/filename_parser_test.go Normal file
View File

@ -0,0 +1,85 @@
package file
import (
"github.com/dimag-jfrog/migrate/migrate/direction"
"testing"
)
type ParsingTest struct {
filename string
filenameExtension string
expectVersion uint64
expectName string
expectDirection direction.Direction
expectErr bool
}
func testParser(t *testing.T, parser FilenameParser, test *ParsingTest) {
version, name, migrate, err := parser.Parse(test.filename)
if test.expectErr && err == nil {
t.Fatal("Expected error, but got none.", test)
}
if !test.expectErr && err != nil {
t.Fatal("Did not expect error, but got one:", err, test)
}
if err == nil {
if version != test.expectVersion {
t.Error("Wrong version number", test)
}
if name != test.expectName {
t.Error("wrong name", test)
}
if migrate != test.expectDirection {
t.Error("wrong migrate", test)
}
}
}
func TestParseDefaultFilenameSchema(t *testing.T) {
var tests = []ParsingTest {
{"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false},
{"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false},
{"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false},
{"-1_test_file.down.sql", "sql", 0, "", direction.Up, true},
{"test_file.down.sql", "sql", 0, "", direction.Up, true},
{"100_test_file.down", "sql", 0, "", direction.Up, true},
{"100_test_file.sql", "sql", 0, "", direction.Up, true},
{"100_test_file", "sql", 0, "", direction.Up, true},
{"test_file", "sql", 0, "", direction.Up, true},
{"100", "sql", 0, "", direction.Up, true},
{".sql", "sql", 0, "", direction.Up, true},
{"up.sql", "sql", 0, "", direction.Up, true},
{"down.sql", "sql", 0, "", direction.Up, true},
}
for _, test := range tests {
parser := DefaultFilenameParser{FilenameExtension:test.filenameExtension}
testParser(t, &parser, &test)
}
}
func TestParseUpDownAndBothFilenameSchema(t *testing.T) {
var tests = []ParsingTest {
{"001_test_file.up.sql", "sql", 1, "test_file", direction.Up, false},
{"001_test_file.down.sql", "sql", 1, "test_file", direction.Down, false},
{"10034_test_file.down.sql", "sql", 10034, "test_file", direction.Down, false},
{"-1_test_file.down.sql", "sql", 0, "", direction.Up, true},
{"test_file.down.sql", "sql", 0, "", direction.Up, true},
{"100_test_file.down", "sql", 0, "", direction.Up, true},
{"100_test_file.sql", "sql", 100, "test_file", direction.Both, false},
{"001_test_file.mgo", "mgo", 1, "test_file", direction.Both, false},
{"-1_test_file.mgo", "sql", 0, "", direction.Up, true},
{"100_test_file", "sql", 0, "", direction.Up, true},
{"test_file", "sql", 0, "", direction.Up, true},
{"100", "sql", 0, "", direction.Up, true},
{".sql", "sql", 0, "", direction.Up, true},
{"up.sql", "sql", 0, "", direction.Up, true},
{"down.sql", "sql", 0, "", direction.Up, true},
}
for _, test := range tests {
parser := UpDownAndBothFilenameParser{FilenameExtension:test.filenameExtension}
testParser(t, &parser, &test)
}
}

1
migrate/direction/direction.go
View File

@ -6,4 +6,5 @@ type Direction int
const (
Up Direction = +1
Down = -1
Both = 0
)