diff --git a/Makefile b/Makefile index efc8e10..0cb226e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go_bindata github aws_s3 google_cloud_storage godoc_vfs -DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse +DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/database/mongodb/README.md b/database/mongodb/README.md index e69de29..9bb0344 100644 --- a/database/mongodb/README.md +++ b/database/mongodb/README.md @@ -0,0 +1,19 @@ +# MongoDB + +* Driver work with mongo through [db.runCommands](https://docs.mongodb.com/manual/reference/command/) +* Migrations support json format. It contains array of commands for `db.runCommand`. Every command is executed in separate request to database +* All keys have to be in quotes `"` +* [Examples](./examples) + +# Usage + +`mongodb://user:password@host:port/dbname?query` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table | +| `dbname` | `DatabaseName` | The name of the database to connect to | +| `user` | | The user to sign in as. Can be omitted | +| `password` | | The user's password. Can be omitted | +| `host` | | The host to connect to | +| `port` | | The port to bind to | \ No newline at end of file diff --git a/database/mongodb/examples/001_create_user.down.json b/database/mongodb/examples/001_create_user.down.json new file mode 100644 index 0000000..6bba284 --- /dev/null +++ b/database/mongodb/examples/001_create_user.down.json @@ -0,0 +1,5 @@ +[ + { + "dropUser": "deminem" + } +] \ No newline at end of file diff --git a/database/mongodb/examples/001_create_user.up.json b/database/mongodb/examples/001_create_user.up.json new file mode 100644 index 0000000..6c37cb7 --- /dev/null +++ b/database/mongodb/examples/001_create_user.up.json @@ -0,0 +1,12 @@ +[ + { + "createUser": "deminem", + "pwd": "gogo", + "roles": [ + { + "role": "readWrite", + "db": "testMigration" + } + ] + } +] \ No newline at end of file diff --git a/database/mongodb/examples/002_create_indexes.down.json b/database/mongodb/examples/002_create_indexes.down.json new file mode 100644 index 0000000..6bba481 --- /dev/null +++ b/database/mongodb/examples/002_create_indexes.down.json @@ -0,0 +1,10 @@ +[ + { + "dropIndexes": "mycollection", + "index": "username_sort_by_asc_created" + }, + { + "dropIndexes": "mycollection", + "index": "unique_email" + } +] \ No newline at end of file diff --git a/database/mongodb/examples/002_create_indexes.up.json b/database/mongodb/examples/002_create_indexes.up.json new file mode 100644 index 0000000..e2995a2 --- /dev/null +++ b/database/mongodb/examples/002_create_indexes.up.json @@ -0,0 +1,21 @@ +[{ + "createIndexes": "mycollection", + "indexes": [ + { + "key": { + "username": 1, + "created": -1 + }, + "name": "username_sort_by_asc_created", + "background": true + }, + { + "key": { + "email": 1 + }, + "name": "unique_email", + "unique": true, + "background": true + } + ] +}] \ No newline at end of file diff --git a/database/mongodb/mongodb.go b/database/mongodb/mongodb.go new file mode 100644 index 0000000..3a9e621 --- /dev/null +++ b/database/mongodb/mongodb.go @@ -0,0 +1,160 @@ +package mongodb + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/url" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/mongodb/mongo-go-driver/bson" + "github.com/mongodb/mongo-go-driver/mongo" + "github.com/mongodb/mongo-go-driver/x/bsonx" + "github.com/mongodb/mongo-go-driver/x/network/connstring" +) + +func init() { + database.Register("mongodb", &Mongo{}) +} + +var DefaultMigrationsTable = "schema_migrations" + +var ( + ErrNoDatabaseName = fmt.Errorf("no database name") + ErrNilConfig = fmt.Errorf("no config") +) + +type Mongo struct { + client *mongo.Client + db *mongo.Database + + config *Config +} + +type Config struct { + DatabaseName string + MigrationsCollection string +} + +type versionInfo struct { + Version int `bson:"version"` + Dirty bool `bson:"dirty"` +} + +func WithInstance(instance *mongo.Client, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + if len(config.DatabaseName) == 0 { + return nil, ErrNoDatabaseName + } + if len(config.MigrationsCollection) == 0 { + config.MigrationsCollection = DefaultMigrationsTable + } + mc := &Mongo{ + client: instance, + db: instance.Database(config.DatabaseName), + config: config, + } + return mc, nil +} + +func (m *Mongo) Open(dsn string) (database.Driver, error) { + uri, err := connstring.Parse(dsn) + if err != nil { + return nil, err + } + if len(uri.Database) == 0 { + return nil, ErrNoDatabaseName + } + + purl, err := url.Parse(dsn) + if err != nil { + return nil, err + } + migrationsCollection := purl.Query().Get("x-migrations-collection") + if len(migrationsCollection) == 0 { + migrationsCollection = DefaultMigrationsTable + } + + q := migrate.FilterCustomQuery(purl) + q.Scheme = "mongodb" + + client, err := mongo.Connect(context.TODO(), q.String()) + if err != nil { + return nil, err + } + if err = client.Ping(context.TODO(), nil); err != nil { + return nil, err + } + mc, err := WithInstance(client, &Config{ + DatabaseName: uri.Database, + MigrationsCollection: migrationsCollection, + }) + if err != nil { + return nil, err + } + return mc, nil +} + +func (m *Mongo) SetVersion(version int, dirty bool) error { + migrationsCollection := m.db.Collection(m.config.MigrationsCollection) + if err := migrationsCollection.Drop(context.TODO()); err != nil { + return &database.Error{OrigErr: err, Err: "drop migrations collection failed"} + } + _, err := migrationsCollection.InsertOne(context.TODO(), bson.M{"version": version, "dirty": dirty}) + if err != nil { + return &database.Error{OrigErr: err, Err: "save version failed"} + } + return nil +} + +func (m *Mongo) Version() (version int, dirty bool, err error) { + var versionInfo versionInfo + err = m.db.Collection(m.config.MigrationsCollection).FindOne(context.TODO(), nil).Decode(&versionInfo) + switch { + case err == mongo.ErrNoDocuments: + return database.NilVersion, false, nil + case err != nil: + return 0, false, &database.Error{OrigErr: err, Err: "failed to get migration version"} + default: + return versionInfo.Version, versionInfo.Dirty, nil + } +} + +func (m *Mongo) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + var cmds []bsonx.Doc + err = bson.UnmarshalExtJSON(migr, true, &cmds) + if err != nil { + return fmt.Errorf("unmarshaling json error: %s", err) + } + for _, cmd := range cmds { + err := m.db.RunCommand(context.TODO(), cmd).Err() + if err != nil { + return err + } + } + return nil +} + +func (m *Mongo) Close() error { + return m.client.Disconnect(context.TODO()) +} + +func (m *Mongo) Drop() error { + return m.db.Drop(context.TODO()) +} + +func (m *Mongo) Lock() error { + return nil +} + +func (m *Mongo) Unlock() error { + return nil +} diff --git a/database/mongodb/mongodb_test.go b/database/mongodb/mongodb_test.go new file mode 100644 index 0000000..ddceb64 --- /dev/null +++ b/database/mongodb/mongodb_test.go @@ -0,0 +1,97 @@ +package mongodb + +import ( + "bytes" + "context" + "fmt" + "io" + "testing" + + dt "github.com/golang-migrate/migrate/v4/database/testing" + mt "github.com/golang-migrate/migrate/v4/testing" + "github.com/mongodb/mongo-go-driver/mongo" +) + +var versions = []mt.Version{ + {Image: "mongo:4"}, + {Image: "mongo:3"}, +} + +func mongoConnectionString(host string, port uint) string { + return fmt.Sprintf("mongodb://%s:%v/testMigration", host, port) +} + +func isReady(i mt.Instance) bool { + client, err := mongo.Connect(context.TODO(), mongoConnectionString(i.Host(), i.Port())) + if err != nil { + return false + } + defer client.Disconnect(context.TODO()) + if err = client.Ping(context.TODO(), nil); err != nil { + switch err { + case io.EOF: + return false + default: + fmt.Println(err) + } + return false + } + return true +} + +func Test(t *testing.T) { + mt.ParallelTest(t, versions, isReady, + func(t *testing.T, i mt.Instance) { + p := &Mongo{} + addr := mongoConnectionString(i.Host(), i.Port()) + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + defer d.Close() + dt.TestNilVersion(t, d) + //TestLockAndUnlock(t, d) driver doesn't support lock on database level + dt.TestRun(t, d, bytes.NewReader([]byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`))) + dt.TestSetVersion(t, d) + dt.TestDrop(t, d) + }) +} + +func TestWithAuth(t *testing.T) { + mt.ParallelTest(t, versions, isReady, + func(t *testing.T, i mt.Instance) { + p := &Mongo{} + addr := mongoConnectionString(i.Host(), i.Port()) + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + defer d.Close() + createUserCMD := []byte(`[{"createUser":"deminem","pwd":"gogo","roles":[{"role":"readWrite","db":"testMigration"}]}]`) + err = d.Run(bytes.NewReader(createUserCMD)) + if err != nil { + t.Fatalf("%v", err) + } + + driverWithAuth, err := p.Open(fmt.Sprintf("mongodb://deminem:gogo@%s:%v/testMigration", i.Host(), i.Port())) + if err != nil { + t.Fatalf("%v", err) + } + defer driverWithAuth.Close() + insertCMD := []byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`) + err = driverWithAuth.Run(bytes.NewReader(insertCMD)) + if err != nil { + t.Fatalf("%v", err) + } + + driverWithWrongAuth, err := p.Open(fmt.Sprintf("mongodb://wrong:auth@%s:%v/testMigration", i.Host(), i.Port())) + if err != nil { + t.Fatalf("%v", err) + } + defer driverWithWrongAuth.Close() + err = driverWithWrongAuth.Run(bytes.NewReader(insertCMD)) + if err == nil { + t.Fatal("no error with wrong authorization") + } + }) +} diff --git a/go.mod b/go.mod index 7e5b805..bf695e3 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/fsouza/fake-gcs-server v1.3.0 github.com/go-ini/ini v1.39.0 // indirect github.com/go-sql-driver/mysql v1.4.1 + github.com/go-stack/stack v1.8.0 // indirect github.com/gocql/gocql v0.0.0-20181012100315-44e29ed5b8a4 github.com/gogo/protobuf v1.1.1 // indirect github.com/google/go-cmp v0.2.0 // indirect @@ -34,6 +35,7 @@ require ( github.com/kshvakov/clickhouse v1.3.4 github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.9.0 + github.com/mongodb/mongo-go-driver v0.1.0 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/pkg/errors v0.8.0 // indirect @@ -43,6 +45,9 @@ require ( github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect github.com/stretchr/testify v1.2.2 // indirect + github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect + github.com/xdg/stringprep v1.0.0 // indirect + golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 // indirect golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced // indirect golang.org/x/sys v0.0.0-20181011152604-fa43e7bc11ba // indirect diff --git a/go.sum b/go.sum index bf267b4..09fe59d 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/go-ini/ini v1.39.0 h1:/CyW/jTlZLjuzy52jc1XnhJm6IUKEuunpJFpecywNeI= github.com/go-ini/ini v1.39.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v0.0.0-20181012100315-44e29ed5b8a4 h1:Zqj6+hV7PwTdjwDMZ78rX9ZMi8VCX9HEJmMhsQhTrSY= github.com/gocql/gocql v0.0.0-20181012100315-44e29ed5b8a4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= @@ -113,6 +115,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mongodb/mongo-go-driver v0.1.0 h1:LcpPFw0tNumIAakvNrkI9S9wdX0iOxvMLw/+hcAdHaU= +github.com/mongodb/mongo-go-driver v0.1.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= @@ -137,8 +141,14 @@ github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbm github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= go.opencensus.io v0.17.0 h1:2Cu88MYg+1LU+WVD+NWwYhyP0kKgRlN9QjWGaX0jKTE= go.opencensus.io v0.17.0/go.mod h1:mp1VrMQxhlqqDpKvH4UcQUa4YwlzNmymAjPrDdfxNpI= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/internal/cli/build_mongodb.go b/internal/cli/build_mongodb.go new file mode 100644 index 0000000..8b6fd7d --- /dev/null +++ b/internal/cli/build_mongodb.go @@ -0,0 +1,7 @@ +// +build mongodb + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/mongodb" +)