Merge pull request #227 from fsouza/gcs-source

source/google-cloud-storage: implement the driver

source/google-cloud-storage/README.md

@@ -0,0 +1,3 @@
+# google-cloud-storage
+
+`gcs://<bucket>/<prefix>`

source/google-cloud-storage/storage.go

@@ -1 +1,119 @@
 package googlecloudstorage
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"github.com/mattes/migrate/source"
+	"golang.org/x/net/context"
+	"google.golang.org/api/iterator"
+)
+
+func init() {
+	source.Register("gcs", &gcs{})
+}
+
+type gcs struct {
+	bucket     *storage.BucketHandle
+	prefix     string
+	migrations *source.Migrations
+}
+
+func (g *gcs) Open(folder string) (source.Driver, error) {
+	u, err := url.Parse(folder)
+	if err != nil {
+		return nil, err
+	}
+	client, err := storage.NewClient(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	driver := gcs{
+		bucket:     client.Bucket(u.Host),
+		prefix:     strings.Trim(u.Path, "/") + "/",
+		migrations: source.NewMigrations(),
+	}
+	err = driver.loadMigrations()
+	if err != nil {
+		return nil, err
+	}
+	return &driver, nil
+}
+
+func (g *gcs) loadMigrations() error {
+	iter := g.bucket.Objects(context.Background(), &storage.Query{
+		Prefix:    g.prefix,
+		Delimiter: "/",
+	})
+	object, err := iter.Next()
+	for ; err == nil; object, err = iter.Next() {
+		_, fileName := path.Split(object.Name)
+		m, parseErr := source.DefaultParse(fileName)
+		if parseErr != nil {
+			continue
+		}
+		if !g.migrations.Append(m) {
+			return fmt.Errorf("unable to parse file %v", object.Name)
+		}
+	}
+	if err != iterator.Done {
+		return err
+	}
+	return nil
+}
+
+func (g *gcs) Close() error {
+	return nil
+}
+
+func (g *gcs) First() (uint, error) {
+	v, ok := g.migrations.First()
+	if !ok {
+		return 0, os.ErrNotExist
+	}
+	return v, nil
+}
+
+func (g *gcs) Prev(version uint) (uint, error) {
+	v, ok := g.migrations.Prev(version)
+	if !ok {
+		return 0, os.ErrNotExist
+	}
+	return v, nil
+}
+
+func (g *gcs) Next(version uint) (uint, error) {
+	v, ok := g.migrations.Next(version)
+	if !ok {
+		return 0, os.ErrNotExist
+	}
+	return v, nil
+}
+
+func (g *gcs) ReadUp(version uint) (io.ReadCloser, string, error) {
+	if m, ok := g.migrations.Up(version); ok {
+		return g.open(m)
+	}
+	return nil, "", os.ErrNotExist
+}
+
+func (g *gcs) ReadDown(version uint) (io.ReadCloser, string, error) {
+	if m, ok := g.migrations.Down(version); ok {
+		return g.open(m)
+	}
+	return nil, "", os.ErrNotExist
+}
+
+func (g *gcs) open(m *source.Migration) (io.ReadCloser, string, error) {
+	objectPath := path.Join(g.prefix, m.Raw)
+	reader, err := g.bucket.Object(objectPath).NewReader(context.Background())
+	if err != nil {
+		return nil, "", err
+	}
+	return reader, m.Identifier, nil
+}

source/google-cloud-storage/storage_test.go

@@ -1 +1,37 @@
 package googlecloudstorage
+
+import (
+	"testing"
+
+	"github.com/fsouza/fake-gcs-server/fakestorage"
+	"github.com/mattes/migrate/source"
+	st "github.com/mattes/migrate/source/testing"
+)
+
+func Test(t *testing.T) {
+	server := fakestorage.NewServer([]fakestorage.Object{
+		{BucketName: "some-bucket", Name: "staging/migrations/1_foobar.up.sql", Content: []byte("1 up")},
+		{BucketName: "some-bucket", Name: "staging/migrations/1_foobar.down.sql", Content: []byte("1 down")},
+		{BucketName: "some-bucket", Name: "prod/migrations/1_foobar.up.sql", Content: []byte("1 up")},
+		{BucketName: "some-bucket", Name: "prod/migrations/1_foobar.down.sql", Content: []byte("1 down")},
+		{BucketName: "some-bucket", Name: "prod/migrations/3_foobar.up.sql", Content: []byte("3 up")},
+		{BucketName: "some-bucket", Name: "prod/migrations/4_foobar.up.sql", Content: []byte("4 up")},
+		{BucketName: "some-bucket", Name: "prod/migrations/4_foobar.down.sql", Content: []byte("4 down")},
+		{BucketName: "some-bucket", Name: "prod/migrations/5_foobar.down.sql", Content: []byte("5 down")},
+		{BucketName: "some-bucket", Name: "prod/migrations/7_foobar.up.sql", Content: []byte("7 up")},
+		{BucketName: "some-bucket", Name: "prod/migrations/7_foobar.down.sql", Content: []byte("7 down")},
+		{BucketName: "some-bucket", Name: "prod/migrations/not-a-migration.txt"},
+		{BucketName: "some-bucket", Name: "prod/migrations/0-random-stuff/whatever.txt"},
+	})
+	defer server.Stop()
+	driver := gcs{
+		bucket:     server.Client().Bucket("some-bucket"),
+		prefix:     "prod/migrations/",
+		migrations: source.NewMigrations(),
+	}
+	err := driver.loadMigrations()
+	if err != nil {
+		t.Fatal(err)
+	}
+	st.Test(t, &driver)
+}