From 7c76166697645c992a4b096db647dfe42eb322b7 Mon Sep 17 00:00:00 2001 From: Kasparas Galdikas Date: Sun, 16 Jun 2019 20:48:01 +0100 Subject: [PATCH] Github Enterprise support (#234) * exported Github struct fields and ReadDirectory method * github ee implementation, tests and docs * build fixes * Github Enterprise API endpoint based on docs * addressing PR comments * code review * make linter happy * parseBool() takes fallback * pr comments * tweaks to Config{} --- Dockerfile | 2 +- Makefile | 6 +- README.md | 1 + internal/cli/build_github_ee.go | 7 +++ source/github/README.md | 2 + source/github/github.go | 80 +++++++++++++++++++------ source/github_ee/.gitignore | 1 + source/github_ee/README.md | 21 +++++++ source/github_ee/github_ee.go | 96 ++++++++++++++++++++++++++++++ source/github_ee/github_ee_test.go | 44 ++++++++++++++ 10 files changed, 238 insertions(+), 22 deletions(-) create mode 100644 internal/cli/build_github_ee.go create mode 100644 source/github_ee/.gitignore create mode 100644 source/github_ee/README.md create mode 100644 source/github_ee/github_ee.go create mode 100644 source/github_ee/github_ee_test.go diff --git a/Dockerfile b/Dockerfile index f97ccc0..d43c463 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ COPY . ./ ENV GO111MODULE=on ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver" -ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab" +ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab" RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate diff --git a/Makefile b/Makefile index 3bc0de9..1ff5300 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE ?= file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab +SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= @@ -33,7 +33,7 @@ test: test-with-flags: - @echo SOURCE: $(SOURCE) + @echo SOURCE: $(SOURCE) @echo DATABASE: $(DATABASE) @go test $(TEST_FLAGS) . @@ -84,7 +84,7 @@ rewrite-import-paths: docs: -make kill-docs nohup godoc -play -http=127.0.0.1:6064 /dev/null 2>&1 & echo $$! > .godoc.pid - cat .godoc.pid + cat .godoc.pid kill-docs: diff --git a/README.md b/README.md index 307fe9b..d251db0 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ Source drivers read migrations from local or remote sources. [Add a new source?] * [Filesystem](source/file) - read from filesystem * [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata)) * [Github](source/github) - read from remote Github repositories +* [Github Enterprise](source/github_ee) - read from remote Github Enterprise repositories * [Gitlab](source/gitlab) - read from remote Gitlab repositories * [AWS S3](source/aws_s3) - read from Amazon Web Services S3 * [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage diff --git a/internal/cli/build_github_ee.go b/internal/cli/build_github_ee.go new file mode 100644 index 0000000..4e705da --- /dev/null +++ b/internal/cli/build_github_ee.go @@ -0,0 +1,7 @@ +// +build github + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/source/github_ee" +) diff --git a/source/github/README.md b/source/github/README.md index b4fbc1a..53f8246 100644 --- a/source/github/README.md +++ b/source/github/README.md @@ -1,5 +1,7 @@ # github +This driver is catered for those that want to source migrations from [github.com](https://github.com). The URL scheme doesn't require a hostname, as it just simply defaults to `github.com`. + `github://user:personal-access-token@owner/repo/path#ref` | URL Query | WithInstance Config | Description | diff --git a/source/github/github.go b/source/github/github.go index 0ff3eba..8eeadcc 100644 --- a/source/github/github.go +++ b/source/github/github.go @@ -29,17 +29,17 @@ var ( ) type Github struct { - client *github.Client - url string - - pathOwner string - pathRepo string - path string + config *Config + client *github.Client options *github.RepositoryContentGetOptions migrations *source.Migrations } type Config struct { + Owner string + Repo string + Path string + Ref string } func (g *Github) Open(url string) (source.Driver, error) { @@ -64,20 +64,21 @@ func (g *Github) Open(url string) (source.Driver, error) { gn := &Github{ client: github.NewClient(tr.Client()), - url: url, migrations: source.NewMigrations(), options: &github.RepositoryContentGetOptions{Ref: u.Fragment}, } + gn.ensureFields() + // set owner, repo and path in repo - gn.pathOwner = u.Host + gn.config.Owner = u.Host pe := strings.Split(strings.Trim(u.Path, "/"), "/") if len(pe) < 1 { return nil, ErrInvalidRepo } - gn.pathRepo = pe[0] + gn.config.Repo = pe[0] if len(pe) > 1 { - gn.path = strings.Join(pe[1:], "/") + gn.config.Path = strings.Join(pe[1:], "/") } if err := gn.readDirectory(); err != nil { @@ -90,16 +91,29 @@ func (g *Github) Open(url string) (source.Driver, error) { func WithInstance(client *github.Client, config *Config) (source.Driver, error) { gn := &Github{ client: client, + config: config, migrations: source.NewMigrations(), + options: &github.RepositoryContentGetOptions{Ref: config.Ref}, } + if err := gn.readDirectory(); err != nil { return nil, err } + return gn, nil } func (g *Github) readDirectory() error { - fileContent, dirContents, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, g.path, g.options) + g.ensureFields() + + fileContent, dirContents, _, err := g.client.Repositories.GetContents( + context.Background(), + g.config.Owner, + g.config.Repo, + g.config.Path, + g.options, + ) + if err != nil { return err } @@ -120,37 +134,58 @@ func (g *Github) readDirectory() error { return nil } +func (g *Github) ensureFields() { + if g.config == nil { + g.config = &Config{} + } +} + func (g *Github) Close() error { return nil } func (g *Github) First() (version uint, er error) { + g.ensureFields() + if v, ok := g.migrations.First(); !ok { - return 0, &os.PathError{Op: "first", Path: g.path, Err: os.ErrNotExist} + return 0, &os.PathError{Op: "first", Path: g.config.Path, Err: os.ErrNotExist} } else { return v, nil } } func (g *Github) Prev(version uint) (prevVersion uint, err error) { + g.ensureFields() + if v, ok := g.migrations.Prev(version); !ok { - return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: g.path, Err: os.ErrNotExist} + return 0, &os.PathError{Op: fmt.Sprintf("prev for version %v", version), Path: g.config.Path, Err: os.ErrNotExist} } else { return v, nil } } func (g *Github) Next(version uint) (nextVersion uint, err error) { + g.ensureFields() + if v, ok := g.migrations.Next(version); !ok { - return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: g.path, Err: os.ErrNotExist} + return 0, &os.PathError{Op: fmt.Sprintf("next for version %v", version), Path: g.config.Path, Err: os.ErrNotExist} } else { return v, nil } } func (g *Github) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { + g.ensureFields() + if m, ok := g.migrations.Up(version); ok { - file, _, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, path.Join(g.path, m.Raw), g.options) + file, _, _, err := g.client.Repositories.GetContents( + context.Background(), + g.config.Owner, + g.config.Repo, + path.Join(g.config.Path, m.Raw), + g.options, + ) + if err != nil { return nil, "", err } @@ -162,12 +197,21 @@ func (g *Github) ReadUp(version uint) (r io.ReadCloser, identifier string, err e return ioutil.NopCloser(strings.NewReader(r)), m.Identifier, nil } } - return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.path, Err: os.ErrNotExist} + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.config.Path, Err: os.ErrNotExist} } func (g *Github) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { + g.ensureFields() + if m, ok := g.migrations.Down(version); ok { - file, _, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, path.Join(g.path, m.Raw), g.options) + file, _, _, err := g.client.Repositories.GetContents( + context.Background(), + g.config.Owner, + g.config.Repo, + path.Join(g.config.Path, m.Raw), + g.options, + ) + if err != nil { return nil, "", err } @@ -179,5 +223,5 @@ func (g *Github) ReadDown(version uint) (r io.ReadCloser, identifier string, err return ioutil.NopCloser(strings.NewReader(r)), m.Identifier, nil } } - return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.path, Err: os.ErrNotExist} + return nil, "", &os.PathError{Op: fmt.Sprintf("read version %v", version), Path: g.config.Path, Err: os.ErrNotExist} } diff --git a/source/github_ee/.gitignore b/source/github_ee/.gitignore new file mode 100644 index 0000000..3006ad5 --- /dev/null +++ b/source/github_ee/.gitignore @@ -0,0 +1 @@ +.github_test_secrets diff --git a/source/github_ee/README.md b/source/github_ee/README.md new file mode 100644 index 0000000..f5c41ed --- /dev/null +++ b/source/github_ee/README.md @@ -0,0 +1,21 @@ +# github ee + +## Github Enterprise Edition + +This driver is catered for those who run Github Enterprise under private infrastructure. + +The below URL scheme illustrates how to source migration files from Github Enterprise. + +Github client for Go requires API and Uploads endpoint hosts in order to create an instance of Github Enterprise Client. We're making an assumption that the API and Uploads are available under `https://api.*` and `https://uploads.*` respectively. [Github Enterprise Installation Guide](https://help.github.com/en/enterprise/2.15/admin/installation/enabling-subdomain-isolation) recommends that you enable Subdomain isolation feature. + +`github-ee://user:personal-access-token@host/owner/repo/path?verify-tls=true#ref` + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| user | | The username of the user connecting | +| personal-access-token | | Personal access token from your Github Enterprise instance | +| owner | | the repo owner | +| repo | | the name of the repository | +| path | | path in repo to migrations | +| ref | | (optional) can be a SHA, branch, or tag | +| verify-tls | | (optional) defaults to `true`. This option sets `tls.Config.InsecureSkipVerify` accordingly | diff --git a/source/github_ee/github_ee.go b/source/github_ee/github_ee.go new file mode 100644 index 0000000..b01aa41 --- /dev/null +++ b/source/github_ee/github_ee.go @@ -0,0 +1,96 @@ +package github_ee + +import ( + "crypto/tls" + "fmt" + "net/http" + nurl "net/url" + "strconv" + "strings" + + "github.com/golang-migrate/migrate/v4/source" + gh "github.com/golang-migrate/migrate/v4/source/github" + "github.com/google/go-github/github" +) + +func init() { + source.Register("github-ee", &GithubEE{}) +} + +type GithubEE struct { + source.Driver +} + +func (g *GithubEE) Open(url string) (source.Driver, error) { + verifyTLS := true + + u, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + if o := u.Query().Get("verify-tls"); o != "" { + verifyTLS = parseBool(o, verifyTLS) + } + + if u.User == nil { + return nil, gh.ErrNoUserInfo + } + + password, ok := u.User.Password() + if !ok { + return nil, gh.ErrNoUserInfo + } + + ghc, err := g.createGithubClient(u.Host, u.User.Username(), password, verifyTLS) + if err != nil { + return nil, err + } + + pe := strings.Split(strings.Trim(u.Path, "/"), "/") + + if len(pe) < 1 { + return nil, gh.ErrInvalidRepo + } + + cfg := &gh.Config{ + Owner: pe[0], + Repo: pe[1], + Ref: u.Fragment, + } + + if len(pe) > 2 { + cfg.Path = strings.Join(pe[2:], "/") + } + + i, err := gh.WithInstance(ghc, cfg) + if err != nil { + return nil, err + } + + return &GithubEE{Driver: i}, nil +} + +func (g *GithubEE) createGithubClient(host, username, password string, verifyTLS bool) (*github.Client, error) { + tr := &github.BasicAuthTransport{ + Username: username, + Password: password, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: !verifyTLS}, + }, + } + + apiHost := fmt.Sprintf("https://%s/api/v3", host) + uploadHost := fmt.Sprintf("https://uploads.%s", host) + + return github.NewEnterpriseClient(apiHost, uploadHost, tr.Client()) +} + +func parseBool(val string, fallback bool) bool { + b, err := strconv.ParseBool(val) + if err != nil { + return fallback + } + + return b +} diff --git a/source/github_ee/github_ee_test.go b/source/github_ee/github_ee_test.go new file mode 100644 index 0000000..3a82249 --- /dev/null +++ b/source/github_ee/github_ee_test.go @@ -0,0 +1,44 @@ +package github_ee + +import ( + "net/http" + "net/http/httptest" + nurl "net/url" + "testing" +) + +func Test(t *testing.T) { + ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v3/repos/mattes/migrate_test_tmp/contents/test" { + w.WriteHeader(http.StatusNotFound) + return + } + + if ref := r.URL.Query().Get("ref"); ref != "452b8003e7" { + w.WriteHeader(http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + _, err := w.Write([]byte("[]")) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + })) + defer ts.Close() + + u, err := nurl.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + g := &GithubEE{} + _, err = g.Open("github-ee://foo:bar@" + u.Host + "/mattes/migrate_test_tmp/test?verify-tls=false#452b8003e7") + + if err != nil { + t.Fatal(err) + } +}