From 14636b77fb0946fb0692fe340d3901777986ea01 Mon Sep 17 00:00:00 2001 From: kshvakov Date: Wed, 21 Jun 2017 15:58:46 +0300 Subject: [PATCH] Add ClickHouse driver --- Makefile | 2 +- README.md | 2 +- cli/build_clickhouse.go | 8 + database/clickhouse/README.md | 12 ++ database/clickhouse/clickhouse.go | 161 ++++++++++++++++++ .../examples/migrations/001_init.down.sql | 1 + .../examples/migrations/001_init.up.sql | 3 + .../migrations/002_create_table.down.sql | 1 + .../migrations/002_create_table.up.sql | 3 + 9 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 cli/build_clickhouse.go create mode 100644 database/clickhouse/README.md create mode 100644 database/clickhouse/clickhouse.go create mode 100644 database/clickhouse/examples/migrations/001_init.down.sql create mode 100644 database/clickhouse/examples/migrations/001_init.up.sql create mode 100644 database/clickhouse/examples/migrations/002_create_table.down.sql create mode 100644 database/clickhouse/examples/migrations/002_create_table.up.sql diff --git a/Makefile b/Makefile index f804816..aa80001 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go-bindata github aws-s3 google-cloud-storage -DATABASE ?= postgres mysql redshift cassandra sqlite3 spanner +DATABASE ?= postgres mysql redshift cassandra sqlite3 spanner clickhouse VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/README.md b/README.md index f8c8993..2ad6f54 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170)) * [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171)) * [Google Cloud Spanner](database/spanner) - + * [ClickHouse](database/clickhouse) ## Migration Sources diff --git a/cli/build_clickhouse.go b/cli/build_clickhouse.go new file mode 100644 index 0000000..c9175e2 --- /dev/null +++ b/cli/build_clickhouse.go @@ -0,0 +1,8 @@ +// +build clickhouse + +package main + +import ( + _ "github.com/kshvakov/clickhouse" + _ "github.com/mattes/migrate/database/clickhouse" +) diff --git a/database/clickhouse/README.md b/database/clickhouse/README.md new file mode 100644 index 0000000..16dbbf9 --- /dev/null +++ b/database/clickhouse/README.md @@ -0,0 +1,12 @@ +# ClickHouse + +`clickhouse://host:port?username=user&password=qwerty&database=clicks` + +| URL Query | Description | +|------------|-------------| +| `x-migrations-table`| Name of the migrations table | +| `database` | The name of the database to connect to | +| `username` | The user to sign in as | +| `password` | The user's password | +| `host` | The host to connect to. | +| `port` | The port to bind to. | diff --git a/database/clickhouse/clickhouse.go b/database/clickhouse/clickhouse.go new file mode 100644 index 0000000..1704a7c --- /dev/null +++ b/database/clickhouse/clickhouse.go @@ -0,0 +1,161 @@ +package clickhouse + +import ( + "database/sql" + "io" + "io/ioutil" + "net/url" + "time" + + "github.com/mattes/migrate" + "github.com/mattes/migrate/database" +) + +var DefaultMigrationsTable = "schema_migrations" + +type config struct { + table string + database string +} + +func init() { + database.Register("clickhouse", &ClickHouse{}) +} + +type ClickHouse struct { + conn *sql.DB + config config +} + +func (ch *ClickHouse) Open(dsn string) (database.Driver, error) { + purl, err := url.Parse(dsn) + if err != nil { + return nil, err + } + q := migrate.FilterCustomQuery(purl) + q.Scheme = "tcp" + conn, err := sql.Open("clickhouse", q.String()) + if err != nil { + return nil, err + } + table := purl.Query().Get("x-migrations-table") + if len(table) == 0 { + table = DefaultMigrationsTable + } + database := purl.Query().Get("database") + if len(database) == 0 { + if err := conn.QueryRow("SELECT currentDatabase()").Scan(&database); err != nil { + return nil, err + } + } + ch = &ClickHouse{ + conn: conn, + config: config{ + table: table, + database: database, + }, + } + if err := ch.ensureVersionTable(); err != nil { + return nil, err + } + return ch, nil +} + +func (ch *ClickHouse) Run(r io.Reader) error { + migration, err := ioutil.ReadAll(r) + if err != nil { + return err + } + if _, err := ch.conn.Exec(string(migration)); err != nil { + return database.Error{OrigErr: err, Err: "migration failed", Query: migration} + } + + return nil +} +func (ch *ClickHouse) Version() (int, bool, error) { + var ( + version int + dirty uint8 + query = "SELECT version, dirty FROM `" + ch.config.table + "` ORDER BY sequence DESC LIMIT 1" + ) + if err := ch.conn.QueryRow(query).Scan(&version, &dirty); err != nil { + if err == sql.ErrNoRows { + return database.NilVersion, false, nil + } + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + } + return version, dirty == 1, nil +} + +func (ch *ClickHouse) SetVersion(version int, dirty bool) error { + var ( + bool = func(v bool) uint8 { + if v { + return 1 + } + return 0 + } + tx, err = ch.conn.Begin() + ) + if err != nil { + return err + } + query := "INSERT INTO " + ch.config.table + " (version, dirty, sequence) VALUES (?, ?, ?)" + if _, err := tx.Exec(query, version, bool(dirty), time.Now().UnixNano()); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return tx.Commit() +} + +func (ch *ClickHouse) ensureVersionTable() error { + var ( + table string + query = "SHOW TABLES FROM " + ch.config.database + " LIKE '" + ch.config.table + "'" + ) + // check if migration table exists + if err := ch.conn.QueryRow(query).Scan(&table); err != nil { + if err != sql.ErrNoRows { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } else { + return nil + } + // if not, create the empty migration table + query = ` + CREATE TABLE ` + ch.config.table + ` ( + version UInt32, + dirty UInt8, + sequence UInt64 + ) Engine=TinyLog + ` + if _, err := ch.conn.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil +} + +func (ch *ClickHouse) Drop() error { + var ( + query = "SHOW TABLES FROM " + ch.config.database + tables, err = ch.conn.Query(query) + ) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer tables.Close() + for tables.Next() { + var table string + if err := tables.Scan(&table); err != nil { + return err + } + query = "DROP TABLE IF EXISTS " + ch.config.database + "." + table + if _, err := ch.conn.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + return ch.ensureVersionTable() +} + +func (ch *ClickHouse) Lock() error { return nil } +func (ch *ClickHouse) Unlock() error { return nil } +func (ch *ClickHouse) Close() error { return ch.conn.Close() } diff --git a/database/clickhouse/examples/migrations/001_init.down.sql b/database/clickhouse/examples/migrations/001_init.down.sql new file mode 100644 index 0000000..51cd8bf --- /dev/null +++ b/database/clickhouse/examples/migrations/001_init.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS test_1; \ No newline at end of file diff --git a/database/clickhouse/examples/migrations/001_init.up.sql b/database/clickhouse/examples/migrations/001_init.up.sql new file mode 100644 index 0000000..5436b6f --- /dev/null +++ b/database/clickhouse/examples/migrations/001_init.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE test_1 ( + Date Date +) Engine=Memory; \ No newline at end of file diff --git a/database/clickhouse/examples/migrations/002_create_table.down.sql b/database/clickhouse/examples/migrations/002_create_table.down.sql new file mode 100644 index 0000000..9d77122 --- /dev/null +++ b/database/clickhouse/examples/migrations/002_create_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS test_2; \ No newline at end of file diff --git a/database/clickhouse/examples/migrations/002_create_table.up.sql b/database/clickhouse/examples/migrations/002_create_table.up.sql new file mode 100644 index 0000000..6b49ed9 --- /dev/null +++ b/database/clickhouse/examples/migrations/002_create_table.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE test_2 ( + Date Date +) Engine=Memory; \ No newline at end of file