Upgrade migrate (#1643)

This commit is contained in:
Adam Babik 2019-10-14 16:10:48 +02:00 committed by GitHub
parent 676602f2ed
commit a244b01a26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 1374 additions and 309 deletions

View File

@ -1,5 +1,5 @@
# Build status-go in a Go builder container
FROM golang:1.12.5-alpine as builder
FROM golang:1.13-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers

6
go.mod
View File

@ -1,6 +1,6 @@
module github.com/status-im/status-go
go 1.12
go 1.13
replace github.com/ethereum/go-ethereum v1.9.5 => github.com/status-im/go-ethereum v1.9.5-status.4
@ -29,9 +29,9 @@ require (
github.com/russolsen/transit v0.0.0-20180705123435-0794b4c4505a
github.com/status-im/doubleratchet v2.0.0+incompatible
github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 // indirect
github.com/status-im/migrate/v4 v4.3.1-status.0.20190822050738-a9d340ec8fb7
github.com/status-im/migrate/v4 v4.6.2-status.2
github.com/status-im/rendezvous v1.3.0
github.com/status-im/status-protocol-go v0.3.1
github.com/status-im/status-protocol-go v0.4.2
github.com/status-im/whisper v1.5.1
github.com/stretchr/testify v1.4.0
github.com/syndtr/goleveldb v1.0.0

33
go.sum
View File

@ -69,6 +69,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvvatEupQk=
github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@ -105,8 +107,8 @@ github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtf
github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v0.0.0-20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ=
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 h1:mzrx39dGtGq0VEnTHjnakmczd4uFbhx2cZU3BJDsLdc=
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
@ -151,10 +153,8 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang-migrate/migrate v3.5.4+incompatible h1:R7OzwvCJTCgwapPCiX6DyBiu2czIUMDCB118gFTKTUA=
github.com/golang-migrate/migrate v3.5.4+incompatible/go.mod h1:IsVUlFN5puWOmXrqjgGUfIRIbU7mr8oNBE2tyERd9Wk=
github.com/golang-migrate/migrate/v4 v4.5.0 h1:ucd2qJu1BAKTtmjh7QlWYiq01DwlE/xXYQkOPE0ZTsI=
github.com/golang-migrate/migrate/v4 v4.5.0/go.mod h1:SzAcz2l+yDJVhQC7fwiF7T2MAFPMIkigJz98klRJ4OE=
github.com/golang-migrate/migrate/v4 v4.6.2 h1:LDDOHo/q1W5UDj6PbkxdCv7lv9yunyZHXvxuwDkGo3k=
github.com/golang-migrate/migrate/v4 v4.6.2/go.mod h1:JYi6reN3+Z734VZ0akNuyOJNcrg45ZL7LDBMW3WGJL0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -417,7 +417,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78=
@ -555,13 +556,14 @@ github.com/status-im/go-multiaddr-ethv4 v1.2.0 h1:OT84UsUzTCwguqCpJqkrCMiL4VZ1Sv
github.com/status-im/go-multiaddr-ethv4 v1.2.0/go.mod h1:2VQ3C+9zEurcceasz12gPAtmEzCeyLUGPeKLSXYQKHo=
github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw=
github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
github.com/status-im/migrate/v4 v4.0.0-20190821140204-a9d340ec8fb76af4afda06acf01740d45d2661ed/go.mod h1:r8HggRBZ/k7TRwByq/Hp3P/ubFppIna0nvyavVK0pjA=
github.com/status-im/migrate/v4 v4.3.1-status.0.20190822050738-a9d340ec8fb7 h1:gWtw0g41Y55lIOjUikkx0qjSfzl5lNZWhsvzWZIdtvQ=
github.com/status-im/migrate/v4 v4.3.1-status.0.20190822050738-a9d340ec8fb7/go.mod h1:r8HggRBZ/k7TRwByq/Hp3P/ubFppIna0nvyavVK0pjA=
github.com/status-im/migrate/v4 v4.6.2-status.2 h1:SdC+sMDl/aI7vUlwD2qj2p7KsK4T60IS9z4/rYCCbI8=
github.com/status-im/migrate/v4 v4.6.2-status.2/go.mod h1:c/kc90n47GZu/58nnz1OMLTf7uE4Da4gZP5qmU+A/v8=
github.com/status-im/rendezvous v1.3.0 h1:7RK/MXXW+tlm0asKm1u7Qp7Yni6AO29a7j8+E4Lbjg4=
github.com/status-im/rendezvous v1.3.0/go.mod h1:+hzjuP+j/XzLPeF6E50b88pWOTLdTcwjvNYt+Gh1W1s=
github.com/status-im/status-protocol-go v0.3.1 h1:UrYdxKeW1GlFDtXWLyR66ebhUxmH1yWy46kal7gaSZ0=
github.com/status-im/status-protocol-go v0.3.1/go.mod h1:GyEipgAnT0gBMF57gN1yPAicFMM66VZEe2E8M5ktErI=
github.com/status-im/status-protocol-go v0.4.1-0.20191014103017-1f15a058b66c h1:e0WWnear7T8mXXmU7kKMOrnxGQ95AJOjDJTp58Do0Iw=
github.com/status-im/status-protocol-go v0.4.1-0.20191014103017-1f15a058b66c/go.mod h1:a1kbCiz4yIs2lxJEHsp5eFjdhB+5RUTN8+Uec6wWa4s=
github.com/status-im/status-protocol-go v0.4.2 h1:HAX51q3j+WAzbyeqryJEY1O6hqzTAbYEjkppIXkcNvg=
github.com/status-im/status-protocol-go v0.4.2/go.mod h1:WsfEnZo8dWadTbMqAoLztC/5p9SDwFeEGCmVDzSEmrQ=
github.com/status-im/whisper v1.5.1 h1:87/XIg0Wjua7lXBGiEXgAfTOqlt2Q1dMDuxugTyZbbA=
github.com/status-im/whisper v1.5.1/go.mod h1:emrOxzJme0k66QtbbQ2bdd3P8RCdLZ8sTD7SkwH1s2s=
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE=
@ -586,8 +588,10 @@ github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2
github.com/uber/jaeger-client-go v0.0.0-20180607151842-f7e0d4744fa6/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v0.0.0-20180615202729-a51202d6f4a7/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/vacp2p/mvds v0.0.21 h1:YeYja8noKsHvrnOHM4pqjBvDwwy+kUzXMkX1IBYJAbU=
github.com/vacp2p/mvds v0.0.21/go.mod h1:pIqr2Hg4cIkTJniGPCp4ptong2jxgxx6uToVoY94+II=
github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d h1:kc/9/ZbwyqIKQ2phzmPpBYGx7v2olwD2P7Uj8KKd+Bw=
github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d/go.mod h1:uUmtiahU7efOVl/5w5yk9jOze5xYpDZDrSrT8TvHXjQ=
github.com/vacp2p/mvds v0.0.23 h1:BKdn7tyGvl/J/Pwv6FlcW6Xbzm+17jv141GB1mFXyOU=
github.com/vacp2p/mvds v0.0.23/go.mod h1:uUmtiahU7efOVl/5w5yk9jOze5xYpDZDrSrT8TvHXjQ=
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
@ -607,6 +611,7 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=

View File

@ -26,8 +26,15 @@ cache:
directories:
- $GOPATH/pkg
before_install:
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.16.0
# Update docker to latest version: https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
- sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- sudo apt-get update
- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
# Install golangci-lint
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.17.1
- echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
install:

View File

@ -1,4 +1,4 @@
FROM golang:1.12-alpine3.9 AS downloader
FROM golang:1.12-alpine3.10 AS downloader
ARG VERSION
RUN apk add --no-cache git gcc musl-dev
@ -11,9 +11,9 @@ ENV GO111MODULE=on
ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver"
ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab"
RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate
RUN go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate
FROM alpine:3.9
FROM alpine:3.10
RUN apk add --no-cache ca-certificates

View File

@ -68,3 +68,9 @@
Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations at the same time
the same database at the same time. For example, the MySQL driver uses the `GET_LOCK` function, while the Postgres driver uses
the `pg_advisory_lock` function.
#### Do I need to create a table for tracking migration version used?
No, it is done automatically.
#### Can I use migrate with a non-Go project?
Yes, you can use the migrate CLI in a non-Go project, but there are probably other libraries/frameworks available that offer better test and deploy integrations in that language/framework.

View File

@ -0,0 +1,43 @@
# Getting started
Before you start, you should understand the concept of forward/up and reverse/down database migrations.
Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases)
## Create migrations
Create some migrations using migrate CLI. Here is an example:
```
migrate create -ext sql -dir db/migrations -seq create_users_table
```
Once you create your files, you should fill them.
**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created his migration later gets it merged to the repository first.
Developers and Teams should keep an eye on such cases (especially during code review).
[Here](https://github.com/golang-migrate/migrate/issues/179#issuecomment-475821264) is the issue summary if you would like to read more.
Consider making your migrations idempotent - we can run the same sql code twice in a row with the same result. This makes our migrations more robust. On the other hand, it causes slightly less control over database schema - e.g. let's say you forgot to drop the table in down migration. You run down migration - the table is still there. When you run up migration again - `CREATE TABLE` would return an error, helping you find an issue in down migration, while `CREATE TABLE IF NOT EXISTS` would not. Use those conditions wisely.
In case you would like to run several commands/queries in one migration, you should wrap them in a transaction (if your database supports it).
This way if one of commands fails, our database will remain unchanged.
## Run migrations
Run your migrations through the CLI or your app and check if they applied expected changes.
Just to give you an idea:
```
migrate -database YOUR_DATBASE_URL -path PATH_TO_YOUR_MIGRATIONS up
```
Just add the code to your app and you're ready to go!
Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
(e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
It's also worth checking your migrations in a separate, containerized environment. You can find some tools in the end of this document.
**IMPORTANT:** If you would like to run multiple instances of your app on different machines be sure to use a database that supports locking when running migrations. Otherwise you may encounter issues.
## Further reading:
- [PostgreSQL tutorial](database/postgres/TUTORIAL.md)
- [Best practices](MIGRATIONS.md)
- [FAQ](FAQ.md)
- Tools for testing your migrations in a container:
- https://github.com/dhui/dktest
- https://github.com/ory/dockertest

View File

@ -5,6 +5,7 @@
[![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/)
![Supported Go Versions](https://img.shields.io/badge/Go-1.11%2C%201.12-lightgrey.svg)
[![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases)
[![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate)](https://goreportcard.com/report/github.com/golang-migrate/migrate)
# migrate
@ -139,6 +140,16 @@ func main() {
}
```
## Getting started
Go to [getting started](GETTING_STARTED.md)
## Tutorials
- [PostgreSQL](database/postgres/TUTORIAL.md)
(more tutorials to come)
## Migration files
Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-files-up-and-down-for-a-migration)

View File

@ -7,8 +7,9 @@ package database
import (
"fmt"
"io"
nurl "net/url"
"sync"
iurl "github.com/golang-migrate/migrate/v4/internal/url"
)
var (
@ -81,21 +82,16 @@ type Driver interface {
// Open returns a new driver instance.
func Open(url string) (Driver, error) {
u, err := nurl.Parse(url)
scheme, err := iurl.SchemeFromURL(url)
if err != nil {
return nil, fmt.Errorf("Unable to parse URL. Did you escape all reserved URL characters? "+
"See: https://github.com/golang-migrate/migrate#database-urls Error: %v", err)
}
if u.Scheme == "" {
return nil, fmt.Errorf("database driver: invalid URL scheme")
return nil, err
}
driversMu.RLock()
d, ok := drivers[u.Scheme]
d, ok := drivers[scheme]
driversMu.RUnlock()
if !ok {
return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", u.Scheme)
return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", scheme)
}
return d.Open(url)

View File

@ -7,10 +7,11 @@ require (
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/cockroachdb/apd v1.1.0 // indirect
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c
github.com/containerd/containerd v1.2.7 // indirect
github.com/cznic/ql v1.2.0
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3
github.com/dhui/dktest v0.3.0
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282
github.com/fsouza/fake-gcs-server v1.7.0
github.com/go-sql-driver/mysql v1.4.1
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4
@ -28,7 +29,7 @@ require (
github.com/kshvakov/clickhouse v1.3.5
github.com/lib/pq v1.0.0
github.com/mattn/go-sqlite3 v1.10.0
github.com/mongodb/mongo-go-driver v0.3.0
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8
github.com/pkg/errors v0.8.1 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
@ -40,6 +41,7 @@ require (
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
github.com/xdg/stringprep v1.0.0 // indirect
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect
go.mongodb.org/mongo-driver v1.1.0
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect

View File

@ -26,6 +26,8 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c h1:2zRrJWIt/f9c9HhNHAgrRgq0San5gRRUJTBXLkchal0=
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvvatEupQk=
github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk=
github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas=
@ -56,8 +58,8 @@ github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c
github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU=
github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ=
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 h1:mzrx39dGtGq0VEnTHjnakmczd4uFbhx2cZU3BJDsLdc=
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
@ -147,8 +149,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mongodb/mongo-go-driver v0.3.0 h1:00tKWMrabkVU1e57/TTP4ZBIfhn/wmjlSiRnIM9d0T8=
github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 h1:P48LjvUQpTReR3TQRbxSeSBsMXzfK0uol7eRcr7VBYQ=
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA=
@ -196,6 +198,8 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
go.mongodb.org/mongo-driver v1.1.0 h1:aeOqSrhl9eDRAap/3T5pCfMBEBxZ0vuXBP+RMtp2KX8=
go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=

View File

@ -0,0 +1,25 @@
package url
import (
"errors"
"strings"
)
var errNoScheme = errors.New("no scheme")
var errEmptyURL = errors.New("URL cannot be empty")
// schemeFromURL returns the scheme from a URL string
func SchemeFromURL(url string) (string, error) {
if url == "" {
return "", errEmptyURL
}
i := strings.Index(url, ":")
// No : or : is the first character.
if i < 1 {
return "", errNoScheme
}
return url[0:i], nil
}

View File

@ -7,12 +7,14 @@ package migrate
import (
"errors"
"fmt"
"github.com/hashicorp/go-multierror"
"os"
"sync"
"time"
"github.com/hashicorp/go-multierror"
"github.com/golang-migrate/migrate/v4/database"
iurl "github.com/golang-migrate/migrate/v4/internal/url"
"github.com/golang-migrate/migrate/v4/source"
)
@ -85,13 +87,13 @@ type Migrate struct {
func New(sourceURL, databaseURL string) (*Migrate, error) {
m := newCommon()
sourceName, err := sourceSchemeFromURL(sourceURL)
sourceName, err := iurl.SchemeFromURL(sourceURL)
if err != nil {
return nil, err
}
m.sourceName = sourceName
databaseName, err := databaseSchemeFromURL(databaseURL)
databaseName, err := iurl.SchemeFromURL(databaseURL)
if err != nil {
return nil, err
}
@ -119,7 +121,7 @@ func New(sourceURL, databaseURL string) (*Migrate, error) {
func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInstance database.Driver) (*Migrate, error) {
m := newCommon()
sourceName, err := schemeFromURL(sourceURL)
sourceName, err := iurl.SchemeFromURL(sourceURL)
if err != nil {
return nil, err
}
@ -145,7 +147,7 @@ func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInst
func NewWithSourceInstance(sourceName string, sourceInstance source.Driver, databaseURL string) (*Migrate, error) {
m := newCommon()
databaseName, err := schemeFromURL(databaseURL)
databaseName, err := iurl.SchemeFromURL(databaseURL)
if err != nil {
return nil, err
}
@ -804,6 +806,7 @@ func (m *Migrate) versionExists(version uint) (result error) {
return err
}
m.logErr(fmt.Errorf("no migration found for version %d", version))
return os.ErrNotExist
}

View File

@ -1,7 +1,6 @@
package migrate
import (
"errors"
"fmt"
nurl "net/url"
"strings"
@ -49,42 +48,6 @@ func suint(n int) uint {
return uint(n)
}
var errNoScheme = errors.New("no scheme")
var errEmptyURL = errors.New("URL cannot be empty")
func sourceSchemeFromURL(url string) (string, error) {
u, err := schemeFromURL(url)
if err != nil {
return "", fmt.Errorf("source: %v", err)
}
return u, nil
}
func databaseSchemeFromURL(url string) (string, error) {
u, err := schemeFromURL(url)
if err != nil {
return "", fmt.Errorf("database: %v", err)
}
return u, nil
}
// schemeFromURL returns the scheme from a URL string
func schemeFromURL(url string) (string, error) {
if url == "" {
return "", errEmptyURL
}
u, err := nurl.Parse(url)
if err != nil {
return "", err
}
if len(u.Scheme) == 0 {
return "", errNoScheme
}
return u.Scheme, nil
}
// FilterCustomQuery filters all query values starting with `x-`
func FilterCustomQuery(u *nurl.URL) *nurl.URL {
ux := *u

View File

@ -1,6 +1,6 @@
run:
# timeout for analysis, e.g. 30s, 5m, default is 1m
deadline: 2m
timeout: 2m
linters:
enable:
#- golint

View File

@ -6,8 +6,8 @@ matrix:
- go: master
include:
# Supported versions of Go: https://golang.org/dl/
- go: "1.11.x"
- go: "1.12.x"
- go: "1.13.x"
- go: master
go_import_path: github.com/golang-migrate/migrate
@ -26,8 +26,15 @@ cache:
directories:
- $GOPATH/pkg
before_install:
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.16.0
# Update docker to latest version: https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
- sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- sudo apt-get update
- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
# Install golangci-lint
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.20.0
- echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
install:

View File

@ -1,4 +1,4 @@
FROM golang:1.12-alpine3.9 AS downloader
FROM golang:1.12-alpine3.10 AS downloader
ARG VERSION
RUN apk add --no-cache git gcc musl-dev
@ -9,11 +9,11 @@ COPY . ./
ENV GO111MODULE=on
ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver"
ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab"
ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab"
RUN go build -a -o build/migrate.linux-386 -ldflags="-X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate
RUN go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate
FROM alpine:3.9
FROM alpine:3.10
RUN apk add --no-cache ca-certificates

View File

@ -68,3 +68,9 @@
Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations at the same time
the same database at the same time. For example, the MySQL driver uses the `GET_LOCK` function, while the Postgres driver uses
the `pg_advisory_lock` function.
#### Do I need to create a table for tracking migration version used?
No, it is done automatically.
#### Can I use migrate with a non-Go project?
Yes, you can use the migrate CLI in a non-Go project, but there are probably other libraries/frameworks available that offer better test and deploy integrations in that language/framework.

View File

@ -0,0 +1,43 @@
# Getting started
Before you start, you should understand the concept of forward/up and reverse/down database migrations.
Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases)
## Create migrations
Create some migrations using migrate CLI. Here is an example:
```
migrate create -ext sql -dir db/migrations -seq create_users_table
```
Once you create your files, you should fill them.
**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created his migration later gets it merged to the repository first.
Developers and Teams should keep an eye on such cases (especially during code review).
[Here](https://github.com/golang-migrate/migrate/issues/179#issuecomment-475821264) is the issue summary if you would like to read more.
Consider making your migrations idempotent - we can run the same sql code twice in a row with the same result. This makes our migrations more robust. On the other hand, it causes slightly less control over database schema - e.g. let's say you forgot to drop the table in down migration. You run down migration - the table is still there. When you run up migration again - `CREATE TABLE` would return an error, helping you find an issue in down migration, while `CREATE TABLE IF NOT EXISTS` would not. Use those conditions wisely.
In case you would like to run several commands/queries in one migration, you should wrap them in a transaction (if your database supports it).
This way if one of commands fails, our database will remain unchanged.
## Run migrations
Run your migrations through the CLI or your app and check if they applied expected changes.
Just to give you an idea:
```
migrate -database YOUR_DATBASE_URL -path PATH_TO_YOUR_MIGRATIONS up
```
Just add the code to your app and you're ready to go!
Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
(e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
It's also worth checking your migrations in a separate, containerized environment. You can find some tools in the end of this document.
**IMPORTANT:** If you would like to run multiple instances of your app on different machines be sure to use a database that supports locking when running migrations. Otherwise you may encounter issues.
## Further reading:
- [PostgreSQL tutorial](database/postgres/TUTORIAL.md)
- [Best practices](MIGRATIONS.md)
- [FAQ](FAQ.md)
- Tools for testing your migrations in a container:
- https://github.com/dhui/dktest
- https://github.com/ory/dockertest

View File

@ -44,9 +44,14 @@ It is suggested that the version number of corresponding `up` and `down` migrati
files be equivalent for clarity, but they are allowed to differ so long as the
relative ordering of the migrations is preserved.
The migration files are permitted to be empty, so in the event that a migration
is a no-op or is irreversible, it is recommended to still include both migration
files, and either leaving them empty or adding a comment as appropriate.
The migration files are permitted to be "empty", in the event that a migration
is a no-op or is irreversible. It is recommended to still include both migration
files by making the whole migration file consist of a comment.
If your database does not support comments, then deleting the migration file will also work.
Note, an actual empty file (e.g. a 0 byte file) may cause issues with your database since migrate
will attempt to run an empty query. In this case, deleting the migration file will also work.
For the rational of this behavior see:
[#244 (comment)](https://github.com/golang-migrate/migrate/issues/244#issuecomment-510758270)
## Migration Content Format

View File

@ -1,4 +1,4 @@
SOURCE ?= file go_bindata github aws_s3 google_cloud_storage godoc_vfs gitlab
SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab
DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver
VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
TEST_FLAGS ?=
@ -27,27 +27,14 @@ test-short:
test:
@-rm -r $(COVERAGE_DIR)
@mkdir $(COVERAGE_DIR)
make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/_$$(RAND).txt -bench=. -benchmem -timeout 20m'
@echo 'mode: atomic' > $(COVERAGE_DIR)/combined.txt
@cat $(COVERAGE_DIR)/_*.txt | grep -v 'mode: atomic' >> $(COVERAGE_DIR)/combined.txt
make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile $$(COVERAGE_DIR)/combined.txt -bench=. -benchmem -timeout 20m'
test-with-flags:
@echo SOURCE: $(SOURCE)
@echo SOURCE: $(SOURCE)
@echo DATABASE: $(DATABASE)
@go test $(TEST_FLAGS) .
@go test $(TEST_FLAGS) ./cli/...
@go test $(TEST_FLAGS) ./database
@go test $(TEST_FLAGS) ./testing/...
@echo -n '$(SOURCE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./source/{}
@go test $(TEST_FLAGS) ./source/testing/...
@go test $(TEST_FLAGS) ./source/stub/...
@echo -n '$(DATABASE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./database/{}
@go test $(TEST_FLAGS) ./database/testing/...
@go test $(TEST_FLAGS) ./database/stub/...
@go test $(TEST_FLAGS) ./...
kill-orphaned-docker-containers:
@ -84,7 +71,7 @@ rewrite-import-paths:
docs:
-make kill-docs
nohup godoc -play -http=127.0.0.1:6064 </dev/null >/dev/null 2>&1 & echo $$! > .godoc.pid
cat .godoc.pid
cat .godoc.pid
kill-docs:

View File

@ -3,19 +3,19 @@
[![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master)
[![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs)
[![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/)
![Supported Go Versions](https://img.shields.io/badge/Go-1.11%2C%201.12-lightgrey.svg)
![Supported Go Versions](https://img.shields.io/badge/Go-1.12%2C%201.13-lightgrey.svg)
[![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases)
[![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate)](https://goreportcard.com/report/github.com/golang-migrate/migrate)
# migrate
__Database migrations written in Go. Use as [CLI](#cli-usage) or import as [library](#use-in-your-go-project).__
* Migrate reads migrations from [sources](#migration-sources)
* Migrate reads migrations from [sources](#migration-sources)
and applies them in correct order to a [database](#databases).
* Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof.
* Drivers are "dumb", migrate glues everything together and makes sure the logic is bulletproof.
(Keeps the drivers lightweight, too.)
* Database drivers don't assume things or try to correct user input. When in doubt, fail.
* Database drivers don't assume things or try to correct user input. When in doubt, fail.
Forked from [mattes/migrate](https://github.com/mattes/migrate)
@ -23,21 +23,21 @@ Forked from [mattes/migrate](https://github.com/mattes/migrate)
Database drivers run migrations. [Add a new database?](database/driver.go)
* [PostgreSQL](database/postgres)
* [Redshift](database/redshift)
* [Ql](database/ql)
* [Cassandra](database/cassandra)
* [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
* [MySQL/ MariaDB](database/mysql)
* [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167))
* [MongoDB](database/mongodb)
* [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
* [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171))
* [Google Cloud Spanner](database/spanner)
* [CockroachDB](database/cockroachdb)
* [ClickHouse](database/clickhouse)
* [Firebird](database/firebird) ([todo #49](https://github.com/golang-migrate/migrate/issues/49))
* [MS SQL Server](database/sqlserver)
* [PostgreSQL](database/postgres)
* [Redshift](database/redshift)
* [Ql](database/ql)
* [Cassandra](database/cassandra)
* [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
* [MySQL/ MariaDB](database/mysql)
* [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167))
* [MongoDB](database/mongodb)
* [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
* [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171))
* [Google Cloud Spanner](database/spanner)
* [CockroachDB](database/cockroachdb)
* [ClickHouse](database/clickhouse)
* [Firebird](database/firebird) ([todo #49](https://github.com/golang-migrate/migrate/issues/49))
* [MS SQL Server](database/sqlserver)
### Database URLs
@ -49,6 +49,7 @@ Explicitly, the following characters need to be escaped:
`!`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, `*`, `+`, `,`, `/`, `:`, `;`, `=`, `?`, `@`, `[`, `]`
It's easiest to always run the URL parts of your DB connection URL (e.g. username, password, etc) through an URL encoder. See the example Python snippets below:
```bash
$ python3 -c 'import urllib.parse; print(urllib.parse.quote(input("String to encode: "), ""))'
String to encode: FAKEpassword!#$%&'()*+,/:;=?@[]
@ -63,44 +64,43 @@ $
Source drivers read migrations from local or remote sources. [Add a new source?](source/driver.go)
* [Filesystem](source/file) - read from fileystem
* [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
* [Github](source/github) - read from remote Github repositories
* [Gitlab](source/gitlab) - read from remote Gitlab repositories
* [AWS S3](source/aws_s3) - read from Amazon Web Services S3
* [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage
* [Filesystem](source/file) - read from filesystem
* [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
* [Github](source/github) - read from remote Github repositories
* [Github Enterprise](source/github_ee) - read from remote Github Enterprise repositories
* [Gitlab](source/gitlab) - read from remote Gitlab repositories
* [AWS S3](source/aws_s3) - read from Amazon Web Services S3
* [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage
## CLI usage
* Simple wrapper around this library.
* Handles ctrl+c (SIGINT) gracefully.
* No config search paths, no config files, no magic ENV var injections.
* Simple wrapper around this library.
* Handles ctrl+c (SIGINT) gracefully.
* No config search paths, no config files, no magic ENV var injections.
__[CLI Documentation](cli)__
__[CLI Documentation](cmd/migrate)__
### Basic usage:
### Basic usage
```
```bash
$ migrate -source file://path/to/migrations -database postgres://localhost:5432/database up 2
```
### Docker usage
```
```bash
$ docker run -v {{ migration dir }}:/migrations --network host migrate/migrate
-path=/migrations/ -database postgres://localhost:5432/database up 2
```
## Use in your Go project
* API is stable and frozen for this release (v3 & v4).
* Uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies.
* To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`.
* Bring your own logger.
* Uses `io.Reader` streams internally for low memory overhead.
* Thread-safe and no goroutine leaks.
* API is stable and frozen for this release (v3 & v4).
* Uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies.
* To help prevent database corruptions, it supports graceful stops via `GracefulStop chan bool`.
* Bring your own logger.
* Uses `io.Reader` streams internally for low memory overhead.
* Thread-safe and no goroutine leaks.
__[Go Documentation](https://godoc.org/github.com/golang-migrate/migrate)__
@ -140,11 +140,21 @@ func main() {
}
```
## Getting started
Go to [getting started](GETTING_STARTED.md)
## Tutorials
- [PostgreSQL](database/postgres/TUTORIAL.md)
(more tutorials to come)
## Migration files
Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-files-up-and-down-for-a-migration)
```
```bash
1481574547_create_users_table.up.sql
1481574547_create_users_table.down.sql
```
@ -166,8 +176,6 @@ read the [development guide](CONTRIBUTING.md).
Also have a look at the [FAQ](FAQ.md).
---
Looking for alternatives? [https://awesome-go.com/#database](https://awesome-go.com/#database).

View File

@ -0,0 +1,148 @@
# PostgreSQL tutorial for beginners
## Create/configure database
For the purpose of this tutorial let's create PostgreSQL database called `example`.
Our user here is `postgres`, password `password`, and host is `localhost`.
```
psql -h localhost -U postgres -w -c "create database example;"
```
When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convienience:
```
export POSTGRESQL_URL=postgres://postgres:password@localhost:5432/example?sslmode=disable
```
`sslmode=disable` means that the connection with our database will not be encrypted. Enabling it is left as an exercise.
You can find further description of database URLs [here](README.md#database-urls).
## Create migrations
Let's create table called `users`:
```
migrate create -ext sql -dir db/migrations -seq create_users_table
```
If there were no errors, we should have two files available under `db/migrations` folder:
- 000001_create_users_table.down.sql
- 000001_create_users_table.up.sql
Note the `sql` extension that we provided.
In the `.up.sql` file let's create the table:
```
CREATE TABLE IF NOT EXISTS users(
user_id serial PRIMARY KEY,
username VARCHAR (50) UNIQUE NOT NULL,
password VARCHAR (50) NOT NULL,
email VARCHAR (300) UNIQUE NOT NULL
);
```
And in the `.down.sql` let's delete it:
```
DROP TABLE IF EXISTS users;
```
By adding `IF EXISTS/IF NOT EXISTS` we are making migrations idempotent - you can read more about idempotency in [getting started](GETTING_STARTED.md#create-migrations)
## Run migrations
```
migrate -database ${POSTGRESQL_URL} -path db/migrations up
```
Let's check if the table was created properly by running `psql example -c "\d users"`.
The output you are supposed to see:
```
Table "public.users"
Column | Type | Modifiers
----------+------------------------+---------------------------------------------------------
user_id | integer | not null default nextval('users_user_id_seq'::regclass)
username | character varying(50) | not null
password | character varying(50) | not null
email | character varying(300) | not null
Indexes:
"users_pkey" PRIMARY KEY, btree (user_id)
"users_email_key" UNIQUE CONSTRAINT, btree (email)
"users_username_key" UNIQUE CONSTRAINT, btree (username)
```
Great! Now let's check if running reverse migration also works:
```
migrate -database ${POSTGRESQL_URL} -path db/migrations down
```
Make sure to check if your database changed as expected in this case as well.
## Database transactions
To show database transactions usage, let's create another set of migrations by running:
```
migrate create -ext sql -dir db/migrations -seq add_mood_to_users
```
Again, it should create for us two migrations files:
- 000002_add_mood_to_users.down.sql
- 000002_add_mood_to_users.up.sql
In Postgres, when we want our queries to be done in a transaction, we need to wrap it with `BEGIN` and `COMMIT` commands.
In our example, we are going to add a column to our database that can only accept enumerable values or NULL.
Migration up:
```
BEGIN;
CREATE TYPE enum_mood AS ENUM (
'happy',
'sad',
'neutral'
);
ALTER TABLE users ADD COLUMN mood enum_mood;
COMMIT;
```
Migration down:
```
BEGIN;
ALTER TABLE users DROP COLUMN mood;
DROP TYPE enum_mood;
COMMIT;
```
Now we can run our new migration and check the database:
```
migrate -database ${POSTGRESQL_URL} -path db/migrations up
psql example -c "\d users"
```
Expected output:
```
Table "public.users"
Column | Type | Modifiers
----------+------------------------+---------------------------------------------------------
user_id | integer | not null default nextval('users_user_id_seq'::regclass)
username | character varying(50) | not null
password | character varying(50) | not null
email | character varying(300) | not null
mood | enum_mood |
Indexes:
"users_pkey" PRIMARY KEY, btree (user_id)
"users_email_key" UNIQUE CONSTRAINT, btree (email)
"users_username_key" UNIQUE CONSTRAINT, btree (username)
```
## Optional: Run migrations within your Go app
Here is a very simple app running migrations for the above configuration:
```
import (
"log"
"github.com/golang-migrate/migrate/v4"
_ "github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
)
func main() {
m, err := migrate.New(
"file://db/migrations",
"postgres://postgres:postgres@localhost:5432/example?sslmode=disable")
if err != nil {
log.Fatal(err)
}
if err := m.Up(); err != nil {
log.Fatal(err)
}
}
```
You can find details [here](README.md#use-in-your-go-project)

View File

@ -150,8 +150,7 @@ func (p *Postgres) Lock() error {
return err
}
// This will either obtain the lock immediately and return true,
// or return false if the lock cannot be acquired immediately.
// This will wait indefinitely until the lock can be acquired.
query := `SELECT pg_advisory_lock($1)`
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}

View File

@ -7,10 +7,11 @@ require (
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/cockroachdb/apd v1.1.0 // indirect
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c
github.com/containerd/containerd v1.2.7 // indirect
github.com/cznic/ql v1.2.0
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3
github.com/dhui/dktest v0.3.0
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282
github.com/fsouza/fake-gcs-server v1.7.0
github.com/go-sql-driver/mysql v1.4.1
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4
@ -28,7 +29,7 @@ require (
github.com/kshvakov/clickhouse v1.3.5
github.com/lib/pq v1.0.0
github.com/mattn/go-sqlite3 v1.10.0
github.com/mongodb/mongo-go-driver v0.3.0
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8
github.com/pkg/errors v0.8.1 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
@ -40,6 +41,7 @@ require (
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
github.com/xdg/stringprep v1.0.0 // indirect
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect
go.mongodb.org/mongo-driver v1.1.0
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
@ -52,3 +54,5 @@ require (
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb
google.golang.org/grpc v1.20.1 // indirect
)
go 1.12

View File

@ -26,6 +26,8 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c h1:2zRrJWIt/f9c9HhNHAgrRgq0San5gRRUJTBXLkchal0=
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvvatEupQk=
github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 h1:UHFGPvSxX4C4YBApSPvmUfL8tTvWLj2ryqvT9K4Jcuk=
github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f h1:7uSNgsgcarNk4oiN/nNkO0J7KAjlsF5Yv5Gf/tFdHas=
@ -56,8 +58,8 @@ github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c
github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtfRN0TFIwt6YFL7N9RU=
github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ=
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282 h1:mzrx39dGtGq0VEnTHjnakmczd4uFbhx2cZU3BJDsLdc=
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
@ -147,8 +149,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mongodb/mongo-go-driver v0.3.0 h1:00tKWMrabkVU1e57/TTP4ZBIfhn/wmjlSiRnIM9d0T8=
github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 h1:P48LjvUQpTReR3TQRbxSeSBsMXzfK0uol7eRcr7VBYQ=
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA=
@ -196,6 +198,8 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
go.mongodb.org/mongo-driver v1.1.0 h1:aeOqSrhl9eDRAap/3T5pCfMBEBxZ0vuXBP+RMtp2KX8=
go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=

View File

@ -0,0 +1,25 @@
package url
import (
"errors"
"strings"
)
var errNoScheme = errors.New("no scheme")
var errEmptyURL = errors.New("URL cannot be empty")
// schemeFromURL returns the scheme from a URL string
func SchemeFromURL(url string) (string, error) {
if url == "" {
return "", errEmptyURL
}
i := strings.Index(url, ":")
// No : or : is the first character.
if i < 1 {
return "", errNoScheme
}
return url[0:i], nil
}

View File

@ -7,12 +7,14 @@ package migrate
import (
"errors"
"fmt"
"github.com/hashicorp/go-multierror"
"os"
"sync"
"time"
"github.com/hashicorp/go-multierror"
"github.com/golang-migrate/migrate/v4/database"
iurl "github.com/status-im/migrate/v4/internal/url"
"github.com/golang-migrate/migrate/v4/source"
)
@ -85,13 +87,13 @@ type Migrate struct {
func New(sourceURL, databaseURL string) (*Migrate, error) {
m := newCommon()
sourceName, err := sourceSchemeFromURL(sourceURL)
sourceName, err := iurl.SchemeFromURL(sourceURL)
if err != nil {
return nil, err
}
m.sourceName = sourceName
databaseName, err := databaseSchemeFromURL(databaseURL)
databaseName, err := iurl.SchemeFromURL(databaseURL)
if err != nil {
return nil, err
}
@ -119,7 +121,7 @@ func New(sourceURL, databaseURL string) (*Migrate, error) {
func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInstance database.Driver) (*Migrate, error) {
m := newCommon()
sourceName, err := schemeFromURL(sourceURL)
sourceName, err := iurl.SchemeFromURL(sourceURL)
if err != nil {
return nil, err
}
@ -145,7 +147,7 @@ func NewWithDatabaseInstance(sourceURL string, databaseName string, databaseInst
func NewWithSourceInstance(sourceName string, sourceInstance source.Driver, databaseURL string) (*Migrate, error) {
m := newCommon()
databaseName, err := schemeFromURL(databaseURL)
databaseName, err := iurl.SchemeFromURL(databaseURL)
if err != nil {
return nil, err
}
@ -804,6 +806,7 @@ func (m *Migrate) versionExists(version uint) (result error) {
return err
}
m.logErr(fmt.Errorf("no migration found for version %d", version))
return os.ErrNotExist
}
@ -950,7 +953,7 @@ func (m *Migrate) unlock() error {
// if a prevErr is not nil.
func (m *Migrate) unlockErr(prevErr error) error {
if err := m.unlock(); err != nil {
return NewMultiError(prevErr, err)
return multierror.Append(prevErr, err)
}
return prevErr
}

View File

@ -1,18 +1,22 @@
package migrate
import (
"errors"
"fmt"
nurl "net/url"
"strings"
)
// MultiError holds multiple errors.
//
// Deprecated: Use github.com/hashicorp/go-multierror instead
type MultiError struct {
Errs []error
}
// NewMultiError returns an error type holding multiple errors.
//
// Deprecated: Use github.com/hashicorp/go-multierror instead
//
func NewMultiError(errs ...error) MultiError {
compactErrs := make([]error, 0)
for _, e := range errs {
@ -44,42 +48,6 @@ func suint(n int) uint {
return uint(n)
}
var errNoScheme = errors.New("no scheme")
var errEmptyURL = errors.New("URL cannot be empty")
func sourceSchemeFromURL(url string) (string, error) {
u, err := schemeFromURL(url)
if err != nil {
return "", fmt.Errorf("source: %v", err)
}
return u, nil
}
func databaseSchemeFromURL(url string) (string, error) {
u, err := schemeFromURL(url)
if err != nil {
return "", fmt.Errorf("database: %v", err)
}
return u, nil
}
// schemeFromURL returns the scheme from a URL string
func schemeFromURL(url string) (string, error) {
if url == "" {
return "", errEmptyURL
}
u, err := nurl.Parse(url)
if err != nil {
return "", err
}
if len(u.Scheme) == 0 {
return "", errNoScheme
}
return u.Scheme, nil
}
// FilterCustomQuery filters all query values starting with `x-`
func FilterCustomQuery(u *nurl.URL) *nurl.URL {
ux := *u

View File

@ -2,10 +2,13 @@ package statusproto
import (
"crypto/ecdsa"
"crypto/sha1"
"encoding/hex"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/google/uuid"
statusproto "github.com/status-im/status-protocol-go/types"
protocol "github.com/status-im/status-protocol-go/v1"
)
type ChatType int
@ -52,6 +55,55 @@ type Chat struct {
MembershipUpdates []ChatMembershipUpdate `json:"membershipUpdates"`
}
func (c *Chat) MembersAsPublicKeys() ([]*ecdsa.PublicKey, error) {
publicKeys := make([]string, len(c.Members))
for idx, item := range c.Members {
publicKeys[idx] = item.ID
}
return stringSliceToPublicKeys(publicKeys, true)
}
func (c *Chat) updateChatFromProtocolGroup(g *protocol.Group) {
// ID
c.ID = g.ChatID()
// Name
c.Name = g.Name()
// Members
members := g.Members()
admins := g.Admins()
joined := g.Joined()
chatMembers := make([]ChatMember, 0, len(members))
for _, m := range members {
chatMember := ChatMember{
ID: m,
}
chatMember.Admin = stringSliceContains(admins, m)
chatMember.Joined = stringSliceContains(joined, m)
chatMembers = append(chatMembers, chatMember)
}
c.Members = chatMembers
// MembershipUpdates
updates := g.Updates()
membershipUpdates := make([]ChatMembershipUpdate, 0, len(updates))
for _, update := range updates {
membershipUpdate := ChatMembershipUpdate{
Type: update.Type,
Name: update.Name,
ClockValue: uint64(update.ClockValue), // TODO: get rid of type casting
Signature: update.Signature,
From: update.From,
Member: update.Member,
Members: update.Members,
}
membershipUpdate.setID()
membershipUpdates = append(membershipUpdates, membershipUpdate)
}
c.MembershipUpdates = membershipUpdates
}
// ChatMembershipUpdate represent an event on membership of the chat
type ChatMembershipUpdate struct {
// Unique identifier for the event
@ -72,6 +124,11 @@ type ChatMembershipUpdate struct {
Members []string `json:"members,omitempty"`
}
func (u *ChatMembershipUpdate) setID() {
sum := sha1.Sum([]byte(u.Signature))
u.ID = hex.EncodeToString(sum[:])
}
// ChatMember represents a member who participates in a group chat
type ChatMember struct {
// ID is the hex encoded public key of the member
@ -113,14 +170,8 @@ func CreatePublicChat(name string) Chat {
}
}
func groupChatID(creator *ecdsa.PublicKey) string {
return uuid.New().String() + statusproto.EncodeHex(crypto.FromECDSAPub(creator))
}
func CreateGroupChat(name string, creator *ecdsa.PublicKey) Chat {
func createGroupChat() Chat {
return Chat{
ID: groupChatID(creator),
Name: name,
Active: true,
ChatType: ChatTypePrivateGroupChat,
}
@ -134,3 +185,35 @@ func findChatByID(chatID string, chats []*Chat) *Chat {
}
return nil
}
func stringSliceToPublicKeys(slice []string, prefixed bool) ([]*ecdsa.PublicKey, error) {
result := make([]*ecdsa.PublicKey, len(slice))
for idx, item := range slice {
var (
b []byte
err error
)
if prefixed {
b, err = hexutil.Decode(item)
} else {
b, err = hex.DecodeString(item)
}
if err != nil {
return nil, err
}
result[idx], err = crypto.UnmarshalPubkey(b)
if err != nil {
return nil, err
}
}
return result, nil
}
func stringSliceContains(slice []string, item string) bool {
for _, s := range slice {
if s == item {
return true
}
}
return false
}

View File

@ -0,0 +1,28 @@
package statusproto
import (
protocol "github.com/status-im/status-protocol-go/v1"
)
func newProtocolGroupFromChat(chat *Chat) (*protocol.Group, error) {
return protocol.NewGroup(chat.ID, chatToFlattenMembershipUpdate(chat))
}
func chatToFlattenMembershipUpdate(chat *Chat) []protocol.MembershipUpdateFlat {
result := make([]protocol.MembershipUpdateFlat, len(chat.MembershipUpdates))
for idx, update := range chat.MembershipUpdates {
result[idx] = protocol.MembershipUpdateFlat{
ChatID: chat.ID,
From: update.From,
Signature: update.Signature,
MembershipUpdateEvent: protocol.MembershipUpdateEvent{
Name: update.Name,
Type: update.Type,
ClockValue: int64(update.ClockValue), // TODO: remove type difference
Member: update.Member,
Members: update.Members,
},
}
}
return result
}

View File

@ -78,6 +78,7 @@ func VerifySignatures(signaturePairs [][3]string) error {
}
// ExtractSignatures extract from tuples of signatures content a public key
// DEPRECATED: use ExtractSignature
func ExtractSignatures(signaturePairs [][2]string) ([]string, error) {
response := make([]string, len(signaturePairs))
for i, signaturePair := range signaturePairs {
@ -102,6 +103,12 @@ func ExtractSignatures(signaturePairs [][2]string) ([]string, error) {
return response, nil
}
// ExtractSignature returns a public key for a given data and signature.
func ExtractSignature(data, signature []byte) (*ecdsa.PublicKey, error) {
dataHash := crypto.Keccak256(data)
return crypto.SigToPub(dataHash, signature)
}
func EncryptSymmetric(key, plaintext []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {

View File

@ -16,10 +16,10 @@ require (
github.com/russolsen/same v0.0.0-20160222130632-f089df61f51d // indirect
github.com/russolsen/transit v0.0.0-20180705123435-0794b4c4505a
github.com/status-im/doubleratchet v2.0.0+incompatible
github.com/status-im/migrate/v4 v4.0.0-20190821140204-a9d340ec8fb76af4afda06acf01740d45d2661ed
github.com/status-im/migrate/v4 v4.6.2-status.2
github.com/status-im/whisper v1.5.1
github.com/stretchr/testify v1.3.1-0.20190712000136-221dbe5ed467
github.com/vacp2p/mvds v0.0.21
github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d
go.uber.org/zap v1.10.0
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect

View File

@ -44,6 +44,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg=
github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc=
@ -68,8 +69,7 @@ github.com/docker/distribution v2.7.0+incompatible h1:neUDAlf3wX6Ml4HdqTrbcOHXtf
github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v0.0.0-20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf h1:2v/98rHzs3v6X0AHtoCH9u+e56SdnpogB1Z2fFe1KqQ=
github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
@ -100,10 +100,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang-migrate/migrate v3.5.4+incompatible h1:R7OzwvCJTCgwapPCiX6DyBiu2czIUMDCB118gFTKTUA=
github.com/golang-migrate/migrate v3.5.4+incompatible/go.mod h1:IsVUlFN5puWOmXrqjgGUfIRIbU7mr8oNBE2tyERd9Wk=
github.com/golang-migrate/migrate/v4 v4.5.0 h1:ucd2qJu1BAKTtmjh7QlWYiq01DwlE/xXYQkOPE0ZTsI=
github.com/golang-migrate/migrate/v4 v4.5.0/go.mod h1:SzAcz2l+yDJVhQC7fwiF7T2MAFPMIkigJz98klRJ4OE=
github.com/golang-migrate/migrate/v4 v4.6.2 h1:LDDOHo/q1W5UDj6PbkxdCv7lv9yunyZHXvxuwDkGo3k=
github.com/golang-migrate/migrate/v4 v4.6.2/go.mod h1:JYi6reN3+Z734VZ0akNuyOJNcrg45ZL7LDBMW3WGJL0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -189,7 +187,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mutecomm/go-sqlcipher v0.0.0-20190227152316-55dbde17881f h1:hd3r+uv9DNLScbOrnlj82rBldHQf3XWmCeXAWbw8euQ=
github.com/mutecomm/go-sqlcipher v0.0.0-20190227152316-55dbde17881f/go.mod h1:MyUWrZlB1aI5bs7j9/pJ8ckLLZ4QcCYcNiSbsAW32D4=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@ -255,8 +253,8 @@ github.com/status-im/doubleratchet v2.0.0+incompatible h1:s77lF1lDubK0RKftxN2vH8
github.com/status-im/doubleratchet v2.0.0+incompatible/go.mod h1:1sqR0+yhiM/bd+wrdX79AOt2csZuJOni0nUDzKNuqOU=
github.com/status-im/go-ethereum v1.9.5-status.4 h1:F5VrxH9LmTxWl4qwQjs0TI5TgG9dVuZKqGmdwHJ0cWk=
github.com/status-im/go-ethereum v1.9.5-status.4/go.mod h1:Ulij8LMpMvXnbnPcmDqrpI+iXoXSjxItuY/wmbasTZU=
github.com/status-im/migrate/v4 v4.0.0-20190821140204-a9d340ec8fb76af4afda06acf01740d45d2661ed h1:K2iga8l8OQIHnk2bBq2QsZTO2Q38YWy04xIspdITCdM=
github.com/status-im/migrate/v4 v4.0.0-20190821140204-a9d340ec8fb76af4afda06acf01740d45d2661ed/go.mod h1:r8HggRBZ/k7TRwByq/Hp3P/ubFppIna0nvyavVK0pjA=
github.com/status-im/migrate/v4 v4.6.2-status.2 h1:SdC+sMDl/aI7vUlwD2qj2p7KsK4T60IS9z4/rYCCbI8=
github.com/status-im/migrate/v4 v4.6.2-status.2/go.mod h1:c/kc90n47GZu/58nnz1OMLTf7uE4Da4gZP5qmU+A/v8=
github.com/status-im/whisper v1.5.1 h1:87/XIg0Wjua7lXBGiEXgAfTOqlt2Q1dMDuxugTyZbbA=
github.com/status-im/whisper v1.5.1/go.mod h1:emrOxzJme0k66QtbbQ2bdd3P8RCdLZ8sTD7SkwH1s2s=
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
@ -276,12 +274,13 @@ github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2/go.mod h1:Z4AUp2K
github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/uber/jaeger-client-go v0.0.0-20180607151842-f7e0d4744fa6/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v0.0.0-20180615202729-a51202d6f4a7/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/vacp2p/mvds v0.0.21 h1:YeYja8noKsHvrnOHM4pqjBvDwwy+kUzXMkX1IBYJAbU=
github.com/vacp2p/mvds v0.0.21/go.mod h1:pIqr2Hg4cIkTJniGPCp4ptong2jxgxx6uToVoY94+II=
github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d h1:kc/9/ZbwyqIKQ2phzmPpBYGx7v2olwD2P7Uj8KKd+Bw=
github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d/go.mod h1:uUmtiahU7efOVl/5w5yk9jOze5xYpDZDrSrT8TvHXjQ=
github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=

View File

@ -0,0 +1,68 @@
package statusproto
import (
"github.com/pkg/errors"
protocol "github.com/status-im/status-protocol-go/v1"
)
type persistentMessageHandler struct {
persistence *sqlitePersistence
}
func newPersistentMessageHandler(persistence *sqlitePersistence) *persistentMessageHandler {
return &persistentMessageHandler{persistence: persistence}
}
// HandleMembershipUpdate updates a Chat instance according to the membership updates.
// It retrieves chat, if exists, and merges membership updates from the message.
// Finally, the Chat is updated with the new group events.
func (h *persistentMessageHandler) HandleMembershipUpdate(m protocol.MembershipUpdateMessage) error {
chat, err := h.chatID(m.ChatID)
switch err {
case errChatNotFound:
group, err := protocol.NewGroupWithMembershipUpdates(m.ChatID, m.Updates)
if err != nil {
return err
}
newChat := createGroupChat()
newChat.updateChatFromProtocolGroup(group)
chat = &newChat
case nil:
existingGroup, err := newProtocolGroupFromChat(chat)
if err != nil {
return errors.Wrap(err, "failed to create a Group from Chat")
}
updateGroup, err := protocol.NewGroupWithMembershipUpdates(m.ChatID, m.Updates)
if err != nil {
return errors.Wrap(err, "invalid membership update")
}
merged := protocol.MergeFlatMembershipUpdates(existingGroup.Updates(), updateGroup.Updates())
newGroup, err := protocol.NewGroup(chat.ID, merged)
if err != nil {
return errors.Wrap(err, "failed to create a group with new membership updates")
}
chat.updateChatFromProtocolGroup(newGroup)
default:
return err
}
return h.persistence.SaveChat(*chat)
}
func (h *persistentMessageHandler) chatID(chatID string) (*Chat, error) {
var chat *Chat
chats, err := h.persistence.Chats()
if err != nil {
return nil, err
}
for _, ch := range chats {
if chat.ID == chatID {
chat = ch
break
}
}
if chat == nil {
return nil, errChatNotFound
}
return chat, nil
}

View File

@ -28,11 +28,16 @@ const (
whisperPoWTime = 5
)
type messageHandler interface {
HandleMembershipUpdate(m protocol.MembershipUpdateMessage) error
}
type messageProcessor struct {
identity *ecdsa.PrivateKey
datasync *datasync.DataSync
protocol *encryption.Protocol
transport *transport.WhisperServiceTransport
handler messageHandler
logger *zap.Logger
featureFlags featureFlags
@ -43,6 +48,7 @@ func newMessageProcessor(
database *sql.DB,
enc *encryption.Protocol,
transport *transport.WhisperServiceTransport,
handler messageHandler,
logger *zap.Logger,
features featureFlags,
) (*messageProcessor, error) {
@ -65,6 +71,7 @@ func newMessageProcessor(
datasync: ds,
protocol: enc,
transport: transport,
handler: handler,
logger: logger,
featureFlags: features,
}
@ -87,14 +94,14 @@ func (p *messageProcessor) Stop() {
func (p *messageProcessor) SendPrivate(
ctx context.Context,
publicKey *ecdsa.PublicKey,
recipient *ecdsa.PublicKey,
chatID string,
data []byte,
clock int64,
) ([]byte, *protocol.Message, error) {
p.logger.Debug(
"sending a private message",
zap.Binary("public-key", crypto.FromECDSAPub(publicKey)),
zap.Binary("public-key", crypto.FromECDSAPub(recipient)),
)
message := protocol.CreatePrivateTextMessage(data, clock, chatID)
@ -103,7 +110,7 @@ func (p *messageProcessor) SendPrivate(
return nil, nil, errors.Wrap(err, "failed to encode message")
}
messageID, err := p.sendPrivate(ctx, publicKey, encodedMessage)
messageID, err := p.sendPrivate(ctx, recipient, encodedMessage)
if err != nil {
return nil, nil, err
}
@ -113,23 +120,25 @@ func (p *messageProcessor) SendPrivate(
// SendPrivateRaw takes encoded data, encrypts it and sends through the wire.
func (p *messageProcessor) SendPrivateRaw(
ctx context.Context,
publicKey *ecdsa.PublicKey,
recipient *ecdsa.PublicKey,
data []byte,
) ([]byte, error) {
p.logger.Debug(
"sending a private message",
zap.Binary("public-key", crypto.FromECDSAPub(publicKey)),
zap.Binary("public-key", crypto.FromECDSAPub(recipient)),
zap.String("site", "SendPrivateRaw"),
)
return p.sendPrivate(ctx, publicKey, data)
return p.sendPrivate(ctx, recipient, data)
}
// sendPrivate sends data to the recipient identifying with a given public key.
func (p *messageProcessor) sendPrivate(
ctx context.Context,
publicKey *ecdsa.PublicKey,
recipient *ecdsa.PublicKey,
data []byte,
) ([]byte, error) {
p.logger.Debug("sending private message", zap.Binary("recipient", crypto.FromECDSAPub(recipient)))
wrappedMessage, err := p.tryWrapMessageV1(data)
if err != nil {
return nil, errors.Wrap(err, "failed to wrap message")
@ -138,19 +147,19 @@ func (p *messageProcessor) sendPrivate(
messageID := protocol.MessageID(&p.identity.PublicKey, wrappedMessage)
if p.featureFlags.datasync {
if err := p.addToDataSync(publicKey, wrappedMessage); err != nil {
if err := p.addToDataSync(recipient, wrappedMessage); err != nil {
return nil, errors.Wrap(err, "failed to send message with datasync")
}
// No need to call transport tracking.
// It is done in a data sync dispatch step.
} else {
messageSpec, err := p.protocol.BuildDirectMessage(p.identity, publicKey, wrappedMessage)
messageSpec, err := p.protocol.BuildDirectMessage(p.identity, recipient, wrappedMessage)
if err != nil {
return nil, errors.Wrap(err, "failed to encrypt message")
}
hash, newMessage, err := p.sendMessageSpec(ctx, publicKey, messageSpec)
hash, newMessage, err := p.sendMessageSpec(ctx, recipient, messageSpec)
if err != nil {
return nil, errors.Wrap(err, "failed to send a message spec")
}
@ -161,6 +170,61 @@ func (p *messageProcessor) sendPrivate(
return messageID, nil
}
func (p *messageProcessor) SendGroup(
ctx context.Context,
recipients []*ecdsa.PublicKey,
chatID string,
data []byte,
clock int64,
) ([][]byte, []*protocol.Message, error) {
p.logger.Debug("sending a group message", zap.Int("membersCount", len(recipients)))
message := protocol.CreatePrivateGroupTextMessage(data, clock, chatID)
encodedMessage, err := p.encodeMessage(message)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to encode message")
}
var resultIDs [][]byte
for _, recipient := range recipients {
messageID, err := p.sendPrivate(ctx, recipient, encodedMessage)
if err != nil {
return nil, nil, err
}
resultIDs = append(resultIDs, messageID)
}
return resultIDs, nil, nil
}
func (p *messageProcessor) SendMembershipUpdate(
ctx context.Context,
recipients []*ecdsa.PublicKey,
chatID string,
updates []protocol.MembershipUpdate,
clock int64,
) ([][]byte, error) {
p.logger.Debug("sending a membership update", zap.Int("membersCount", len(recipients)))
message := protocol.MembershipUpdateMessage{
ChatID: chatID,
Updates: updates,
}
encodedMessage, err := protocol.EncodeMembershipUpdateMessage(message)
if err != nil {
return nil, errors.Wrap(err, "failed to encode membership update message")
}
var resultIDs [][]byte
for _, recipient := range recipients {
messageID, err := p.sendPrivate(ctx, recipient, encodedMessage)
if err != nil {
return nil, err
}
resultIDs = append(resultIDs, messageID)
}
return resultIDs, nil
}
func (p *messageProcessor) SendPublic(ctx context.Context, chatID string, data []byte, clock int64) ([]byte, error) {
logger := p.logger.With(zap.String("site", "SendPublic"))
logger.Debug("sending a public message", zap.String("chatID", chatID))
@ -249,6 +313,18 @@ func (p *messageProcessor) Process(shhMessage *whispertypes.Message) ([]*protoco
m.ID = statusMessage.ID
m.SigPubKey = statusMessage.SigPubKey()
decodedMessages = append(decodedMessages, &m)
case protocol.MembershipUpdateMessage:
// Handle user message that can be attached to the membership update.
userMessage := m.Message
if userMessage != nil {
userMessage.ID = statusMessage.ID
userMessage.SigPubKey = statusMessage.SigPubKey()
decodedMessages = append(decodedMessages, userMessage)
}
if err := p.processMembershipUpdate(m); err != nil {
hlogger.Error("failed to process MembershipUpdateMessage", zap.Error(err))
}
case protocol.PairMessage:
fromOurDevice := isPubKeyEqual(statusMessage.SigPubKey(), &p.identity.PublicKey)
if !fromOurDevice {
@ -267,6 +343,16 @@ func (p *messageProcessor) Process(shhMessage *whispertypes.Message) ([]*protoco
return decodedMessages, nil
}
func (p *messageProcessor) processMembershipUpdate(m protocol.MembershipUpdateMessage) error {
if err := m.Verify(); err != nil {
return err
}
if p.handler != nil {
return p.handler.HandleMembershipUpdate(m)
}
return errors.New("missing handler")
}
func (p *messageProcessor) processPairMessage(m protocol.PairMessage) error {
metadata := &multidevice.InstallationMetadata{
Name: m.Name,

View File

@ -9,6 +9,8 @@ import (
"strings"
"time"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/pkg/errors"
"go.uber.org/zap"
@ -284,6 +286,7 @@ func NewMessenger(
database,
encryptionProtocol,
t,
newPersistentMessageHandler(&sqlitePersistence{db: database}),
logger,
c.featureFlags,
)
@ -454,12 +457,20 @@ func (m *Messenger) Mailservers() ([]string, error) {
}
func (m *Messenger) Join(chat Chat) error {
if chat.PublicKey != nil {
switch chat.ChatType {
case ChatTypeOneToOne:
return m.transport.JoinPrivate(chat.PublicKey)
} else if chat.Name != "" {
case ChatTypePrivateGroupChat:
members, err := chat.MembersAsPublicKeys()
if err != nil {
return err
}
return m.transport.JoinGroup(members)
case ChatTypePublic:
return m.transport.JoinPublic(chat.Name)
default:
return errors.New("chat is neither public nor private")
}
return errors.New("chat is neither public nor private")
}
func (m *Messenger) Leave(chat Chat) error {
@ -471,6 +482,94 @@ func (m *Messenger) Leave(chat Chat) error {
return errors.New("chat is neither public nor private")
}
// TODO: consider moving to a ChatManager ???
func (m *Messenger) CreateGroupChat(name string) (*Chat, error) {
chat := createGroupChat()
group, err := protocol.NewGroupWithCreator(name, m.identity)
if err != nil {
return nil, err
}
chat.updateChatFromProtocolGroup(group)
return &chat, nil
}
func (m *Messenger) AddMembersToChat(ctx context.Context, chat *Chat, members []*ecdsa.PublicKey) error {
group, err := newProtocolGroupFromChat(chat)
if err != nil {
return err
}
encodedMembers := make([]string, len(members))
for idx, member := range members {
encodedMembers[idx] = hexutil.Encode(crypto.FromECDSAPub(member))
}
event := protocol.NewMembersAddedEvent(encodedMembers, group.NextClockValue())
err = group.ProcessEvent(&m.identity.PublicKey, event)
if err != nil {
return err
}
if err := m.propagateMembershipUpdates(ctx, group); err != nil {
return err
}
chat.updateChatFromProtocolGroup(group)
return m.SaveChat(*chat)
}
func (m *Messenger) ConfirmJoiningGroup(ctx context.Context, chat *Chat) error {
group, err := newProtocolGroupFromChat(chat)
if err != nil {
return err
}
event := protocol.NewMemberJoinedEvent(
statusproto.EncodeHex(crypto.FromECDSAPub(&m.identity.PublicKey)),
group.NextClockValue(),
)
err = group.ProcessEvent(&m.identity.PublicKey, event)
if err != nil {
return err
}
if err := m.propagateMembershipUpdates(ctx, group); err != nil {
return err
}
chat.updateChatFromProtocolGroup(group)
return m.SaveChat(*chat)
}
func (m *Messenger) propagateMembershipUpdates(ctx context.Context, group *protocol.Group) error {
events := make([]protocol.MembershipUpdateEvent, len(group.Updates()))
for idx, event := range group.Updates() {
events[idx] = event.MembershipUpdateEvent
}
update := protocol.MembershipUpdate{
ChatID: group.ChatID(),
Events: events,
}
if err := update.Sign(m.identity); err != nil {
return err
}
recipients, err := stringSliceToPublicKeys(group.Members(), true)
if err != nil {
return err
}
// Filter out my key from the recipients
n := 0
for _, recipient := range recipients {
if !isPubKeyEqual(recipient, &m.identity.PublicKey) {
recipients[n] = recipient
n++
}
}
recipients = recipients[:n]
// Finally send membership updates to all recipients.
_, err = m.processor.SendMembershipUpdate(
ctx,
recipients,
group.ChatID(),
[]protocol.MembershipUpdate{update},
group.NextClockValue(),
)
return err
}
func (m *Messenger) SaveChat(chat Chat) error {
return m.persistence.SaveChat(chat)
}
@ -522,7 +621,7 @@ func (m *Messenger) Contacts() ([]*Contact, error) {
return m.persistence.Contacts()
}
func (m *Messenger) Send(ctx context.Context, chatID string, data []byte) ([]byte, error) {
func (m *Messenger) Send(ctx context.Context, chatID string, data []byte) ([][]byte, error) {
logger := m.logger.With(zap.String("site", "Send"), zap.String("chatID", chatID))
// A valid added chat is required.
@ -538,7 +637,8 @@ func (m *Messenger) Send(ctx context.Context, chatID string, data []byte) ([]byt
logger.Debug("last message clock received", zap.Int64("clock", clock))
if chat.PublicKey != nil {
switch chat.ChatType {
case ChatTypeOneToOne:
logger.Debug("sending private message", zap.Binary("publicKey", crypto.FromECDSAPub(chat.PublicKey)))
id, message, err := m.processor.SendPrivate(ctx, chat.PublicKey, chat.ID, data, clock)
@ -546,27 +646,64 @@ func (m *Messenger) Send(ctx context.Context, chatID string, data []byte) ([]byt
return nil, err
}
// Save our message because it won't be received from the transport layer.
message.ID = id // a Message need ID to be properly stored in the db
message.SigPubKey = &m.identity.PublicKey
message.ChatID = chatID
if err := m.cacheOwnMessage(chatID, id, message); err != nil {
return nil, err
}
if m.messagesPersistenceEnabled {
_, err = m.persistence.SaveMessages([]*protocol.Message{message})
if err != nil {
return [][]byte{id}, nil
case ChatTypePublic:
logger.Debug("sending public message", zap.String("chatName", chat.Name))
id, err := m.processor.SendPublic(ctx, chat.ID, data, clock)
if err != nil {
return nil, err
}
return [][]byte{id}, nil
case ChatTypePrivateGroupChat:
logger.Debug("sending group message", zap.String("chatName", chat.Name))
recipients, err := chat.MembersAsPublicKeys()
if err != nil {
return nil, err
}
// Filter me out of recipients.
n := 0
for _, item := range recipients {
if !isPubKeyEqual(item, &m.identity.PublicKey) {
recipients[n] = item
n++
}
}
ids, messages, err := m.processor.SendGroup(ctx, recipients[:n], chat.ID, data, clock)
if err != nil {
return nil, err
}
for idx, message := range messages {
if err := m.cacheOwnMessage(chatID, ids[idx], message); err != nil {
return nil, err
}
}
// Cache it to be returned in Retrieve().
m.ownMessages = append(m.ownMessages, message)
return id, nil
} else if chat.Name != "" {
logger.Debug("sending public message", zap.String("chatName", chat.Name))
return m.processor.SendPublic(ctx, chat.ID, data, clock)
return ids, nil
default:
return nil, errors.New("chat is neither public nor private")
}
return nil, errors.New("chat is neither public nor private")
}
func (m *Messenger) cacheOwnMessage(chatID string, id []byte, message *protocol.Message) error {
// Save our message because it won't be received from the transport layer.
message.ID = id // a Message need ID to be properly stored in the db
message.SigPubKey = &m.identity.PublicKey
message.ChatID = chatID
if m.messagesPersistenceEnabled {
_, err := m.persistence.SaveMessages([]*protocol.Message{message})
if err != nil {
return err
}
}
// Cache it to be returned in Retrieve().
m.ownMessages = append(m.ownMessages, message)
return nil
}
// SendRaw takes encoded data, encrypts it and sends through the wire.

View File

@ -81,6 +81,9 @@ type WhisperServiceTransport struct {
}
// NewWhisperServiceTransport returns a new WhisperServiceTransport.
// TODO: leaving a chat should verify that for a given public key
// there are no other chats. It may happen that we leave a private chat
// but still have a public chat for a given public key.
func NewWhisperServiceTransport(
shh whispertypes.Whisper,
privateKey *ecdsa.PrivateKey,
@ -185,6 +188,30 @@ func (a *WhisperServiceTransport) LeavePrivate(publicKey *ecdsa.PublicKey) error
return a.filters.Remove(filters...)
}
func (a *WhisperServiceTransport) JoinGroup(publicKeys []*ecdsa.PublicKey) error {
_, err := a.filters.LoadDiscovery()
if err != nil {
return err
}
for _, pk := range publicKeys {
_, err = a.filters.LoadContactCode(pk)
if err != nil {
return err
}
}
return nil
}
func (a *WhisperServiceTransport) LeaveGroup(publicKeys []*ecdsa.PublicKey) error {
for _, publicKey := range publicKeys {
filters := a.filters.FiltersByPublicKey(publicKey)
if err := a.filters.Remove(filters...); err != nil {
return err
}
}
return nil
}
type Message struct {
Message *whispertypes.Message
Public bool

View File

@ -163,10 +163,6 @@ func membershipUpdateMessageHandler(d transit.Decoder, value interface{}) (inter
if !ok {
break
}
update.From, ok = value[transit.Keyword("from")].(string)
if !ok {
break
}
update.Signature, ok = value[transit.Keyword("signature")].(string)
if !ok {
break

View File

@ -77,7 +77,6 @@ func (messageValueEncoder) Encode(e transit.Encoder, value reflect.Value, asStri
element := map[interface{}]interface{}{
transit.Keyword("chat-id"): update.ChatID,
transit.Keyword("from"): update.From,
transit.Keyword("events"): events,
transit.Keyword("signature"): update.Signature,
}

View File

@ -3,10 +3,18 @@ package statusproto
import (
"bytes"
"crypto/ecdsa"
"encoding/hex"
"encoding/json"
"fmt"
"reflect"
"sort"
"strings"
"time"
"github.com/ethereum/go-ethereum/common/hexutil"
gethcrypto "github.com/ethereum/go-ethereum/crypto"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/status-im/status-protocol-go/crypto"
)
@ -29,11 +37,35 @@ type MembershipUpdateMessage struct {
Message *Message `json:"message"` // optional message
}
// Verify makes sure that the received update message has a valid signature.
// It also extracts public key from the signature available as From field.
// It does not verify the updates and their events. This should be done
// separately using Group struct.
func (m *MembershipUpdateMessage) Verify() error {
for idx, update := range m.Updates {
if err := update.extractFrom(); err != nil {
return errors.Wrapf(err, "failed to extract an author of %d update", idx)
}
m.Updates[idx] = update
}
return nil
}
// EncodeMembershipUpdateMessage encodes a MembershipUpdateMessage using Transit serialization.
func EncodeMembershipUpdateMessage(value MembershipUpdateMessage) ([]byte, error) {
var buf bytes.Buffer
encoder := NewMessageEncoder(&buf)
if err := encoder.Encode(value); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
type MembershipUpdate struct {
ChatID string `json:"chatId"`
From string `json:"from"`
Signature string `json:"signature"`
Signature string `json:"signature"` // hex-encoded without 0x prefix
Events []MembershipUpdateEvent `json:"events"`
From string `json:"from"` // hex-encoded with 0x prefix
}
// Sign creates a signature from MembershipUpdateEvents
@ -41,22 +73,7 @@ type MembershipUpdate struct {
// It follows the algorithm describe in the spec:
// https://github.com/status-im/specs/blob/master/status-group-chats-spec.md#signature.
func (u *MembershipUpdate) Sign(identity *ecdsa.PrivateKey) error {
sort.Slice(u.Events, func(i, j int) bool {
return u.Events[i].ClockValue < u.Events[j].ClockValue
})
tuples := make([]interface{}, len(u.Events))
for idx, event := range u.Events {
tuples[idx] = tupleMembershipUpdateEvent(event)
}
structureToSign := []interface{}{
tuples,
u.ChatID,
}
data, err := json.Marshal(structureToSign)
if err != nil {
return err
}
signature, err := crypto.SignBytesAsHex(data, identity)
signature, err := createMembershipUpdateSignature(u.ChatID, u.Events, identity)
if err != nil {
return err
}
@ -64,6 +81,38 @@ func (u *MembershipUpdate) Sign(identity *ecdsa.PrivateKey) error {
return nil
}
func (u *MembershipUpdate) extractFrom() error {
content, err := stringifyMembershipUpdateEvents(u.ChatID, u.Events)
if err != nil {
return errors.Wrap(err, "failed to stringify events")
}
signatureBytes, err := hex.DecodeString(u.Signature)
if err != nil {
return errors.Wrap(err, "failed to decode signature")
}
publicKey, err := crypto.ExtractSignature(content, signatureBytes)
if err != nil {
return errors.Wrap(err, "failed to extract signature")
}
u.From = hexutil.Encode(gethcrypto.FromECDSAPub(publicKey))
return nil
}
func (u *MembershipUpdate) Flat() []MembershipUpdateFlat {
result := make([]MembershipUpdateFlat, 0, len(u.Events))
for _, event := range u.Events {
result = append(result, MembershipUpdateFlat{
MembershipUpdateEvent: event,
ChatID: u.ChatID,
Signature: u.Signature,
From: u.From,
})
}
return result
}
// MembershipUpdateEvent contains an event information.
// Member and Members are hex-encoded values with 0x prefix.
type MembershipUpdateEvent struct {
Type string `json:"type"`
ClockValue int64 `json:"clockValue"`
@ -72,10 +121,49 @@ type MembershipUpdateEvent struct {
Name string `json:"name,omitempty"` // name of the group chat
}
func NewChatCreatedEvent(name string, clock int64) MembershipUpdateEvent {
func (u MembershipUpdateEvent) Equal(update MembershipUpdateEvent) bool {
return u.Type == update.Type &&
u.ClockValue == update.ClockValue &&
u.Member == update.Member &&
stringSliceEquals(u.Members, update.Members) &&
u.Name == update.Name
}
type MembershipUpdateFlat struct {
MembershipUpdateEvent
ChatID string `json:"chatId"`
Signature string `json:"signature"`
From string `json:"from"`
}
func (u MembershipUpdateFlat) Equal(update MembershipUpdateFlat) bool {
return u.ChatID == update.ChatID &&
u.Signature == update.Signature &&
u.From == update.From &&
u.MembershipUpdateEvent.Equal(update.MembershipUpdateEvent)
}
func MergeFlatMembershipUpdates(dest []MembershipUpdateFlat, src []MembershipUpdateFlat) []MembershipUpdateFlat {
for _, update := range src {
var exists bool
for _, existing := range dest {
if existing.Equal(update) {
exists = true
break
}
}
if !exists {
dest = append(dest, update)
}
}
return dest
}
func NewChatCreatedEvent(name string, admin string, clock int64) MembershipUpdateEvent {
return MembershipUpdateEvent{
Type: MembershipUpdateChatCreated,
Name: name,
Member: admin,
ClockValue: clock,
}
}
@ -128,14 +216,27 @@ func NewAdminRemovedEvent(admin string, clock int64) MembershipUpdateEvent {
}
}
// EncodeMembershipUpdateMessage encodes a MembershipUpdateMessage using Transit serialization.
func EncodeMembershipUpdateMessage(value MembershipUpdateMessage) ([]byte, error) {
var buf bytes.Buffer
encoder := NewMessageEncoder(&buf)
if err := encoder.Encode(value); err != nil {
return nil, err
func stringifyMembershipUpdateEvents(chatID string, events []MembershipUpdateEvent) ([]byte, error) {
sort.Slice(events, func(i, j int) bool {
return events[i].ClockValue < events[j].ClockValue
})
tuples := make([]interface{}, len(events))
for idx, event := range events {
tuples[idx] = tupleMembershipUpdateEvent(event)
}
return buf.Bytes(), nil
structureToSign := []interface{}{
tuples,
chatID,
}
return json.Marshal(structureToSign)
}
func createMembershipUpdateSignature(chatID string, events []MembershipUpdateEvent, identity *ecdsa.PrivateKey) (string, error) {
data, err := stringifyMembershipUpdateEvents(chatID, events)
if err != nil {
return "", err
}
return crypto.SignBytesAsHex(data, identity)
}
var membershipUpdateEventFieldNamesCompat = map[string]string{
@ -173,41 +274,222 @@ func tupleMembershipUpdateEvent(update MembershipUpdateEvent) [][]interface{} {
}
type Group struct {
ChatID string
Admins []string
Contacts []string
chatID string
name string
updates []MembershipUpdateFlat
admins *stringSet
members *stringSet
}
// ValidateEvent returns true if a given event is valid.
func (g *Group) ValidateEvent(from string, event MembershipUpdateEvent) bool {
func groupChatID(creator *ecdsa.PublicKey) string {
return uuid.New().String() + "-" + hexutil.Encode(gethcrypto.FromECDSAPub(creator))
}
func NewGroupWithMembershipUpdates(chatID string, updates []MembershipUpdate) (*Group, error) {
flatten := make([]MembershipUpdateFlat, 0, len(updates))
for _, update := range updates {
flatten = append(flatten, update.Flat()...)
}
return newGroup(chatID, flatten)
}
func NewGroupWithCreator(name string, creator *ecdsa.PrivateKey) (*Group, error) {
chatID := groupChatID(&creator.PublicKey)
creatorHex := publicKeyToString(&creator.PublicKey)
clock := TimestampInMsFromTime(time.Now())
chatCreated := NewChatCreatedEvent(name, creatorHex, int64(clock))
update := MembershipUpdate{
ChatID: chatID,
From: creatorHex,
Events: []MembershipUpdateEvent{chatCreated},
}
if err := update.Sign(creator); err != nil {
return nil, err
}
return newGroup(chatID, update.Flat())
}
func NewGroup(chatID string, updates []MembershipUpdateFlat) (*Group, error) {
return newGroup(chatID, updates)
}
func newGroup(chatID string, updates []MembershipUpdateFlat) (*Group, error) {
g := Group{
chatID: chatID,
updates: updates,
admins: newStringSet(),
members: newStringSet(),
}
if err := g.init(); err != nil {
return nil, err
}
return &g, nil
}
func (g *Group) init() error {
g.sortEvents()
var chatID string
for _, update := range g.updates {
if chatID == "" {
chatID = update.ChatID
} else if update.ChatID != chatID {
return errors.New("updates contain different chat IDs")
}
valid := g.validateEvent(update.From, update.MembershipUpdateEvent)
if !valid {
return fmt.Errorf("invalid event %#+v from %s", update.MembershipUpdateEvent, update.From)
}
g.processEvent(update.From, update.MembershipUpdateEvent)
}
valid := g.validateChatID(g.chatID)
if !valid {
return fmt.Errorf("invalid chat ID: %s", g.chatID)
}
if chatID != g.chatID {
return fmt.Errorf("expected chat ID equal %s, got %s", g.chatID, chatID)
}
return nil
}
func (g Group) ChatID() string {
return g.chatID
}
func (g Group) Updates() []MembershipUpdateFlat {
return g.updates
}
func (g Group) Name() string {
return g.name
}
func (g Group) Members() []string {
return g.members.List()
}
func (g Group) Admins() []string {
return g.admins.List()
}
func (g Group) Joined() []string {
var result []string
for _, update := range g.updates {
if update.Type == MembershipUpdateMemberJoined {
result = append(result, update.Member)
}
}
return result
}
func (g *Group) ProcessEvents(from *ecdsa.PublicKey, events []MembershipUpdateEvent) error {
for _, event := range events {
err := g.ProcessEvent(from, event)
if err != nil {
return err
}
}
return nil
}
func (g *Group) ProcessEvent(from *ecdsa.PublicKey, event MembershipUpdateEvent) error {
fromHex := hexutil.Encode(gethcrypto.FromECDSAPub(from))
if !g.validateEvent(fromHex, event) {
return fmt.Errorf("invalid event %#+v from %s", event, from)
}
update := MembershipUpdate{
ChatID: g.chatID,
From: fromHex,
Events: []MembershipUpdateEvent{event},
}
g.updates = append(g.updates, update.Flat()...)
g.processEvent(fromHex, event)
return nil
}
func (g Group) LastClockValue() int64 {
if len(g.updates) == 0 {
return 0
}
return g.updates[len(g.updates)-1].ClockValue
}
func (g Group) NextClockValue() int64 {
return g.LastClockValue() + 1
}
func (g Group) creator() (string, error) {
if len(g.updates) == 0 {
return "", errors.New("no events in the group")
}
first := g.updates[0]
if first.Type != MembershipUpdateChatCreated {
return "", fmt.Errorf("expected first event to be 'chat-created', got %s", first.Type)
}
return first.From, nil
}
func (g Group) validateChatID(chatID string) bool {
creator, err := g.creator()
if err != nil || creator == "" {
return false
}
// TODO: It does not verify that the prefix is a valid UUID.
// Improve it so that the prefix follows UUIDv4 spec.
return strings.HasSuffix(chatID, creator) && chatID != creator
}
// validateEvent returns true if a given event is valid.
func (g Group) validateEvent(from string, event MembershipUpdateEvent) bool {
switch event.Type {
case MembershipUpdateChatCreated:
return len(g.Admins) == 0 && len(g.Contacts) == 0
return g.admins.Empty() && g.members.Empty()
case MembershipUpdateNameChanged:
return stringSliceContains(g.Admins, from) && len(event.Name) > 0
return g.admins.Has(from) && len(event.Name) > 0
case MembershipUpdateMembersAdded:
return stringSliceContains(g.Admins, from)
return g.admins.Has(from)
case MembershipUpdateMemberJoined:
return stringSliceContains(g.Contacts, from) && from == event.Member
return g.members.Has(from) && from == event.Member
case MembershipUpdateMemberRemoved:
// Member can remove themselves or admin can remove a member.
return from == event.Member || (stringSliceContains(g.Admins, from) && !stringSliceContains(g.Admins, event.Member))
return from == event.Member || (g.admins.Has(from) && !g.admins.Has(event.Member))
case MembershipUpdateAdminsAdded:
return stringSliceContains(g.Admins, from) && stringSliceSubset(event.Members, g.Contacts)
return g.admins.Has(from) && stringSliceSubset(event.Members, g.members.List())
case MembershipUpdateAdminRemoved:
return stringSliceContains(g.Admins, from) && from == event.Member
return g.admins.Has(from) && from == event.Member
default:
return false
}
}
func stringSliceContains(slice []string, item string) bool {
for _, s := range slice {
if s == item {
return true
}
func (g *Group) processEvent(from string, event MembershipUpdateEvent) {
switch event.Type {
case MembershipUpdateChatCreated:
g.name = event.Name
g.members.Add(event.Member)
g.admins.Add(event.Member)
case MembershipUpdateNameChanged:
g.name = event.Name
case MembershipUpdateAdminsAdded:
g.admins.Add(event.Members...)
case MembershipUpdateAdminRemoved:
g.admins.Remove(event.Member)
case MembershipUpdateMembersAdded:
g.members.Add(event.Members...)
case MembershipUpdateMemberRemoved:
g.members.Remove(event.Member)
case MembershipUpdateMemberJoined:
g.members.Add(event.Member)
}
return false
}
func (g *Group) sortEvents() {
sort.Slice(g.updates, func(i, j int) bool {
return g.updates[i].ClockValue < g.updates[j].ClockValue
})
}
func stringSliceSubset(subset []string, set []string) bool {
@ -225,3 +507,82 @@ func stringSliceSubset(subset []string, set []string) bool {
}
return false
}
func stringSliceEquals(slice1, slice2 []string) bool {
set := map[string]struct{}{}
for _, s := range slice1 {
set[s] = struct{}{}
}
for _, s := range slice2 {
_, ok := set[s]
if !ok {
return false
}
}
return true
}
func publicKeyToString(publicKey *ecdsa.PublicKey) string {
return hexutil.Encode(gethcrypto.FromECDSAPub(publicKey))
}
type stringSet struct {
m map[string]struct{}
items []string
}
func newStringSet() *stringSet {
return &stringSet{
m: make(map[string]struct{}),
}
}
func newStringSetFromSlice(s []string) *stringSet {
set := newStringSet()
if len(s) > 0 {
set.Add(s...)
}
return set
}
func (s *stringSet) Add(items ...string) {
for _, item := range items {
if _, ok := s.m[item]; !ok {
s.m[item] = struct{}{}
s.items = append(s.items, item)
}
}
}
func (s *stringSet) Remove(items ...string) {
for _, item := range items {
if _, ok := s.m[item]; ok {
delete(s.m, item)
s.removeFromItems(item)
}
}
}
func (s *stringSet) Has(item string) bool {
_, ok := s.m[item]
return ok
}
func (s *stringSet) Empty() bool {
return len(s.items) == 0
}
func (s *stringSet) List() []string {
return s.items
}
func (s *stringSet) removeFromItems(dropped string) {
n := 0
for _, item := range s.items {
if item != dropped {
s.items[n] = item
n++
}
}
s.items = s.items[:n]
}

View File

@ -492,7 +492,7 @@ func (n *Node) onMessage(sender state.PeerID, msg protobuf.Message) error {
)
err := n.syncState.Remove(id, sender)
if err != nil {
if err != nil && err != state.ErrStateNotFound {
return err
}

10
vendor/modules.txt vendored
View File

@ -122,9 +122,10 @@ github.com/go-stack/stack
# github.com/gogo/protobuf v1.3.0
github.com/gogo/protobuf/io
github.com/gogo/protobuf/proto
# github.com/golang-migrate/migrate/v4 v4.5.0
# github.com/golang-migrate/migrate/v4 v4.6.2
github.com/golang-migrate/migrate/v4
github.com/golang-migrate/migrate/v4/database
github.com/golang-migrate/migrate/v4/internal/url
github.com/golang-migrate/migrate/v4/source
# github.com/golang/mock v1.3.1
github.com/golang/mock/gomock
@ -339,16 +340,17 @@ github.com/status-im/doubleratchet
github.com/status-im/go-multiaddr-ethv4
# github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48
github.com/status-im/keycard-go/derivationpath
# github.com/status-im/migrate/v4 v4.3.1-status.0.20190822050738-a9d340ec8fb7
# github.com/status-im/migrate/v4 v4.6.2-status.2
github.com/status-im/migrate/v4
github.com/status-im/migrate/v4/database/postgres
github.com/status-im/migrate/v4/database/sqlcipher
github.com/status-im/migrate/v4/internal/url
github.com/status-im/migrate/v4/source/go_bindata
# github.com/status-im/rendezvous v1.3.0
github.com/status-im/rendezvous
github.com/status-im/rendezvous/protocol
github.com/status-im/rendezvous/server
# github.com/status-im/status-protocol-go v0.3.1
# github.com/status-im/status-protocol-go v0.4.1-0.20191014103017-1f15a058b66c
github.com/status-im/status-protocol-go
github.com/status-im/status-protocol-go/applicationmetadata
github.com/status-im/status-protocol-go/bridge/geth
@ -396,7 +398,7 @@ github.com/syndtr/goleveldb/leveldb/util
# github.com/tyler-smith/go-bip39 v1.0.2
github.com/tyler-smith/go-bip39
github.com/tyler-smith/go-bip39/wordlists
# github.com/vacp2p/mvds v0.0.21
# github.com/vacp2p/mvds v0.0.23-0.20191014101555-026e462c829d
github.com/vacp2p/mvds/node
github.com/vacp2p/mvds/node/migrations
github.com/vacp2p/mvds/peers