From 9d7c570593b1f88e9688204d8e1041d57b98da1f Mon Sep 17 00:00:00 2001 From: Andrea Maria Piana Date: Fri, 15 Nov 2019 09:52:28 +0100 Subject: [PATCH] Add PrepareContent and upgrade status-go (#1674) --- Makefile | 2 +- VERSION | 2 +- go.mod | 4 +- go.sum | 14 +- services/shhext/api.go | 5 + .../github.com/gomarkdown/markdown/.gitignore | 13 + vendor/github.com/gomarkdown/markdown/.gitpod | 7 + .../gomarkdown/markdown/.travis.yml | 17 + .../gomarkdown/markdown/LICENSE.txt | 31 + .../github.com/gomarkdown/markdown/README.md | 325 ++++ .../gomarkdown/markdown/ast/attribute.go | 10 + .../github.com/gomarkdown/markdown/ast/doc.go | 4 + .../gomarkdown/markdown/ast/node.go | 731 +++++++++ .../gomarkdown/markdown/ast/print.go | 158 ++ .../markdown/changes-from-blackfriday.md | 27 + .../gomarkdown/markdown/codecov.yml | 8 + vendor/github.com/gomarkdown/markdown/doc.go | 35 + vendor/github.com/gomarkdown/markdown/fuzz.go | 9 + vendor/github.com/gomarkdown/markdown/go.mod | 11 + vendor/github.com/gomarkdown/markdown/go.sum | 21 + .../gomarkdown/markdown/markdown.go | 66 + .../gomarkdown/markdown/parser/aside.go | 73 + .../gomarkdown/markdown/parser/attribute.go | 116 ++ .../gomarkdown/markdown/parser/block.go | 1415 +++++++++++++++++ .../gomarkdown/markdown/parser/callout.go | 29 + .../gomarkdown/markdown/parser/caption.go | 70 + .../gomarkdown/markdown/parser/citation.go | 86 + .../gomarkdown/markdown/parser/esc.go | 20 + .../gomarkdown/markdown/parser/figures.go | 119 ++ .../gomarkdown/markdown/parser/include.go | 129 ++ .../gomarkdown/markdown/parser/inline.go | 1238 ++++++++++++++ .../gomarkdown/markdown/parser/matter.go | 36 + .../gomarkdown/markdown/parser/options.go | 32 + .../gomarkdown/markdown/parser/parser.go | 803 ++++++++++ .../gomarkdown/markdown/parser/ref.go | 89 ++ vendor/github.com/gomarkdown/markdown/todo.md | 7 + .../gomarkdown/markdown/tracking-perf.md | 189 +++ .../status-protocol-go/datasync/datasync.go | 28 +- .../status-im/status-protocol-go/go.mod | 3 + .../status-im/status-protocol-go/go.sum | 11 + .../status-im/status-protocol-go/messenger.go | 27 + .../status-protocol-go/persistence.go | 250 ++- .../status-protocol-go/persistence_legacy.go | 61 +- .../status-protocol-go/v1/content.go | 38 + .../status-protocol-go/v1/message.go | 8 - vendor/modules.txt | 6 +- 46 files changed, 6189 insertions(+), 194 deletions(-) create mode 100644 vendor/github.com/gomarkdown/markdown/.gitignore create mode 100644 vendor/github.com/gomarkdown/markdown/.gitpod create mode 100644 vendor/github.com/gomarkdown/markdown/.travis.yml create mode 100644 vendor/github.com/gomarkdown/markdown/LICENSE.txt create mode 100644 vendor/github.com/gomarkdown/markdown/README.md create mode 100644 vendor/github.com/gomarkdown/markdown/ast/attribute.go create mode 100644 vendor/github.com/gomarkdown/markdown/ast/doc.go create mode 100644 vendor/github.com/gomarkdown/markdown/ast/node.go create mode 100644 vendor/github.com/gomarkdown/markdown/ast/print.go create mode 100644 vendor/github.com/gomarkdown/markdown/changes-from-blackfriday.md create mode 100644 vendor/github.com/gomarkdown/markdown/codecov.yml create mode 100644 vendor/github.com/gomarkdown/markdown/doc.go create mode 100644 vendor/github.com/gomarkdown/markdown/fuzz.go create mode 100644 vendor/github.com/gomarkdown/markdown/go.mod create mode 100644 vendor/github.com/gomarkdown/markdown/go.sum create mode 100644 vendor/github.com/gomarkdown/markdown/markdown.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/aside.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/attribute.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/block.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/callout.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/caption.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/citation.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/esc.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/figures.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/include.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/inline.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/matter.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/options.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/parser.go create mode 100644 vendor/github.com/gomarkdown/markdown/parser/ref.go create mode 100644 vendor/github.com/gomarkdown/markdown/todo.md create mode 100644 vendor/github.com/gomarkdown/markdown/tracking-perf.md create mode 100644 vendor/github.com/status-im/status-protocol-go/v1/content.go diff --git a/Makefile b/Makefile index 10794bb9d..9a6f9919b 100644 --- a/Makefile +++ b/Makefile @@ -267,7 +267,7 @@ canary-test: node-canary lint-install: @# The following installs a specific version of golangci-lint, which is appropriate for a CI server to avoid different results from build to build - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | BINARY=$(GOLANGCI_BINARY) bash -s -- -d -b $(GOPATH)/bin v1.17.1 + curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | BINARY=$(GOLANGCI_BINARY) bash -s -- -d -b $(GOPATH)/bin v1.18.0 lint: @echo "lint" diff --git a/VERSION b/VERSION index 364cd3446..f54da3517 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.34.0-beta.7 +0.34.0-beta.8 diff --git a/go.mod b/go.mod index 80dcc693e..9470f96c0 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/russolsen/transit v0.0.0-20180705123435-0794b4c4505a github.com/status-im/migrate/v4 v4.6.2-status.2 github.com/status-im/rendezvous v1.3.0 - github.com/status-im/status-protocol-go v0.4.5-0.20191107122821-775d17008edf + github.com/status-im/status-protocol-go v0.5.1 github.com/status-im/whisper v1.5.2 github.com/stretchr/testify v1.4.0 github.com/syndtr/goleveldb v1.0.0 @@ -42,3 +42,5 @@ require ( gopkg.in/go-playground/validator.v9 v9.29.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) + +replace github.com/gomarkdown/markdown => github.com/status-im/markdown v0.0.0-20191113114344-af599402d015 diff --git a/go.sum b/go.sum index b78cc9894..1f4218bac 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -119,6 +120,7 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -129,6 +131,7 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB github.com/elastic/gosigar v0.0.0-20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo= github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/ethereum/go-ethereum v1.8.20/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY= github.com/ethereum/go-ethereum v1.9.2/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -529,6 +532,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= @@ -572,18 +576,21 @@ github.com/status-im/go-multiaddr-ethv4 v1.2.0 h1:OT84UsUzTCwguqCpJqkrCMiL4VZ1Sv github.com/status-im/go-multiaddr-ethv4 v1.2.0/go.mod h1:2VQ3C+9zEurcceasz12gPAtmEzCeyLUGPeKLSXYQKHo= github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw= github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/status-im/markdown v0.0.0-20191113114344-af599402d015 h1:ijC73VP0hucsy/MRn4cmtoQVB1mKdLcvurMYPvmPc4Y= +github.com/status-im/markdown v0.0.0-20191113114344-af599402d015/go.mod h1:tmG2bxyvZ2EItDO5JewbdFvV45j13IYQgvnMJ3+qAaE= github.com/status-im/migrate/v4 v4.6.2-status.2 h1:SdC+sMDl/aI7vUlwD2qj2p7KsK4T60IS9z4/rYCCbI8= github.com/status-im/migrate/v4 v4.6.2-status.2/go.mod h1:c/kc90n47GZu/58nnz1OMLTf7uE4Da4gZP5qmU+A/v8= github.com/status-im/rendezvous v1.3.0 h1:7RK/MXXW+tlm0asKm1u7Qp7Yni6AO29a7j8+E4Lbjg4= github.com/status-im/rendezvous v1.3.0/go.mod h1:+hzjuP+j/XzLPeF6E50b88pWOTLdTcwjvNYt+Gh1W1s= -github.com/status-im/status-protocol-go v0.4.5-0.20191107122821-775d17008edf h1:1boOd5yMePhXxYei97Rm/hFF45alUpMl87ZAWvlSKtg= -github.com/status-im/status-protocol-go v0.4.5-0.20191107122821-775d17008edf/go.mod h1:r8TgqNOpY+fGKkBfR9PldxSSaBN0EsEEY4a3WsIh9LY= +github.com/status-im/status-protocol-go v0.5.1 h1:mCqYJrL/zWMScFjSLdboL5WANLn01Cz8bAxBwPxww7k= +github.com/status-im/status-protocol-go v0.5.1/go.mod h1:KR/eihnrUq2dZegUOVjrA/1poSNhasA/o82VYyRgeB0= github.com/status-im/whisper v1.5.2 h1:26NgiKusmPic38eQdtXnaY+iaQ/LuQ3Dh0kCGYT/Uxs= github.com/status-im/whisper v1.5.2/go.mod h1:emrOxzJme0k66QtbbQ2bdd3P8RCdLZ8sTD7SkwH1s2s= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -640,6 +647,7 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -736,6 +744,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425222832-ad9eeb80039a/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191109212701-97ad0ed33101/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/services/shhext/api.go b/services/shhext/api.go index 5d653465a..ab30634fe 100644 --- a/services/shhext/api.go +++ b/services/shhext/api.go @@ -28,6 +28,7 @@ import ( statustransp "github.com/status-im/status-protocol-go/transport/whisper" whispertypes "github.com/status-im/status-protocol-go/transport/whisper/types" statusproto_types "github.com/status-im/status-protocol-go/types" + statusprotomessage "github.com/status-im/status-protocol-go/v1" ) const ( @@ -444,6 +445,10 @@ func (api *PublicAPI) SendPublicMessage(ctx context.Context, msg SendPublicMessa return api.service.messenger.SendRaw(ctx, chat, msg.Payload) } +func (api *PublicAPI) PrepareContent(ctx context.Context, content statusprotomessage.Content) statusprotomessage.Content { + return api.service.messenger.PrepareContent(content) +} + // SendDirectMessage sends a 1:1 chat message to the underlying transport // Message's payload is a transit encoded message. // It's important to call PublicAPI.afterSend() so that the client receives a signal diff --git a/vendor/github.com/gomarkdown/markdown/.gitignore b/vendor/github.com/gomarkdown/markdown/.gitignore new file mode 100644 index 000000000..8f15b677f --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/.gitignore @@ -0,0 +1,13 @@ +*.out +*.swp +*.8 +*.6 +_obj +_test* +markdown +tags +fuzz-workdir/ +markdown-fuzz.zip +coverage.txt +testdata/*_got.md +testdata/*_ast.txt diff --git a/vendor/github.com/gomarkdown/markdown/.gitpod b/vendor/github.com/gomarkdown/markdown/.gitpod new file mode 100644 index 000000000..ad5feff66 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/.gitpod @@ -0,0 +1,7 @@ +checkoutLocation: "src/github.com/gomarkdown/markdown" +workspaceLocation: "." +tasks: + - command: > + cd /workspace/src/github.com/gomarkdown/markdown && + go get -v ./... && + go test -c diff --git a/vendor/github.com/gomarkdown/markdown/.travis.yml b/vendor/github.com/gomarkdown/markdown/.travis.yml new file mode 100644 index 000000000..4ec5d7b0c --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/.travis.yml @@ -0,0 +1,17 @@ +dist: bionic +language: go + +go: + - "1.12.x" + +install: + - go build -v ./... + +script: + - go test -v ./... + - go test -run=^$ -bench=BenchmarkReference -benchmem + - ./s/test_with_codecoverage.sh + - ./s/ci_fuzzit.sh + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/gomarkdown/markdown/LICENSE.txt b/vendor/github.com/gomarkdown/markdown/LICENSE.txt new file mode 100644 index 000000000..688046102 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/LICENSE.txt @@ -0,0 +1,31 @@ +Markdown is distributed under the Simplified BSD License: + +Copyright © 2011 Russ Ross +Copyright © 2018 Krzysztof Kowalczyk +Copyright © 2018 Authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided with + the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/gomarkdown/markdown/README.md b/vendor/github.com/gomarkdown/markdown/README.md new file mode 100644 index 000000000..3bca71db9 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/README.md @@ -0,0 +1,325 @@ +# Markdown Parser and HTML Renderer for Go + +[![GoDoc](https://godoc.org/github.com/gomarkdown/markdown?status.svg)](https://godoc.org/github.com/gomarkdown/markdown) [![codecov](https://codecov.io/gh/gomarkdown/markdown/branch/master/graph/badge.svg)](https://codecov.io/gh/gomarkdown/markdown) + +Package `github.com/gomarkdown/markdown` is a very fast Go library for parsing [Markdown](https://daringfireball.net/projects/markdown/) documents and rendering them to HTML. + +It's fast and supports common extensions. + +## Installation + + go get -u github.com/gomarkdown/markdown + +API Docs: + +- https://godoc.org/github.com/gomarkdown/markdown : top level package +- https://godoc.org/github.com/gomarkdown/markdown/ast : defines abstract syntax tree of parsed markdown document +- https://godoc.org/github.com/gomarkdown/markdown/parser : parser +- https://godoc.org/github.com/gomarkdown/markdown/html : html renderer + +## Usage + +To convert markdown text to HTML using reasonable defaults: + +```go +md := []byte("## markdown document") +output := markdown.ToHTML(md, nil, nil) +``` + +## Customizing markdown parser + +Markdown format is loosely specified and there are multiple extensions invented after original specification was created. + +The parser supports several [extensions](https://godoc.org/github.com/gomarkdown/markdown/parser#Extensions). + +Default parser uses most common `parser.CommonExtensions` but you can easily use parser with custom extension: + +```go +import ( + "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/parser" +) + +extensions := parser.CommonExtensions | parser.AutoHeadingIDs +parser := parser.NewWithExtensions(extensions) + +md := []byte("markdown text") +html := markdown.ToHTML(md, parser, nil) +``` + +## Customizing HTML renderer + +Similarly, HTML renderer can be configured with different [options](https://godoc.org/github.com/gomarkdown/markdown/html#RendererOptions) + +Here's how to use a custom renderer: + +```go +import ( + "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/html" +) + +htmlFlags := html.CommonFlags | html.HrefTargetBlank +opts := html.RendererOptions{Flags: htmlFlags} +renderer := html.NewRenderer(opts) + +md := []byte("markdown text") +html := markdown.ToHTML(md, nil, renderer) +``` + +HTML renderer also supports reusing most of the logic and overriding rendering of only specifc nodes. + +You can provide [RenderNodeFunc](https://godoc.org/github.com/gomarkdown/markdown/html#RenderNodeFunc) in [RendererOptions](https://godoc.org/github.com/gomarkdown/markdown/html#RendererOptions). + +The function is called for each node in AST, you can implement custom rendering logic and tell HTML renderer to skip rendering this node. + +Here's the simplest example that drops all code blocks from the output: + +````go +import ( + "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/ast" + "github.com/gomarkdown/markdown/html" +) + +// return (ast.GoToNext, true) to tell html renderer to skip rendering this node +// (because you've rendered it) +func renderHookDropCodeBlock(w io.Writer, node ast.Node, entering bool) (ast.WalkStatus, bool) { + // skip all nodes that are not CodeBlock nodes + if _, ok := node.(*ast.CodeBlock); !ok { + return ast.GoToNext, false + } + // custom rendering logic for ast.CodeBlock. By doing nothing it won't be + // present in the output + return ast.GoToNext, true +} + +opts := html.RendererOptions{ + Flags: html.CommonFlags, + RenderNodeHook: renderHookDropCodeBlock, +} +renderer := html.NewRenderer(opts) +md := "test\n```\nthis code block will be dropped from output\n```\ntext" +html := markdown.ToHTML([]byte(s), nil, renderer) +```` + +## Sanitize untrusted content + +We don't protect against malicious content. When dealing with user-provided +markdown, run renderer HTML through HTML sanitizer such as [Bluemonday](https://github.com/microcosm-cc/bluemonday). + +Here's an example of simple usage with Bluemonday: + +```go +import ( + "github.com/microcosm-cc/bluemonday" + "github.com/gomarkdown/markdown" +) + +// ... +maybeUnsafeHTML := markdown.ToHTML(md, nil, nil) +html := bluemonday.UGCPolicy().SanitizeBytes(maybeUnsafeHTML) +``` + +## mdtohtml command-line tool + +https://github.com/gomarkdown/mdtohtml is a command-line markdown to html +converter built using this library. + +You can also use it as an example of how to use the library. + +You can install it with: + + go get -u github.com/gomarkdown/mdtohtml + +To run: `mdtohtml input-file [output-file]` + +## Features + +- **Compatibility**. The Markdown v1.0.3 test suite passes with + the `--tidy` option. Without `--tidy`, the differences are + mostly in whitespace and entity escaping, where this package is + more consistent and cleaner. + +- **Common extensions**, including table support, fenced code + blocks, autolinks, strikethroughs, non-strict emphasis, etc. + +- **Safety**. Markdown is paranoid when parsing, making it safe + to feed untrusted user input without fear of bad things + happening. The test suite stress tests this and there are no + known inputs that make it crash. If you find one, please let me + know and send me the input that does it. + + NOTE: "safety" in this context means _runtime safety only_. In order to + protect yourself against JavaScript injection in untrusted content, see + [this example](https://github.com/gomarkdown/markdown#sanitize-untrusted-content). + +- **Fast**. It is fast enough to render on-demand in + most web applications without having to cache the output. + +- **Thread safety**. You can run multiple parsers in different + goroutines without ill effect. There is no dependence on global + shared state. + +- **Minimal dependencies**. Only depends on standard library packages in Go. + +- **Standards compliant**. Output successfully validates using the + W3C validation tool for HTML 4.01 and XHTML 1.0 Transitional. + +## Extensions + +In addition to the standard markdown syntax, this package +implements the following extensions: + +- **Intra-word emphasis supression**. The `_` character is + commonly used inside words when discussing code, so having + markdown interpret it as an emphasis command is usually the + wrong thing. We let you treat all emphasis markers as + normal characters when they occur inside a word. + +- **Tables**. Tables can be created by drawing them in the input + using a simple syntax: + + ``` + Name | Age + --------|------ + Bob | 27 + Alice | 23 + ``` + + Table footers are supported as well and can be added with equal signs (`=`): + + ``` + Name | Age + --------|------ + Bob | 27 + Alice | 23 + ========|====== + Total | 50 + ``` + +- **Fenced code blocks**. In addition to the normal 4-space + indentation to mark code blocks, you can explicitly mark them + and supply a language (to make syntax highlighting simple). Just + mark it like this: + + ```go + func getTrue() bool { + return true + } + ``` + + You can use 3 or more backticks to mark the beginning of the + block, and the same number to mark the end of the block. + +- **Definition lists**. A simple definition list is made of a single-line + term followed by a colon and the definition for that term. + + Cat + : Fluffy animal everyone likes + + Internet + : Vector of transmission for pictures of cats + + Terms must be separated from the previous definition by a blank line. + +- **Footnotes**. A marker in the text that will become a superscript number; + a footnote definition that will be placed in a list of footnotes at the + end of the document. A footnote looks like this: + + This is a footnote.[^1] + + [^1]: the footnote text. + +- **Autolinking**. We can find URLs that have not been + explicitly marked as links and turn them into links. + +- **Strikethrough**. Use two tildes (`~~`) to mark text that + should be crossed out. + +- **Hard line breaks**. With this extension enabled newlines in the input + translate into line breaks in the output. This extension is off by default. + +- **Non blocking space**. With this extension enabled spaces preceeded by an backslash n the input + translate non-blocking spaces in the output. This extension is off by default. + +- **Smart quotes**. Smartypants-style punctuation substitution is + supported, turning normal double- and single-quote marks into + curly quotes, etc. + +- **LaTeX-style dash parsing** is an additional option, where `--` + is translated into `–`, and `---` is translated into + `—`. This differs from most smartypants processors, which + turn a single hyphen into an ndash and a double hyphen into an + mdash. + +- **Smart fractions**, where anything that looks like a fraction + is translated into suitable HTML (instead of just a few special + cases like most smartypant processors). For example, `4/5` + becomes `4⁄5`, which renders as + 4⁄5. + +- **MathJaX Support** is an additional feature which is supported by + many markdown editor. It translate inline math equation quoted by `$` + and display math block quoted by `$$` into MathJax compatible format. + hyphen `_` won't break LaTeX render within a math element any more. + + ``` + $$ + \left[ \begin{array}{a} a^l_1 \\ ⋮ \\ a^l_{d_l} \end{array}\right] + = \sigma( + \left[ \begin{matrix} + w^l_{1,1} & ⋯ & w^l_{1,d_{l-1}} \\ + ⋮ & ⋱ & ⋮ \\ + w^l_{d_l,1} & ⋯ & w^l_{d_l,d_{l-1}} \\ + \end{matrix}\right] · + \left[ \begin{array}{x} a^{l-1}_1 \\ ⋮ \\ ⋮ \\ a^{l-1}_{d_{l-1}} \end{array}\right] + + \left[ \begin{array}{b} b^l_1 \\ ⋮ \\ b^l_{d_l} \end{array}\right]) + $$ + ``` + +- **Ordered list start number**. With this extension enabled an ordered list will start with the + the number that was used to start it. + +- **Super and subscript**. With this extension enabled sequences between ^ will indicate + superscript and ~ will become a subscript. For example: H~2~O is a liquid, 2^10^ is 1024. + +- **Block level attributes**, allow setting attributes (ID, classes and key/value pairs) on block + level elements. The attribute must be enclosed with braces and be put on a line before the + element. + + ``` + {#id3 .myclass fontsize="tiny"} + # Header 1 + ``` + + Will convert into `

Header 1

`. + +- **Mmark support**, see for all new syntax elements this adds. + +## Todo + +- port https://github.com/russross/blackfriday/issues/348 +- port [LaTeX output](https://github.com/Ambrevar/Blackfriday-LaTeX): + renders output as LaTeX. +- port https://github.com/shurcooL/github_flavored_markdown to markdown +- port [markdownfmt](https://github.com/shurcooL/markdownfmt): like gofmt, + but for markdown. +- More unit testing +- Improve unicode support. It does not understand all unicode + rules (about what constitutes a letter, a punctuation symbol, + etc.), so it may fail to detect word boundaries correctly in + some instances. It is safe on all utf-8 input. + +## History + +markdown is a fork of v2 of https://github.com/russross/blackfriday that is: + +- actively maintained (sadly in Feb 2018 blackfriday was inactive for 5 months with many bugs and pull requests accumulated) +- refactored API (split into ast/parser/html sub-packages) + +Blackfriday itself was based on C implementation [sundown](https://github.com/vmg/sundown) which in turn was based on [libsoldout](http://fossil.instinctive.eu/libsoldout/home). + +## License + +[Simplified BSD License](LICENSE.txt) diff --git a/vendor/github.com/gomarkdown/markdown/ast/attribute.go b/vendor/github.com/gomarkdown/markdown/ast/attribute.go new file mode 100644 index 000000000..002c6a2ec --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/ast/attribute.go @@ -0,0 +1,10 @@ +package ast + +// An attribute can be attached to block elements. They are specified as +// {#id .classs key="value"} where quotes for values are mandatory, multiple +// key/value pairs are separated by whitespace. +type Attribute struct { + ID []byte + Classes [][]byte + Attrs map[string][]byte +} diff --git a/vendor/github.com/gomarkdown/markdown/ast/doc.go b/vendor/github.com/gomarkdown/markdown/ast/doc.go new file mode 100644 index 000000000..376dc67cc --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/ast/doc.go @@ -0,0 +1,4 @@ +/* +Package ast defines tree representation of a parsed markdown document. +*/ +package ast diff --git a/vendor/github.com/gomarkdown/markdown/ast/node.go b/vendor/github.com/gomarkdown/markdown/ast/node.go new file mode 100644 index 000000000..a6f0327c7 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/ast/node.go @@ -0,0 +1,731 @@ +package ast + +import "encoding/json" + +// ListType contains bitwise or'ed flags for list and list item objects. +type ListType int + +// These are the possible flag values for the ListItem renderer. +// Multiple flag values may be ORed together. +// These are mostly of interest if you are writing a new output format. +const ( + ListTypeOrdered ListType = 1 << iota + ListTypeDefinition + ListTypeTerm + + ListItemContainsBlock + ListItemBeginningOfList // TODO: figure out if this is of any use now + ListItemEndOfList +) + +// CellAlignFlags holds a type of alignment in a table cell. +type CellAlignFlags int + +// These are the possible flag values for the table cell renderer. +// Only a single one of these values will be used; they are not ORed together. +// These are mostly of interest if you are writing a new output format. +const ( + TableAlignmentLeft CellAlignFlags = 1 << iota + TableAlignmentRight + TableAlignmentCenter = (TableAlignmentLeft | TableAlignmentRight) +) + +func (a CellAlignFlags) String() string { + switch a { + case TableAlignmentLeft: + return "left" + case TableAlignmentRight: + return "right" + case TableAlignmentCenter: + return "center" + default: + return "" + } +} + +// DocumentMatters holds the type of a {front,main,back}matter in the document +type DocumentMatters int + +// These are all possible Document divisions. +const ( + DocumentMatterNone DocumentMatters = iota + DocumentMatterFront + DocumentMatterMain + DocumentMatterBack +) + +// CitationTypes holds the type of a citation, informative, normative or suppressed +type CitationTypes int + +const ( + CitationTypeNone CitationTypes = iota + CitationTypeSuppressed + CitationTypeInformative + CitationTypeNormative +) + +// Node defines an ast node +type Node interface { + AsContainer() *Container + AsLeaf() *Leaf + GetParent() Node + SetParent(newParent Node) + GetChildren() []Node + SetChildren(newChildren []Node) +} + +// Container is a type of node that can contain children +type Container struct { + Parent Node `json:"-"` + Children []Node + + Literal []byte // Text contents of the leaf nodes + Content []byte // Markdown content of the block nodes + + *Attribute // Block level attribute +} + +func (c *Container) MarshalJSON() ([]byte, error) { + type ContainerJSON struct { + Children []Node `json:"children"` + Literal string `json:"literal"` + *Attribute + } + var c1 ContainerJSON + c1.Children = c.Children + c1.Literal = string(c.Literal) + c1.Attribute = c.Attribute + + return json.Marshal(&c1) + +} + +// AsContainer returns itself as *Container +func (c *Container) AsContainer() *Container { + return c +} + +// AsLeaf returns nil +func (c *Container) AsLeaf() *Leaf { + return nil +} + +// GetParent returns parent node +func (c *Container) GetParent() Node { + return c.Parent +} + +// SetParent sets the parent node +func (c *Container) SetParent(newParent Node) { + c.Parent = newParent +} + +// GetChildren returns children nodes +func (c *Container) GetChildren() []Node { + return c.Children +} + +// SetChildren sets children node +func (c *Container) SetChildren(newChildren []Node) { + c.Children = newChildren +} + +// Leaf is a type of node that cannot have children +type Leaf struct { + Parent Node `json:"-"` + + Literal []byte // Text contents of the leaf nodes + Content []byte // Markdown content of the block nodes + + *Attribute // Block level attribute +} + +func (c *Leaf) MarshalJSON() ([]byte, error) { + type LeafJSON struct { + Literal string `json:"literal"` + } + var c1 LeafJSON + c1.Literal = string(c.Literal) + + return json.Marshal(&c1) +} + +// AsContainer returns nil +func (l *Leaf) AsContainer() *Container { + return nil +} + +// AsLeaf returns itself as *Leaf +func (l *Leaf) AsLeaf() *Leaf { + return l +} + +// GetParent returns parent node +func (l *Leaf) GetParent() Node { + return l.Parent +} + +// SetParent sets the parent nodd +func (l *Leaf) SetParent(newParent Node) { + l.Parent = newParent +} + +// GetChildren returns nil because Leaf cannot have children +func (l *Leaf) GetChildren() []Node { + return nil +} + +// SetChildren will panic becuase Leaf cannot have children +func (l *Leaf) SetChildren(newChildren []Node) { + panic("leaf node cannot have children") +} + +// Document represents markdown document node, a root of ast +type Document struct { + Container +} + +func (doc *Document) MarshalJSON() ([]byte, error) { + children := doc.GetChildren() + if len(children) != 0 { + return json.Marshal(children) + } + return []byte("[]"), nil +} + +// DocumentMatter represents markdown node that signals a document +// division: frontmatter, mainmatter or backmatter. +type DocumentMatter struct { + Container + + Matter DocumentMatters +} + +// BlockQuote represents markdown block quote node +type BlockQuote struct { + Container +} + +func (c *BlockQuote) MarshalJSON() ([]byte, error) { + type BlockQuoteJSON struct { + Type string `json:"type"` + Children []Node `json:"children"` + Literal string `json:"literal"` + *Attribute + } + var c1 BlockQuoteJSON + c1.Children = c.Children + c1.Literal = string(c.Literal) + c1.Type = "blockquote" + + return json.Marshal(&c1) +} + +// Aside represents an markdown aside node. +type Aside struct { + Container +} + +// List represents markdown list node +type List struct { + Container + + ListFlags ListType + Tight bool // Skip

s around list item data if true + BulletChar byte // '*', '+' or '-' in bullet lists + Delimiter byte // '.' or ')' after the number in ordered lists + Start int // for ordered lists this indicates the starting number if > 0 + RefLink []byte // If not nil, turns this list item into a footnote item and triggers different rendering + IsFootnotesList bool // This is a list of footnotes +} + +// ListItem represents markdown list item node +type ListItem struct { + Container + + ListFlags ListType + Tight bool // Skip

s around list item data if true + BulletChar byte // '*', '+' or '-' in bullet lists + Delimiter byte // '.' or ')' after the number in ordered lists + RefLink []byte // If not nil, turns this list item into a footnote item and triggers different rendering + IsFootnotesList bool // This is a list of footnotes +} + +// Paragraph represents markdown paragraph node +type Paragraph struct { + Container +} + +func (c *Paragraph) MarshalJSON() ([]byte, error) { + type ParagraphJSON struct { + Type string `json:"type"` + Children []Node `json:"children"` + } + var c1 ParagraphJSON + c1.Children = c.Children + c1.Type = "paragraph" + return json.Marshal(&c1) +} + +// Math represents markdown MathAjax inline node +type Math struct { + Leaf +} + +// MathBlock represents markdown MathAjax block node +type MathBlock struct { + Container +} + +// Heading represents markdown heading node +type Heading struct { + Container + + Level int // This holds the heading level number + HeadingID string // This might hold heading ID, if present + IsTitleblock bool // Specifies whether it's a title block + IsSpecial bool // We are a special heading (starts with .#) +} + +func (c *Heading) MarshalJSON() ([]byte, error) { + type HeadingJSON struct { + Type string `json:"type"` + Children []Node `json:"children"` + Literal string `json:"literal"` + Level int `json:"level"` + IsTitleblock bool `json:"isTitleBlock"` + + *Attribute + } + var c1 HeadingJSON + c1.Children = c.Children + c1.Literal = string(c.Literal) + c1.Attribute = c.Attribute + c1.Type = "heading" + c1.Level = c.Level + c1.IsTitleblock = c.IsTitleblock + + return json.Marshal(&c1) + +} + +// HorizontalRule represents markdown horizontal rule node +type HorizontalRule struct { + Leaf +} + +// Emph represents markdown emphasis node +type Emph struct { + Leaf +} + +func (c *Emph) MarshalJSON() ([]byte, error) { + type EmphJSON struct { + Type string `json:"type"` + Literal string `json:"literal"` + *Attribute + } + var c1 EmphJSON + c1.Literal = string(c.Literal) + c1.Attribute = c.Attribute + c1.Type = "emph" + + return json.Marshal(&c1) +} + +type StatusTag struct { + Leaf +} + +func (c *StatusTag) MarshalJSON() ([]byte, error) { + type StatusTagJSON struct { + Type string `json:"type"` + Literal string `json:"literal"` + } + var c1 StatusTagJSON + c1.Literal = string(c.Literal) + c1.Type = "status-tag" + return json.Marshal(&c1) +} + +// Strong represents markdown strong node +type Strong struct { + Leaf +} + +func (c *Strong) MarshalJSON() ([]byte, error) { + type StrongJSON struct { + Type string `json:"type"` + Literal string `json:"literal"` + *Attribute + } + var c1 StrongJSON + c1.Literal = string(c.Literal) + c1.Attribute = c.Attribute + c1.Type = "strong" + + return json.Marshal(&c1) +} + +// Del represents markdown del node +type Del struct { + Leaf +} + +// Link represents markdown link node +type Link struct { + Container + + Destination []byte // Destination is what goes into a href + Title []byte // Title is the tooltip thing that goes in a title attribute + NoteID int // NoteID contains a serial number of a footnote, zero if it's not a footnote + Footnote Node // If it's a footnote, this is a direct link to the footnote Node. Otherwise nil. + DeferredID []byte // If a deferred link this holds the original ID. +} + +func (c *Link) MarshalJSON() ([]byte, error) { + type LinkJSON struct { + Type string `json:"type"` + Children []Node `json:"children"` + Literal string `json:"literal"` + Title string `json:"title"` + Destination string `json:"destination"` + *Attribute + } + var c1 LinkJSON + c1.Children = c.Children + c1.Literal = string(c.Literal) + c1.Attribute = c.Attribute + c1.Title = string(c.Title) + c1.Destination = string(c.Destination) + c1.Type = "link" + + return json.Marshal(&c1) +} + +// CrossReference is a reference node. +type CrossReference struct { + Container + + Destination []byte // Destination is where the reference points to +} + +// Citation is a citation node. +type Citation struct { + Leaf + + Destination [][]byte // Destination is where the citation points to. Multiple ones are allowed. + Type []CitationTypes // 1:1 mapping of destination and citation type + Suffix [][]byte // Potential citation suffix, i.e. [@!RFC1035, p. 144] +} + +// Image represents markdown image node +type Image struct { + Container + + Destination []byte // Destination is what goes into a href + Title []byte // Title is the tooltip thing that goes in a title attribute +} + +// Text represents markdown text node +type Text struct { + Leaf +} + +// HTMLBlock represents markdown html node +type HTMLBlock struct { + Leaf +} + +// CodeBlock represents markdown code block node +type CodeBlock struct { + Leaf + + IsFenced bool // Specifies whether it's a fenced code block or an indented one + Info []byte // This holds the info string + FenceChar byte + FenceLength int + FenceOffset int +} + +func (c *CodeBlock) MarshalJSON() ([]byte, error) { + type CodeBlockJSON struct { + Type string `json:"type"` + Literal string `json:"literal"` + *Attribute + } + var c1 CodeBlockJSON + c1.Literal = string(c.Literal) + c1.Type = "codeblock" + c1.Attribute = c.Attribute + + return json.Marshal(&c1) +} + +// Softbreak represents markdown softbreak node +// Note: not used currently +type Softbreak struct { + Leaf +} + +// Hardbreak represents markdown hard break node +type Hardbreak struct { + Leaf +} + +// NonBlockingSpace represents markdown non-blocking space node +type NonBlockingSpace struct { + Leaf +} + +// Code represents markdown code node +type Code struct { + Leaf +} + +func (c *Code) MarshalJSON() ([]byte, error) { + type CodeJSON struct { + Type string `json:"type"` + Literal string `json:"literal"` + } + var c1 CodeJSON + c1.Literal = string(c.Literal) + c1.Type = "code" + + return json.Marshal(&c1) +} + +// HTMLSpan represents markdown html span node +type HTMLSpan struct { + Leaf +} + +// Table represents markdown table node +type Table struct { + Container +} + +// TableCell represents markdown table cell node +type TableCell struct { + Container + + IsHeader bool // This tells if it's under the header row + Align CellAlignFlags // This holds the value for align attribute +} + +// TableHeader represents markdown table head node +type TableHeader struct { + Container +} + +// TableBody represents markdown table body node +type TableBody struct { + Container +} + +// TableRow represents markdown table row node +type TableRow struct { + Container +} + +// TableFooter represents markdown table foot node +type TableFooter struct { + Container +} + +// Caption represents a figure, code or quote caption +type Caption struct { + Container +} + +// CaptionFigure is a node (blockquote or codeblock) that has a caption +type CaptionFigure struct { + Container + + HeadingID string // This might hold heading ID, if present +} + +// Callout is a node that can exist both in text (where it is an actual node) and in a code block. +type Callout struct { + Leaf + + ID []byte // number of this callout +} + +// Index is a node that contains an Index item and an optional, subitem. +type Index struct { + Leaf + + Primary bool + Item []byte + Subitem []byte + ID string // ID of the index +} + +// Subscript is a subscript node +type Subscript struct { + Leaf +} + +// Subscript is a superscript node +type Superscript struct { + Leaf +} + +// Footnotes is a node that contains all footnotes +type Footnotes struct { + Container +} + +func removeNodeFromArray(a []Node, node Node) []Node { + n := len(a) + for i := 0; i < n; i++ { + if a[i] == node { + return append(a[:i], a[i+1:]...) + } + } + return nil +} + +// AppendChild appends child to children of parent +// It panics if either node is nil. +func AppendChild(parent Node, child Node) { + RemoveFromTree(child) + child.SetParent(parent) + newChildren := append(parent.GetChildren(), child) + parent.SetChildren(newChildren) +} + +// RemoveFromTree removes this node from tree +func RemoveFromTree(n Node) { + if n.GetParent() == nil { + return + } + // important: don't clear n.Children if n has no parent + // we're called from AppendChild and that might happen on a node + // that accumulated Children but hasn't been inserted into the tree + n.SetChildren(nil) + p := n.GetParent() + newChildren := removeNodeFromArray(p.GetChildren(), n) + if newChildren != nil { + p.SetChildren(newChildren) + } +} + +// GetLastChild returns last child of node n +// It's implemented as stand-alone function to keep Node interface small +func GetLastChild(n Node) Node { + a := n.GetChildren() + if len(a) > 0 { + return a[len(a)-1] + } + return nil +} + +// GetFirstChild returns first child of node n +// It's implemented as stand-alone function to keep Node interface small +func GetFirstChild(n Node) Node { + a := n.GetChildren() + if len(a) > 0 { + return a[0] + } + return nil +} + +// GetNextNode returns next sibling of node n (node after n) +// We can't make it part of Container or Leaf because we loose Node identity +func GetNextNode(n Node) Node { + parent := n.GetParent() + if parent == nil { + return nil + } + a := parent.GetChildren() + len := len(a) - 1 + for i := 0; i < len; i++ { + if a[i] == n { + return a[i+1] + } + } + return nil +} + +// GetPrevNode returns previous sibling of node n (node before n) +// We can't make it part of Container or Leaf because we loose Node identity +func GetPrevNode(n Node) Node { + parent := n.GetParent() + if parent == nil { + return nil + } + a := parent.GetChildren() + len := len(a) + for i := 1; i < len; i++ { + if a[i] == n { + return a[i-1] + } + } + return nil +} + +// WalkStatus allows NodeVisitor to have some control over the tree traversal. +// It is returned from NodeVisitor and different values allow Node.Walk to +// decide which node to go to next. +type WalkStatus int + +const ( + // GoToNext is the default traversal of every node. + GoToNext WalkStatus = iota + // SkipChildren tells walker to skip all children of current node. + SkipChildren + // Terminate tells walker to terminate the traversal. + Terminate +) + +// NodeVisitor is a callback to be called when traversing the syntax tree. +// Called twice for every node: once with entering=true when the branch is +// first visited, then with entering=false after all the children are done. +type NodeVisitor interface { + Visit(node Node, entering bool) WalkStatus +} + +// NodeVisitorFunc casts a function to match NodeVisitor interface +type NodeVisitorFunc func(node Node, entering bool) WalkStatus + +// Walk traverses tree recursively +func Walk(n Node, visitor NodeVisitor) WalkStatus { + isContainer := n.AsContainer() != nil + status := visitor.Visit(n, true) // entering + if status == Terminate { + // even if terminating, close container node + if isContainer { + visitor.Visit(n, false) + } + return status + } + if isContainer && status != SkipChildren { + children := n.GetChildren() + for _, n := range children { + status = Walk(n, visitor) + if status == Terminate { + return status + } + } + } + if isContainer { + status = visitor.Visit(n, false) // exiting + if status == Terminate { + return status + } + } + return GoToNext +} + +// Visit calls visitor function +func (f NodeVisitorFunc) Visit(node Node, entering bool) WalkStatus { + return f(node, entering) +} + +// WalkFunc is like Walk but accepts just a callback function +func WalkFunc(n Node, f NodeVisitorFunc) { + visitor := NodeVisitorFunc(f) + Walk(n, visitor) +} diff --git a/vendor/github.com/gomarkdown/markdown/ast/print.go b/vendor/github.com/gomarkdown/markdown/ast/print.go new file mode 100644 index 000000000..35ea0c758 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/ast/print.go @@ -0,0 +1,158 @@ +package ast + +import ( + "bytes" + "fmt" + "io" + "strings" +) + +// Print is for debugging. It prints a string representation of parsed +// markdown doc (result of parser.Parse()) to dst. +// +// To make output readable, it shortens text output. +func Print(dst io.Writer, doc Node) { + PrintWithPrefix(dst, doc, " ") +} + +// PrintWithPrefix is like Print but allows customizing prefix used for +// indentation. By default it's 2 spaces. You can change it to e.g. tab +// by passing "\t" +func PrintWithPrefix(w io.Writer, doc Node, prefix string) { + // for more compact output, don't print outer Document + if _, ok := doc.(*Document); ok { + for _, c := range doc.GetChildren() { + printRecur(w, c, prefix, 0) + } + } else { + printRecur(w, doc, prefix, 0) + } +} + +// ToString is like Dump but returns result as a string +func ToString(doc Node) string { + var buf bytes.Buffer + Print(&buf, doc) + return buf.String() +} + +func getContent(node Node) string { + if c := node.AsContainer(); c != nil { + return string(c.Literal) + } + leaf := node.AsLeaf() + return string(leaf.Literal) +} + +func shortenString(s string, maxLen int) string { + // for cleaner, one-line ouput, replace some white-space chars + // with their escaped version + s = strings.Replace(s, "\n", `\n`, -1) + s = strings.Replace(s, "\r", `\r`, -1) + s = strings.Replace(s, "\t", `\t`, -1) + if maxLen < 0 { + return s + } + if len(s) < maxLen { + return s + } + // add "..." to indicate truncation + return s[:maxLen-3] + "..." +} + +// get a short name of the type of v which excludes package name +// and strips "()" from the end +func getNodeType(node Node) string { + s := fmt.Sprintf("%T", node) + s = strings.TrimSuffix(s, "()") + if idx := strings.Index(s, "."); idx != -1 { + return s[idx+1:] + } + return s +} + +func printDefault(w io.Writer, indent string, typeName string, content string) { + content = strings.TrimSpace(content) + if len(content) > 0 { + fmt.Fprintf(w, "%s%s '%s'\n", indent, typeName, content) + } else { + fmt.Fprintf(w, "%s%s\n", indent, typeName) + } +} + +func getListFlags(f ListType) string { + var s string + if f&ListTypeOrdered != 0 { + s += "ordered " + } + if f&ListTypeDefinition != 0 { + s += "definition " + } + if f&ListTypeTerm != 0 { + s += "term " + } + if f&ListItemContainsBlock != 0 { + s += "has_block " + } + if f&ListItemBeginningOfList != 0 { + s += "start " + } + if f&ListItemEndOfList != 0 { + s += "end " + } + s = strings.TrimSpace(s) + return s +} + +func printRecur(w io.Writer, node Node, prefix string, depth int) { + if node == nil { + return + } + indent := strings.Repeat(prefix, depth) + + content := shortenString(getContent(node), 40) + typeName := getNodeType(node) + switch v := node.(type) { + case *Link: + content := "url=" + string(v.Destination) + printDefault(w, indent, typeName, content) + case *StatusTag: + content := "tag=" + string(v.Literal) + printDefault(w, indent, typeName, content) + case *Image: + content := "url=" + string(v.Destination) + printDefault(w, indent, typeName, content) + case *List: + if v.Start > 1 { + content += fmt.Sprintf("start=%d ", v.Start) + } + if v.Tight { + content += "tight " + } + if v.IsFootnotesList { + content += "footnotes " + } + flags := getListFlags(v.ListFlags) + if len(flags) > 0 { + content += "flags=" + flags + " " + } + printDefault(w, indent, typeName, content) + case *ListItem: + if v.Tight { + content += "tight " + } + if v.IsFootnotesList { + content += "footnotes " + } + flags := getListFlags(v.ListFlags) + if len(flags) > 0 { + content += "flags=" + flags + " " + } + printDefault(w, indent, typeName, content) + default: + printDefault(w, indent, typeName, content) + } + for _, child := range node.GetChildren() { + printRecur(w, child, prefix, depth+1) + } +} diff --git a/vendor/github.com/gomarkdown/markdown/changes-from-blackfriday.md b/vendor/github.com/gomarkdown/markdown/changes-from-blackfriday.md new file mode 100644 index 000000000..b618dfefc --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/changes-from-blackfriday.md @@ -0,0 +1,27 @@ +## Changes from blackfriday + +This library is derived from blackfriday library. Here's a list of changes. + +**Redesigned API** + +- split into 3 separate packages: ast, parser and html (for html renderer). This makes the API more manageable. It also separates e.g. parser option from renderer options +- changed how AST node is represented from union-like representation (manually keeping track of the type of the node) to using interface{} (which is a Go way to combine an arbitrary value with its type) + +**Allow re-using most of html renderer logic** + +You can implement your own renderer by implementing `Renderer` interface. + +Implementing a full renderer is a lot of work and often you just want to tweak html rendering of few node typs. + +I've added a way to hook `Renderer.Render` function in html renderer with a custom function that can take over rendering of specific nodes. + +I use it myself to do syntax-highlighting of code snippets. + +**Speed up go test** + +Running `go test` was really slow (17 secs) because it did a poor man's version of fuzzing by feeding the parser all subsets of test strings in order to find panics +due to incorrect parsing logic. + +I've moved that logic to `cmd/crashtest`, so that it can be run on CI but not slow down regular development. + +Now `go test` is blazing fast. diff --git a/vendor/github.com/gomarkdown/markdown/codecov.yml b/vendor/github.com/gomarkdown/markdown/codecov.yml new file mode 100644 index 000000000..f681ff11d --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + # basic + target: 60% + threshold: 2% + base: auto diff --git a/vendor/github.com/gomarkdown/markdown/doc.go b/vendor/github.com/gomarkdown/markdown/doc.go new file mode 100644 index 000000000..b623024d9 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/doc.go @@ -0,0 +1,35 @@ +/* +Package markdown implements markdown parser and HTML renderer. + +It parses markdown into AST format which can be serialized to HTML +(using html.Renderer) or possibly other formats (using alternate renderers). + +Convert markdown to HTML + +The simplest way to convert markdown document to HTML + + md := []byte("## markdown document") + html := markdown.ToHTML(md, nil, nil) + +Customizing parsing and HTML rendering + +You can customize parser and HTML renderer: + + import ( + "github.com/gomarkdown/markdown/parser" + "github.com/gomarkdown/markdown/renderer" + "github.com/gomarkdown/markdown" + ) + extensions := parser.CommonExtensions | parser.AutoHeadingIDs + p := parser.NewWithExtensions(extensions) + + htmlFlags := html.CommonFlags | html.HrefTargetBlank + opts := html.RendererOptions{Flags: htmlFlags} + renderer := html.NewRenderer(opts) + + md := []byte("markdown text") + html := markdown.ToHTML(md, p, renderer) + +For a cmd-line tool see https://github.com/gomarkdown/mdtohtml +*/ +package markdown diff --git a/vendor/github.com/gomarkdown/markdown/fuzz.go b/vendor/github.com/gomarkdown/markdown/fuzz.go new file mode 100644 index 000000000..704182b8f --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/fuzz.go @@ -0,0 +1,9 @@ +// +build gofuzz + +package markdown + +// Fuzz is to be used by https://github.com/dvyukov/go-fuzz +func Fuzz(data []byte) int { + Parse(data, nil) + return 0 +} diff --git a/vendor/github.com/gomarkdown/markdown/go.mod b/vendor/github.com/gomarkdown/markdown/go.mod new file mode 100644 index 000000000..09b133b8b --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/go.mod @@ -0,0 +1,11 @@ +module github.com/gomarkdown/markdown + +go 1.12 + +require ( + github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681 // indirect + github.com/elazarl/go-bindata-assetfs v1.0.0 // indirect + github.com/stephens2424/writerset v1.0.2 // indirect + golang.org/dl v0.0.0-20190829154251-82a15e2f2ead // indirect + golang.org/x/tools v0.0.0-20191109212701-97ad0ed33101 // indirect +) diff --git a/vendor/github.com/gomarkdown/markdown/go.sum b/vendor/github.com/gomarkdown/markdown/go.sum new file mode 100644 index 000000000..ba7840eeb --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/go.sum @@ -0,0 +1,21 @@ +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681 h1:3WV5aRRj1ELP3RcLlBp/v0WJTuy47OQMkL9GIQq8QEE= +github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/stephens2424/writerset v1.0.2 h1:znRLgU6g8RS5euYRcy004XeE4W+Tu44kALzy7ghPif8= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191109212701-97ad0ed33101 h1:LCmXVkvpQCDj724eX6irUTPCJP5GelFHxqGSWL2D1R0= +golang.org/x/tools v0.0.0-20191109212701-97ad0ed33101/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vendor/github.com/gomarkdown/markdown/markdown.go b/vendor/github.com/gomarkdown/markdown/markdown.go new file mode 100644 index 000000000..29c23a485 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/markdown.go @@ -0,0 +1,66 @@ +package markdown + +import ( + "bytes" + "io" + + "github.com/gomarkdown/markdown/ast" + "github.com/gomarkdown/markdown/parser" +) + +// Renderer is an interface for implementing custom renderers. +type Renderer interface { + // RenderNode renders markdown node to w. + // It's called once for a leaf node. + // It's called twice for non-leaf nodes: + // * first with entering=true + // * then with entering=false + // + // Return value is a way to tell the calling walker to adjust its walk + // pattern: e.g. it can terminate the traversal by returning Terminate. Or it + // can ask the walker to skip a subtree of this node by returning SkipChildren. + // The typical behavior is to return GoToNext, which asks for the usual + // traversal to the next node. + RenderNode(w io.Writer, node ast.Node, entering bool) ast.WalkStatus + + // RenderHeader is a method that allows the renderer to produce some + // content preceding the main body of the output document. The header is + // understood in the broad sense here. For example, the default HTML + // renderer will write not only the HTML document preamble, but also the + // table of contents if it was requested. + // + // The method will be passed an entire document tree, in case a particular + // implementation needs to inspect it to produce output. + // + // The output should be written to the supplied writer w. If your + // implementation has no header to write, supply an empty implementation. + RenderHeader(w io.Writer, ast ast.Node) + + // RenderFooter is a symmetric counterpart of RenderHeader. + RenderFooter(w io.Writer, ast ast.Node) +} + +// Parse parsers a markdown document using provided parser. If parser is nil, +// we use parser configured with parser.CommonExtensions. +// +// It returns AST (abstract syntax tree) that can be converted to another +// format using Render function. +func Parse(markdown []byte, p *parser.Parser) ast.Node { + if p == nil { + p = parser.New() + } + return p.Parse(markdown) +} + +// Render uses renderer to convert parsed markdown document into a different format. +// +// To convert to HTML, pass html.Renderer +func Render(doc ast.Node, renderer Renderer) []byte { + var buf bytes.Buffer + renderer.RenderHeader(&buf, doc) + ast.WalkFunc(doc, func(node ast.Node, entering bool) ast.WalkStatus { + return renderer.RenderNode(&buf, node, entering) + }) + renderer.RenderFooter(&buf, doc) + return buf.Bytes() +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/aside.go b/vendor/github.com/gomarkdown/markdown/parser/aside.go new file mode 100644 index 000000000..96e25fe06 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/aside.go @@ -0,0 +1,73 @@ +package parser + +import ( + "bytes" + + "github.com/gomarkdown/markdown/ast" +) + +// returns aisde prefix length +func (p *Parser) asidePrefix(data []byte) int { + i := 0 + n := len(data) + for i < 3 && i < n && data[i] == ' ' { + i++ + } + if i+1 < n && data[i] == 'A' && data[i+1] == '>' { + if i+2 < n && data[i+2] == ' ' { + return i + 3 + } + return i + 2 + } + return 0 +} + +// aside ends with at least one blank line +// followed by something without a aside prefix +func (p *Parser) terminateAside(data []byte, beg, end int) bool { + if p.isEmpty(data[beg:]) <= 0 { + return false + } + if end >= len(data) { + return true + } + return p.asidePrefix(data[end:]) == 0 && p.isEmpty(data[end:]) == 0 +} + +// parse a aside fragment +func (p *Parser) aside(data []byte) int { + var raw bytes.Buffer + beg, end := 0, 0 + // identical to quote + for beg < len(data) { + end = beg + // Step over whole lines, collecting them. While doing that, check for + // fenced code and if one's found, incorporate it altogether, + // irregardless of any contents inside it + for end < len(data) && data[end] != '\n' { + if p.extensions&FencedCode != 0 { + if i := p.fencedCodeBlock(data[end:], false); i > 0 { + // -1 to compensate for the extra end++ after the loop: + end += i - 1 + break + } + } + end++ + } + end = skipCharN(data, end, '\n', 1) + if pre := p.asidePrefix(data[beg:]); pre > 0 { + // skip the prefix + beg += pre + } else if p.terminateAside(data, beg, end) { + break + } + // this line is part of the aside + raw.Write(data[beg:end]) + beg = end + } + + block := p.addBlock(&ast.Aside{}) + p.block(raw.Bytes()) + p.finalize(block) + return end +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/attribute.go b/vendor/github.com/gomarkdown/markdown/parser/attribute.go new file mode 100644 index 000000000..5fdb07095 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/attribute.go @@ -0,0 +1,116 @@ +package parser + +import ( + "bytes" + + "github.com/gomarkdown/markdown/ast" +) + +// attribute parses a (potential) block attribute and adds it to p. +func (p *Parser) attribute(data []byte) []byte { + if len(data) < 3 { + return data + } + i := 0 + if data[i] != '{' { + return data + } + i++ + + // last character must be a } otherwise it's not an attribute + end := skipUntilChar(data, i, '\n') + if data[end-1] != '}' { + return data + } + + i = skipSpace(data, i) + b := &ast.Attribute{Attrs: make(map[string][]byte)} + + esc := false + quote := false + trail := 0 +Loop: + for ; i < len(data); i++ { + switch data[i] { + case ' ', '\t', '\f', '\v': + if quote { + continue + } + chunk := data[trail+1 : i] + if len(chunk) == 0 { + trail = i + continue + } + switch { + case chunk[0] == '.': + b.Classes = append(b.Classes, chunk[1:]) + case chunk[0] == '#': + b.ID = chunk[1:] + default: + k, v := keyValue(chunk) + if k != nil && v != nil { + b.Attrs[string(k)] = v + } else { + // this is illegal in an attribute + return data + } + } + trail = i + case '"': + if esc { + esc = !esc + continue + } + quote = !quote + case '\\': + esc = !esc + case '}': + if esc { + esc = !esc + continue + } + chunk := data[trail+1 : i] + if len(chunk) == 0 { + return data + } + switch { + case chunk[0] == '.': + b.Classes = append(b.Classes, chunk[1:]) + case chunk[0] == '#': + b.ID = chunk[1:] + default: + k, v := keyValue(chunk) + if k != nil && v != nil { + b.Attrs[string(k)] = v + } else { + return data + } + } + i++ + break Loop + default: + esc = false + } + } + + p.attr = b + return data[i:] +} + +// key="value" quotes are mandatory. +func keyValue(data []byte) ([]byte, []byte) { + chunk := bytes.SplitN(data, []byte{'='}, 2) + if len(chunk) != 2 { + return nil, nil + } + key := chunk[0] + value := chunk[1] + + if len(value) < 3 || len(key) == 0 { + return nil, nil + } + if value[0] != '"' || value[len(value)-1] != '"' { + return key, nil + } + return key, value[1 : len(value)-1] +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/block.go b/vendor/github.com/gomarkdown/markdown/parser/block.go new file mode 100644 index 000000000..86c43c3b3 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/block.go @@ -0,0 +1,1415 @@ +package parser + +import ( + "bytes" + "html" + "regexp" + "unicode" + + "github.com/gomarkdown/markdown/ast" +) + +// Parsing block-level elements. + +const ( + charEntity = "&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});" + escapable = "[!\"#$%&'()*+,./:;<=>?@[\\\\\\]^_`{|}~-]" +) + +var ( + reBackslashOrAmp = regexp.MustCompile("[\\&]") + reEntityOrEscapedChar = regexp.MustCompile("(?i)\\\\" + escapable + "|" + charEntity) + + // blockTags is a set of tags that are recognized as HTML block tags. + // Any of these can be included in markdown text without special escaping. + blockTags = map[string]struct{}{ + "blockquote": struct{}{}, + "del": struct{}{}, + "div": struct{}{}, + "dl": struct{}{}, + "fieldset": struct{}{}, + "form": struct{}{}, + "h1": struct{}{}, + "h2": struct{}{}, + "h3": struct{}{}, + "h4": struct{}{}, + "h5": struct{}{}, + "h6": struct{}{}, + "iframe": struct{}{}, + "ins": struct{}{}, + "math": struct{}{}, + "noscript": struct{}{}, + "ol": struct{}{}, + "pre": struct{}{}, + "p": struct{}{}, + "script": struct{}{}, + "style": struct{}{}, + "table": struct{}{}, + "ul": struct{}{}, + + // HTML5 + "address": struct{}{}, + "article": struct{}{}, + "aside": struct{}{}, + "canvas": struct{}{}, + "figcaption": struct{}{}, + "figure": struct{}{}, + "footer": struct{}{}, + "header": struct{}{}, + "hgroup": struct{}{}, + "main": struct{}{}, + "nav": struct{}{}, + "output": struct{}{}, + "progress": struct{}{}, + "section": struct{}{}, + "video": struct{}{}, + } +) + +// sanitizeAnchorName returns a sanitized anchor name for the given text. +// Taken from https://github.com/shurcooL/sanitized_anchor_name/blob/master/main.go#L14:1 +func sanitizeAnchorName(text string) string { + var anchorName []rune + var futureDash = false + for _, r := range text { + switch { + case unicode.IsLetter(r) || unicode.IsNumber(r): + if futureDash && len(anchorName) > 0 { + anchorName = append(anchorName, '-') + } + futureDash = false + anchorName = append(anchorName, unicode.ToLower(r)) + default: + futureDash = true + } + } + return string(anchorName) +} + +// Parse block-level data. +// Note: this function and many that it calls assume that +// the input buffer ends with a newline. +func (p *Parser) block(data []byte) { + // this is called recursively: enforce a maximum depth + for len(data) > 0 { + if i := p.fencedCodeBlock(data, true); i > 0 { + data = data[i:] + continue + } + + if p.quotePrefix(data) > 0 { + data = data[p.quote(data):] + continue + } + + idx := p.paragraph(data) + data = data[idx:] + } + //p.renderParagraph(data) + return + +} + +func (p *Parser) addBlock(n ast.Node) ast.Node { + p.closeUnmatchedBlocks() + + if p.attr != nil { + if c := n.AsContainer(); c != nil { + c.Attribute = p.attr + } + if l := n.AsLeaf(); l != nil { + l.Attribute = p.attr + } + p.attr = nil + } + return p.addChild(n) +} + +func (p *Parser) isPrefixHeading(data []byte) bool { + return len(data) > 1 && data[0] == '#' && isSpace(data[1]) +} + +func (p *Parser) prefixHeading(data []byte) int { + level := skipCharN(data, 0, '#', 6) + i := skipChar(data, level, ' ') + end := skipUntilChar(data, i, '\n') + skip := end + for end > 0 && data[end-1] == '#' { + if isBackslashEscaped(data, end-1) { + break + } + end-- + } + for end > 0 && data[end-1] == ' ' { + end-- + } + if end > i { + block := &ast.Heading{ + Level: level, + } + block.Literal = data[i:end] + p.addBlock(block) + } + return skip +} + +func (p *Parser) isPrefixSpecialHeading(data []byte) bool { + if len(data) < 4 { + return false + } + if data[0] != '.' { + return false + } + if data[1] != '#' { + return false + } + if data[2] == '#' { // we don't support level, so nack this. + return false + } + + if p.extensions&SpaceHeadings != 0 { + if data[2] != ' ' { + return false + } + } + return true +} + +func (p *Parser) prefixSpecialHeading(data []byte) int { + i := skipChar(data, 2, ' ') // ".#" skipped + end := skipUntilChar(data, i, '\n') + skip := end + id := "" + if p.extensions&HeadingIDs != 0 { + j, k := 0, 0 + // find start/end of heading id + for j = i; j < end-1 && (data[j] != '{' || data[j+1] != '#'); j++ { + } + for k = j + 1; k < end && data[k] != '}'; k++ { + } + // extract heading id iff found + if j < end && k < end { + id = string(data[j+2 : k]) + end = j + skip = k + 1 + for end > 0 && data[end-1] == ' ' { + end-- + } + } + } + for end > 0 && data[end-1] == '#' { + if isBackslashEscaped(data, end-1) { + break + } + end-- + } + for end > 0 && data[end-1] == ' ' { + end-- + } + if end > i { + if id == "" && p.extensions&AutoHeadingIDs != 0 { + id = sanitizeAnchorName(string(data[i:end])) + } + block := &ast.Heading{ + HeadingID: id, + IsSpecial: true, + Level: 1, // always level 1. + } + block.Literal = data[i:end] + block.Content = data[i:end] + p.addBlock(block) + } + return skip +} + +func (p *Parser) isUnderlinedHeading(data []byte) int { + // test of level 1 heading + if data[0] == '=' { + i := skipChar(data, 1, '=') + i = skipChar(data, i, ' ') + if i < len(data) && data[i] == '\n' { + return 1 + } + return 0 + } + + // test of level 2 heading + if data[0] == '-' { + i := skipChar(data, 1, '-') + i = skipChar(data, i, ' ') + if i < len(data) && data[i] == '\n' { + return 2 + } + return 0 + } + + return 0 +} + +func (p *Parser) titleBlock(data []byte, doRender bool) int { + if data[0] != '%' { + return 0 + } + splitData := bytes.Split(data, []byte("\n")) + var i int + for idx, b := range splitData { + if !bytes.HasPrefix(b, []byte("%")) { + i = idx // - 1 + break + } + } + + data = bytes.Join(splitData[0:i], []byte("\n")) + consumed := len(data) + data = bytes.TrimPrefix(data, []byte("% ")) + data = bytes.Replace(data, []byte("\n% "), []byte("\n"), -1) + block := &ast.Heading{ + Level: 1, + IsTitleblock: true, + } + block.Content = data + p.addBlock(block) + + return consumed +} + +func (*Parser) isEmpty(data []byte) int { + // it is okay to call isEmpty on an empty buffer + if len(data) == 0 { + return 0 + } + + var i int + for i = 0; i < len(data) && data[i] != '\n'; i++ { + if data[i] != ' ' && data[i] != '\t' { + return 0 + } + } + i = skipCharN(data, i, '\n', 1) + return i +} + +func (*Parser) isHRule(data []byte) bool { + i := 0 + + // skip up to three spaces + for i < 3 && data[i] == ' ' { + i++ + } + + // look at the hrule char + if data[i] != '*' && data[i] != '-' && data[i] != '_' { + return false + } + c := data[i] + + // the whole line must be the char or whitespace + n := 0 + for i < len(data) && data[i] != '\n' { + switch { + case data[i] == c: + n++ + case data[i] != ' ': + return false + } + i++ + } + + return n >= 3 +} + +// isFenceLine checks if there's a fence line (e.g., ``` or ``` go) at the beginning of data, +// and returns the end index if so, or 0 otherwise. It also returns the marker found. +// If syntax is not nil, it gets set to the syntax specified in the fence line. +func isFenceLine(data []byte, syntax *string, oldmarker string) (end int, marker string) { + i, size := 0, 0 + + n := len(data) + // skip up to three spaces + for i < n && i < 3 && data[i] == ' ' { + i++ + } + + // check for the marker characters: ~ or ` + if i >= n { + return 0, "" + } + if data[i] != '~' && data[i] != '`' { + return 0, "" + } + + c := data[i] + + // the whole line must be the same char or whitespace + for i < n && data[i] == c { + size++ + i++ + } + + // the marker char must occur at least 3 times + if size < 3 { + return 0, "" + } + marker = string(data[i-size : i]) + + // if this is the end marker, it must match the beginning marker + if oldmarker != "" && marker != oldmarker { + return 0, "" + } + + // TODO(shurcooL): It's probably a good idea to simplify the 2 code paths here + // into one, always get the syntax, and discard it if the caller doesn't care. + if syntax != nil { + syn := 0 + i = skipChar(data, i, ' ') + + if i >= n { + if i == n { + return i, marker + } + return 0, "" + } + + syntaxStart := i + + if data[i] == '{' { + i++ + syntaxStart++ + + for i < n && data[i] != '}' && data[i] != '\n' { + syn++ + i++ + } + + if i >= n || data[i] != '}' { + return 0, "" + } + + // strip all whitespace at the beginning and the end + // of the {} block + for syn > 0 && isSpace(data[syntaxStart]) { + syntaxStart++ + syn-- + } + + for syn > 0 && isSpace(data[syntaxStart+syn-1]) { + syn-- + } + + i++ + } else { + for i < n && !isSpace(data[i]) { + syn++ + i++ + } + } + + *syntax = string(data[syntaxStart : syntaxStart+syn]) + } + + i = skipChar(data, i, ' ') + if i >= n || data[i] != '\n' { + if i == n { + return i, marker + } + return 0, "" + } + return i + 1, marker // Take newline into account. +} + +// fencedCodeBlock returns the end index if data contains a fenced code block at the beginning, +// or 0 otherwise. It writes to out if doRender is true, otherwise it has no side effects. +// If doRender is true, a final newline is mandatory to recognize the fenced code block. +func (p *Parser) fencedCodeBlock(data []byte, doRender bool) int { + var syntax string + beg, marker := isFenceLine(data, &syntax, "") + if beg == 0 || beg >= len(data) { + return 0 + } + + var work bytes.Buffer + work.WriteString(syntax) + work.WriteByte('\n') + + for { + // safe to assume beg < len(data) + + // check for the end of the code block + fenceEnd, _ := isFenceLine(data[beg:], nil, marker) + if fenceEnd != 0 { + beg += fenceEnd + break + } + + // copy the current line + end := skipUntilChar(data, beg, '\n') + 1 + + // did we reach the end of the buffer without a closing marker? + if end >= len(data) { + return 0 + } + + // verbatim copy to the working buffer + if doRender { + work.Write(data[beg:end]) + } + beg = end + } + + if doRender { + codeBlock := &ast.CodeBlock{ + IsFenced: true, + } + codeBlock.Content = work.Bytes() // TODO: get rid of temp buffer + + // Check for caption and if found make it a figure. + if captionContent, id, consumed := p.caption(data[beg:], []byte("Figure: ")); consumed > 0 { + figure := &ast.CaptionFigure{} + caption := &ast.Caption{} + figure.HeadingID = id + p.Inline(caption, captionContent) + + p.addBlock(figure) + codeBlock.AsLeaf().Attribute = figure.AsContainer().Attribute + p.addChild(codeBlock) + finalizeCodeBlock(codeBlock) + p.addChild(caption) + p.finalize(figure) + + beg += consumed + + return beg + } + + // Still here, normal block + p.addBlock(codeBlock) + finalizeCodeBlock(codeBlock) + } + + return beg +} + +func unescapeChar(str []byte) []byte { + if str[0] == '\\' { + return []byte{str[1]} + } + return []byte(html.UnescapeString(string(str))) +} + +func unescapeString(str []byte) []byte { + if reBackslashOrAmp.Match(str) { + return reEntityOrEscapedChar.ReplaceAllFunc(str, unescapeChar) + } + return str +} + +func finalizeCodeBlock(code *ast.CodeBlock) { + c := code.Content + if code.IsFenced { + newlinePos := bytes.IndexByte(c, '\n') + firstLine := c[:newlinePos] + rest := c[newlinePos+1:] + code.Info = unescapeString(bytes.Trim(firstLine, "\n")) + code.Literal = rest + } else { + code.Literal = c + } + code.Content = nil +} + +func (p *Parser) table(data []byte) int { + i, columns, table := p.tableHeader(data) + if i == 0 { + return 0 + } + + p.addBlock(&ast.TableBody{}) + + for i < len(data) { + pipes, rowStart := 0, i + for ; i < len(data) && data[i] != '\n'; i++ { + if data[i] == '|' { + pipes++ + } + } + + if pipes == 0 { + i = rowStart + break + } + + // include the newline in data sent to tableRow + i = skipCharN(data, i, '\n', 1) + + if p.tableFooter(data[rowStart:i]) { + continue + } + + p.tableRow(data[rowStart:i], columns, false) + } + if captionContent, id, consumed := p.caption(data[i:], []byte("Table: ")); consumed > 0 { + caption := &ast.Caption{} + p.Inline(caption, captionContent) + + // Some switcheroo to re-insert the parsed table as a child of the captionfigure. + figure := &ast.CaptionFigure{} + figure.HeadingID = id + table2 := &ast.Table{} + // Retain any block level attributes. + table2.AsContainer().Attribute = table.AsContainer().Attribute + children := table.GetChildren() + ast.RemoveFromTree(table) + + table2.SetChildren(children) + ast.AppendChild(figure, table2) + ast.AppendChild(figure, caption) + + p.addChild(figure) + p.finalize(figure) + + i += consumed + } + + return i +} + +// check if the specified position is preceded by an odd number of backslashes +func isBackslashEscaped(data []byte, i int) bool { + backslashes := 0 + for i-backslashes-1 >= 0 && data[i-backslashes-1] == '\\' { + backslashes++ + } + return backslashes&1 == 1 +} + +// tableHeaders parses the header. If recognized it will also add a table. +func (p *Parser) tableHeader(data []byte) (size int, columns []ast.CellAlignFlags, table ast.Node) { + i := 0 + colCount := 1 + for i = 0; i < len(data) && data[i] != '\n'; i++ { + if data[i] == '|' && !isBackslashEscaped(data, i) { + colCount++ + } + } + + // doesn't look like a table header + if colCount == 1 { + return + } + + // include the newline in the data sent to tableRow + j := skipCharN(data, i, '\n', 1) + header := data[:j] + + // column count ignores pipes at beginning or end of line + if data[0] == '|' { + colCount-- + } + if i > 2 && data[i-1] == '|' && !isBackslashEscaped(data, i-1) { + colCount-- + } + + columns = make([]ast.CellAlignFlags, colCount) + + // move on to the header underline + i++ + if i >= len(data) { + return + } + + if data[i] == '|' && !isBackslashEscaped(data, i) { + i++ + } + i = skipChar(data, i, ' ') + + // each column header is of form: / *:?-+:? *|/ with # dashes + # colons >= 3 + // and trailing | optional on last column + col := 0 + n := len(data) + for i < n && data[i] != '\n' { + dashes := 0 + + if data[i] == ':' { + i++ + columns[col] |= ast.TableAlignmentLeft + dashes++ + } + for i < n && data[i] == '-' { + i++ + dashes++ + } + if i < n && data[i] == ':' { + i++ + columns[col] |= ast.TableAlignmentRight + dashes++ + } + for i < n && data[i] == ' ' { + i++ + } + if i == n { + return + } + // end of column test is messy + switch { + case dashes < 3: + // not a valid column + return + + case data[i] == '|' && !isBackslashEscaped(data, i): + // marker found, now skip past trailing whitespace + col++ + i++ + for i < n && data[i] == ' ' { + i++ + } + + // trailing junk found after last column + if col >= colCount && i < len(data) && data[i] != '\n' { + return + } + + case (data[i] != '|' || isBackslashEscaped(data, i)) && col+1 < colCount: + // something else found where marker was required + return + + case data[i] == '\n': + // marker is optional for the last column + col++ + + default: + // trailing junk found after last column + return + } + } + if col != colCount { + return + } + + table = &ast.Table{} + p.addBlock(table) + p.addBlock(&ast.TableHeader{}) + p.tableRow(header, columns, true) + size = skipCharN(data, i, '\n', 1) + return +} + +func (p *Parser) tableRow(data []byte, columns []ast.CellAlignFlags, header bool) { + p.addBlock(&ast.TableRow{}) + i, col := 0, 0 + + if data[i] == '|' && !isBackslashEscaped(data, i) { + i++ + } + + n := len(data) + for col = 0; col < len(columns) && i < n; col++ { + for i < n && data[i] == ' ' { + i++ + } + + cellStart := i + + for i < n && (data[i] != '|' || isBackslashEscaped(data, i)) && data[i] != '\n' { + i++ + } + + cellEnd := i + + // skip the end-of-cell marker, possibly taking us past end of buffer + i++ + + for cellEnd > cellStart && cellEnd-1 < n && data[cellEnd-1] == ' ' { + cellEnd-- + } + + block := &ast.TableCell{ + IsHeader: header, + Align: columns[col], + } + block.Content = data[cellStart:cellEnd] + p.addBlock(block) + } + + // pad it out with empty columns to get the right number + for ; col < len(columns); col++ { + block := &ast.TableCell{ + IsHeader: header, + Align: columns[col], + } + p.addBlock(block) + } + + // silently ignore rows with too many cells +} + +// tableFooter parses the (optional) table footer. +func (p *Parser) tableFooter(data []byte) bool { + colCount := 1 + for i := 0; i < len(data) && data[i] != '\n'; i++ { + if data[i] == '|' && !isBackslashEscaped(data, i) { + colCount++ + continue + } + // remaining data must be the = character + if data[i] != '=' { + return false + } + } + + // doesn't look like a table footer + if colCount == 1 { + return false + } + + p.addBlock(&ast.TableFooter{}) + + return true +} + +// returns blockquote prefix length +func (p *Parser) quotePrefix(data []byte) int { + i := 0 + n := len(data) + for i < 3 && i < n && data[i] == ' ' { + i++ + } + if i < n && data[i] == '>' { + if i+1 < n && data[i+1] == ' ' { + return i + 2 + } + return i + 1 + } + return 0 +} + +// blockquote ends with at least one blank line +// followed by something without a blockquote prefix +func (p *Parser) terminateBlockquote(data []byte, beg, end int) bool { + if p.isEmpty(data[beg:]) <= 0 { + return false + } + if end >= len(data) { + return true + } + return p.quotePrefix(data[end:]) == 0 && p.isEmpty(data[end:]) == 0 +} + +// parse a blockquote fragment +func (p *Parser) quote(data []byte) int { + var raw bytes.Buffer + beg, end := 0, 0 + for beg < len(data) { + end = beg + // Step over whole lines, collecting them. While doing that, check for + // fenced code and if one's found, incorporate it altogether, + // irregardless of any contents inside it + for end < len(data) && data[end] != '\n' { + end++ + } + end = skipCharN(data, end, '\n', 1) + if pre := p.quotePrefix(data[beg:]); pre > 0 { + // skip the prefix + beg += pre + } else if p.terminateBlockquote(data, beg, end) { + break + } + // this line is part of the blockquote + raw.Write(data[beg:end]) + beg = end + } + + quote := &ast.BlockQuote{} + quote.Literal = raw.Bytes() + block := p.addBlock(quote) + p.finalize(block) + + return end +} + +// returns prefix length for block code +func (p *Parser) codePrefix(data []byte) int { + n := len(data) + if n >= 1 && data[0] == '\t' { + return 1 + } + if n >= 4 && data[3] == ' ' && data[2] == ' ' && data[1] == ' ' && data[0] == ' ' { + return 4 + } + return 0 +} + +func (p *Parser) code(data []byte) int { + var work bytes.Buffer + + i := 0 + for i < len(data) { + beg := i + + i = skipUntilChar(data, i, '\n') + i = skipCharN(data, i, '\n', 1) + + blankline := p.isEmpty(data[beg:i]) > 0 + if pre := p.codePrefix(data[beg:i]); pre > 0 { + beg += pre + } else if !blankline { + // non-empty, non-prefixed line breaks the pre + i = beg + break + } + + // verbatim copy to the working buffer + if blankline { + work.WriteByte('\n') + } else { + work.Write(data[beg:i]) + } + } + + // trim all the \n off the end of work + workbytes := work.Bytes() + + eol := backChar(workbytes, len(workbytes), '\n') + + if eol != len(workbytes) { + work.Truncate(eol) + } + + work.WriteByte('\n') + + codeBlock := &ast.CodeBlock{ + IsFenced: false, + } + // TODO: get rid of temp buffer + codeBlock.Content = work.Bytes() + p.addBlock(codeBlock) + finalizeCodeBlock(codeBlock) + + return i +} + +// returns unordered list item prefix +func (p *Parser) uliPrefix(data []byte) int { + // start with up to 3 spaces + i := skipCharN(data, 0, ' ', 3) + + if i >= len(data)-1 { + return 0 + } + // need one of {'*', '+', '-'} followed by a space or a tab + if (data[i] != '*' && data[i] != '+' && data[i] != '-') || + (data[i+1] != ' ' && data[i+1] != '\t') { + return 0 + } + return i + 2 +} + +// returns ordered list item prefix +func (p *Parser) oliPrefix(data []byte) int { + // start with up to 3 spaces + i := skipCharN(data, 0, ' ', 3) + + // count the digits + start := i + for i < len(data) && data[i] >= '0' && data[i] <= '9' { + i++ + } + if start == i || i >= len(data)-1 { + return 0 + } + + // we need >= 1 digits followed by a dot and a space or a tab + if data[i] != '.' || !(data[i+1] == ' ' || data[i+1] == '\t') { + return 0 + } + return i + 2 +} + +// returns definition list item prefix +func (p *Parser) dliPrefix(data []byte) int { + if len(data) < 2 { + return 0 + } + // need a ':' followed by a space or a tab + if data[0] != ':' || !(data[1] == ' ' || data[1] == '\t') { + return 0 + } + i := skipChar(data, 0, ' ') + return i + 2 +} + +// parse ordered or unordered list block +func (p *Parser) list(data []byte, flags ast.ListType, start int) int { + i := 0 + flags |= ast.ListItemBeginningOfList + list := &ast.List{ + ListFlags: flags, + Tight: true, + Start: start, + } + block := p.addBlock(list) + + for i < len(data) { + skip := p.listItem(data[i:], &flags) + if flags&ast.ListItemContainsBlock != 0 { + list.Tight = false + } + i += skip + if skip == 0 || flags&ast.ListItemEndOfList != 0 { + break + } + flags &= ^ast.ListItemBeginningOfList + } + + above := block.GetParent() + finalizeList(list) + p.tip = above + return i +} + +// Returns true if the list item is not the same type as its parent list +func (p *Parser) listTypeChanged(data []byte, flags *ast.ListType) bool { + if p.dliPrefix(data) > 0 && *flags&ast.ListTypeDefinition == 0 { + return true + } else if p.oliPrefix(data) > 0 && *flags&ast.ListTypeOrdered == 0 { + return true + } else if p.uliPrefix(data) > 0 && (*flags&ast.ListTypeOrdered != 0 || *flags&ast.ListTypeDefinition != 0) { + return true + } + return false +} + +// Returns true if block ends with a blank line, descending if needed +// into lists and sublists. +func endsWithBlankLine(block ast.Node) bool { + // TODO: figure this out. Always false now. + for block != nil { + //if block.lastLineBlank { + //return true + //} + switch block.(type) { + case *ast.List, *ast.ListItem: + block = ast.GetLastChild(block) + default: + return false + } + } + return false +} + +func finalizeList(list *ast.List) { + items := list.Parent.GetChildren() + lastItemIdx := len(items) - 1 + for i, item := range items { + isLastItem := i == lastItemIdx + // check for non-final list item ending with blank line: + if !isLastItem && endsWithBlankLine(item) { + list.Tight = false + break + } + // recurse into children of list item, to see if there are spaces + // between any of them: + subItems := item.GetParent().GetChildren() + lastSubItemIdx := len(subItems) - 1 + for j, subItem := range subItems { + isLastSubItem := j == lastSubItemIdx + if (!isLastItem || !isLastSubItem) && endsWithBlankLine(subItem) { + list.Tight = false + break + } + } + } +} + +// Parse a single list item. +// Assumes initial prefix is already removed if this is a sublist. +func (p *Parser) listItem(data []byte, flags *ast.ListType) int { + // keep track of the indentation of the first line + itemIndent := 0 + if data[0] == '\t' { + itemIndent += 4 + } else { + for itemIndent < 3 && data[itemIndent] == ' ' { + itemIndent++ + } + } + + var bulletChar byte = '*' + i := p.uliPrefix(data) + if i == 0 { + i = p.oliPrefix(data) + } else { + bulletChar = data[i-2] + } + if i == 0 { + i = p.dliPrefix(data) + // reset definition term flag + if i > 0 { + *flags &= ^ast.ListTypeTerm + } + } + if i == 0 { + // if in definition list, set term flag and continue + if *flags&ast.ListTypeDefinition != 0 { + *flags |= ast.ListTypeTerm + } else { + return 0 + } + } + + // skip leading whitespace on first line + i = skipChar(data, i, ' ') + + // find the end of the line + line := i + for i > 0 && i < len(data) && data[i-1] != '\n' { + i++ + } + + // get working buffer + var raw bytes.Buffer + + // put the first line into the working buffer + raw.Write(data[line:i]) + line = i + + // process the following lines + containsBlankLine := false + sublist := 0 + +gatherlines: + for line < len(data) { + i++ + + // find the end of this line + for i < len(data) && data[i-1] != '\n' { + i++ + } + + // if it is an empty line, guess that it is part of this item + // and move on to the next line + if p.isEmpty(data[line:i]) > 0 { + containsBlankLine = true + line = i + continue + } + + // calculate the indentation + indent := 0 + indentIndex := 0 + if data[line] == '\t' { + indentIndex++ + indent += 4 + } else { + for indent < 4 && line+indent < i && data[line+indent] == ' ' { + indent++ + indentIndex++ + } + } + + chunk := data[line+indentIndex : i] + + // evaluate how this line fits in + switch { + // is this a nested list item? + case (p.uliPrefix(chunk) > 0 && !p.isHRule(chunk)) || p.oliPrefix(chunk) > 0 || p.dliPrefix(chunk) > 0: + + // to be a nested list, it must be indented more + // if not, it is either a different kind of list + // or the next item in the same list + if indent <= itemIndent { + if p.listTypeChanged(chunk, flags) { + *flags |= ast.ListItemEndOfList + } else if containsBlankLine { + *flags |= ast.ListItemContainsBlock + } + + break gatherlines + } + + if containsBlankLine { + *flags |= ast.ListItemContainsBlock + } + + // is this the first item in the nested list? + if sublist == 0 { + sublist = raw.Len() + // in the case of dliPrefix we are too late and need to search back for the definition item, which + // should be on the previous line, we then adjust sublist to start there. + if p.dliPrefix(chunk) > 0 { + sublist = backUntilChar(raw.Bytes(), raw.Len()-1, '\n') + } + } + + // is this a nested prefix heading? + case p.isPrefixHeading(chunk), p.isPrefixSpecialHeading(chunk): + // if the heading is not indented, it is not nested in the list + // and thus ends the list + if containsBlankLine && indent < 4 { + *flags |= ast.ListItemEndOfList + break gatherlines + } + *flags |= ast.ListItemContainsBlock + + // anything following an empty line is only part + // of this item if it is indented 4 spaces + // (regardless of the indentation of the beginning of the item) + case containsBlankLine && indent < 4: + if *flags&ast.ListTypeDefinition != 0 && i < len(data)-1 { + // is the next item still a part of this list? + next := i + for next < len(data) && data[next] != '\n' { + next++ + } + for next < len(data)-1 && data[next] == '\n' { + next++ + } + if i < len(data)-1 && data[i] != ':' && next < len(data)-1 && data[next] != ':' { + *flags |= ast.ListItemEndOfList + } + } else { + *flags |= ast.ListItemEndOfList + } + break gatherlines + + // a blank line means this should be parsed as a block + case containsBlankLine: + raw.WriteByte('\n') + *flags |= ast.ListItemContainsBlock + } + + // if this line was preceded by one or more blanks, + // re-introduce the blank into the buffer + if containsBlankLine { + containsBlankLine = false + raw.WriteByte('\n') + } + + // add the line into the working buffer without prefix + raw.Write(data[line+indentIndex : i]) + + line = i + } + + rawBytes := raw.Bytes() + + listItem := &ast.ListItem{ + ListFlags: *flags, + Tight: false, + BulletChar: bulletChar, + Delimiter: '.', // Only '.' is possible in Markdown, but ')' will also be possible in CommonMark + } + p.addBlock(listItem) + + // render the contents of the list item + if *flags&ast.ListItemContainsBlock != 0 && *flags&ast.ListTypeTerm == 0 { + // intermediate render of block item, except for definition term + if sublist > 0 { + p.block(rawBytes[:sublist]) + p.block(rawBytes[sublist:]) + } else { + p.block(rawBytes) + } + } else { + // intermediate render of inline item + para := &ast.Paragraph{} + if sublist > 0 { + para.Content = rawBytes[:sublist] + } else { + para.Content = rawBytes + } + p.addChild(para) + if sublist > 0 { + p.block(rawBytes[sublist:]) + } + } + return line +} + +// render a single paragraph that has already been parsed out +func (p *Parser) renderParagraph(data []byte) { + if len(data) == 0 { + return + } + + // trim leading spaces + beg := skipChar(data, 0, ' ') + + end := len(data) + // trim trailing newline + if data[len(data)-1] == '\n' { + end-- + } + + // trim trailing spaces + for end > beg && data[end-1] == ' ' { + end-- + } + para := &ast.Paragraph{} + para.Content = data[beg:end] + p.addBlock(para) +} + +// blockMath handle block surround with $$ +func (p *Parser) blockMath(data []byte) int { + if len(data) <= 4 || data[0] != '$' || data[1] != '$' || data[2] == '$' { + return 0 + } + + // find next $$ + var end int + for end = 2; end+1 < len(data) && (data[end] != '$' || data[end+1] != '$'); end++ { + } + + // $$ not match + if end+1 == len(data) { + return 0 + } + + // render the display math + mathBlock := &ast.MathBlock{} + mathBlock.Literal = data[2:end] + p.addBlock(mathBlock) + + return end + 2 +} + +func (p *Parser) paragraph(data []byte) int { + // prev: index of 1st char of previous line + // line: index of 1st char of current line + // i: index of cursor/end of current line + var prev, line, i int + tabSize := tabSizeDefault + if p.extensions&TabSizeEight != 0 { + tabSize = tabSizeDouble + } + // keep going until we find something to mark the end of the paragraph + for i < len(data) { + // mark the beginning of the current line + prev = line + current := data[i:] + line = i + + // did we find a reference or a footnote? If so, end a paragraph + // preceding it and report that we have consumed up to the end of that + // reference: + if refEnd := isReference(p, current, tabSize); refEnd > 0 { + p.renderParagraph(data[:i]) + return i + refEnd + } + + // did we find a blank line marking the end of the paragraph? + if n := p.isEmpty(current); n > 0 { + p.renderParagraph(data[:i]) + return i + n + } + + // an underline under some text marks a heading, so our paragraph ended on prev line + if i > 0 { + if level := p.isUnderlinedHeading(current); level > 0 { + // render the paragraph + p.renderParagraph(data[:prev]) + + // ignore leading and trailing whitespace + eol := i - 1 + for prev < eol && data[prev] == ' ' { + prev++ + } + for eol > prev && data[eol-1] == ' ' { + eol-- + } + + id := "" + if p.extensions&AutoHeadingIDs != 0 { + id = sanitizeAnchorName(string(data[prev:eol])) + } + + block := &ast.Heading{ + Level: level, + HeadingID: id, + } + block.Content = data[prev:eol] + p.addBlock(block) + + // find the end of the underline + return skipUntilChar(data, i, '\n') + } + } + + // if there's a fenced code block, paragraph is over + if p.extensions&FencedCode != 0 { + if p.fencedCodeBlock(current, false) > 0 { + p.renderParagraph(data[:i]) + return i + } + } + + // otherwise, scan to the beginning of the next line + nl := bytes.IndexByte(data[i:], '\n') + if nl >= 0 { + i += nl + 1 + } else { + i += len(data[i:]) + } + } + + p.renderParagraph(data[:i]) + return i +} + +// skipChar advances i as long as data[i] == c +func skipChar(data []byte, i int, c byte) int { + n := len(data) + for i < n && data[i] == c { + i++ + } + return i +} + +// like skipChar but only skips up to max characters +func skipCharN(data []byte, i int, c byte, max int) int { + n := len(data) + for i < n && max > 0 && data[i] == c { + i++ + max-- + } + return i +} + +// skipUntilChar advances i as long as data[i] != c +func skipUntilChar(data []byte, i int, c byte) int { + n := len(data) + for i < n && data[i] != c { + i++ + } + return i +} + +func skipAlnum(data []byte, i int) int { + n := len(data) + for i < n && isAlnum(data[i]) { + i++ + } + return i +} + +func skipSpace(data []byte, i int) int { + n := len(data) + for i < n && isSpace(data[i]) { + i++ + } + return i +} + +func backChar(data []byte, i int, c byte) int { + for i > 0 && data[i-1] == c { + i-- + } + return i +} + +func backUntilChar(data []byte, i int, c byte) int { + for i > 0 && data[i-1] != c { + i-- + } + return i +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/callout.go b/vendor/github.com/gomarkdown/markdown/parser/callout.go new file mode 100644 index 000000000..15858aa97 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/callout.go @@ -0,0 +1,29 @@ +package parser + +import ( + "bytes" + "strconv" +) + +// IsCallout detects a callout in the following format: <> Where N is a integer > 0. +func IsCallout(data []byte) (id []byte, consumed int) { + if !bytes.HasPrefix(data, []byte("<<")) { + return nil, 0 + } + start := 2 + end := bytes.Index(data[start:], []byte(">>")) + if end < 0 { + return nil, 0 + } + + b := data[start : start+end] + b = bytes.TrimSpace(b) + i, err := strconv.Atoi(string(b)) + if err != nil { + return nil, 0 + } + if i <= 0 { + return nil, 0 + } + return b, start + end + 2 // 2 for >> +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/caption.go b/vendor/github.com/gomarkdown/markdown/parser/caption.go new file mode 100644 index 000000000..54d3f741f --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/caption.go @@ -0,0 +1,70 @@ +package parser + +import ( + "bytes" +) + +// caption checks for a caption, it returns the caption data and a potential "headingID". +func (p *Parser) caption(data, caption []byte) ([]byte, string, int) { + if !bytes.HasPrefix(data, caption) { + return nil, "", 0 + } + j := len(caption) + data = data[j:] + end := p.linesUntilEmpty(data) + + data = data[:end] + + id, start := captionID(data) + if id != "" { + return data[:start], id, end + j + } + + return data, "", end + j +} + +// linesUntilEmpty scans lines up to the first empty line. +func (p *Parser) linesUntilEmpty(data []byte) int { + line, i := 0, 0 + + for line < len(data) { + i++ + + // find the end of this line + for i < len(data) && data[i-1] != '\n' { + i++ + } + + if p.isEmpty(data[line:i]) == 0 { + line = i + continue + } + + break + } + return i +} + +// captionID checks if the caption *ends* in {#....}. If so the text after {# is taken to be +// the ID/anchor of the entire figure block. +func captionID(data []byte) (string, int) { + end := len(data) + + j, k := 0, 0 + // find start/end of heading id + for j = 0; j < end-1 && (data[j] != '{' || data[j+1] != '#'); j++ { + } + for k = j + 1; k < end && data[k] != '}'; k++ { + } + // remains must be whitespace. + for l := k + 1; l < end; l++ { + if !isSpace(data[l]) { + return "", 0 + } + } + + if j > 0 && k > 0 && j+2 < k { + return string(data[j+2 : k]), j + } + return "", 0 +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/citation.go b/vendor/github.com/gomarkdown/markdown/parser/citation.go new file mode 100644 index 000000000..8ea1fbeee --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/citation.go @@ -0,0 +1,86 @@ +package parser + +import ( + "bytes" + + "github.com/gomarkdown/markdown/ast" +) + +// citation parses a citation. In its most simple form [@ref], we allow multiple +// being separated by semicolons and a sub reference inside ala pandoc: [@ref, p. 23]. +// Each citation can have a modifier: !, ? or - wich mean: +// +// ! - normative +// ? - formative +// - - suppressed +// +// The suffix starts after a comma, we strip any whitespace before and after. If the output +// allows for it, this can be rendered. +func citation(p *Parser, data []byte, offset int) (int, ast.Node) { + // look for the matching closing bracket + i := offset + 1 + for level := 1; level > 0 && i < len(data); i++ { + switch { + case data[i] == '\n': + // no newlines allowed. + return 0, nil + + case data[i-1] == '\\': + continue + + case data[i] == '[': + level++ + + case data[i] == ']': + level-- + if level <= 0 { + i-- // compensate for extra i++ in for loop + } + } + } + + if i >= len(data) { + return 0, nil + } + + node := &ast.Citation{} + + citations := bytes.Split(data[1:i], []byte(";")) + for _, citation := range citations { + var suffix []byte + citation = bytes.TrimSpace(citation) + j := 0 + if citation[j] != '@' { + // not a citation, drop out entirely. + return 0, nil + } + if c := bytes.Index(citation, []byte(",")); c > 0 { + part := citation[:c] + suff := citation[c+1:] + part = bytes.TrimSpace(part) + suff = bytes.TrimSpace(suff) + + citation = part + suffix = suff + } + + citeType := ast.CitationTypeInformative + j = 1 + switch citation[j] { + case '!': + citeType = ast.CitationTypeNormative + j++ + case '?': + citeType = ast.CitationTypeInformative + j++ + case '-': + citeType = ast.CitationTypeSuppressed + j++ + } + node.Destination = append(node.Destination, citation[j:]) + node.Type = append(node.Type, citeType) + node.Suffix = append(node.Suffix, suffix) + } + + return i + 1, node +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/esc.go b/vendor/github.com/gomarkdown/markdown/parser/esc.go new file mode 100644 index 000000000..0a79aa35e --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/esc.go @@ -0,0 +1,20 @@ +package parser + +// isEscape returns true if byte i is prefixed by an odd number of backslahses. +func isEscape(data []byte, i int) bool { + if i == 0 { + return false + } + if i == 1 { + return data[0] == '\\' + } + j := i - 1 + for ; j >= 0; j-- { + if data[j] != '\\' { + break + } + } + j++ + // odd number of backslahes means escape + return (i-j)%2 != 0 +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/figures.go b/vendor/github.com/gomarkdown/markdown/parser/figures.go new file mode 100644 index 000000000..6615449cf --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/figures.go @@ -0,0 +1,119 @@ +package parser + +import ( + "bytes" + + "github.com/gomarkdown/markdown/ast" +) + +// sFigureLine checks if there's a figure line (e.g., !--- ) at the beginning of data, +// and returns the end index if so, or 0 otherwise. +func sFigureLine(data []byte, oldmarker string) (end int, marker string) { + i, size := 0, 0 + + n := len(data) + // skip up to three spaces + for i < n && i < 3 && data[i] == ' ' { + i++ + } + + // check for the marker characters: ! + if i+1 >= n { + return 0, "" + } + if data[i] != '!' || data[i+1] != '-' { + return 0, "" + } + i++ + + c := data[i] // i.e. the - + + // the whole line must be the same char or whitespace + for i < n && data[i] == c { + size++ + i++ + } + + // the marker char must occur at least 3 times + if size < 3 { + return 0, "" + } + marker = string(data[i-size : i]) + + // if this is the end marker, it must match the beginning marker + if oldmarker != "" && marker != oldmarker { + return 0, "" + } + + // there is no syntax modifier although it might be an idea to re-use this space for something? + + i = skipChar(data, i, ' ') + if i >= n || data[i] != '\n' { + if i == n { + return i, marker + } + return 0, "" + } + return i + 1, marker // Take newline into account. +} + +// figureBlock returns the end index if data contains a figure block at the beginning, +// or 0 otherwise. It writes to out if doRender is true, otherwise it has no side effects. +// If doRender is true, a final newline is mandatory to recognize the figure block. +func (p *Parser) figureBlock(data []byte, doRender bool) int { + beg, marker := sFigureLine(data, "") + if beg == 0 || beg >= len(data) { + return 0 + } + + var raw bytes.Buffer + + for { + // safe to assume beg < len(data) + + // check for the end of the code block + figEnd, _ := sFigureLine(data[beg:], marker) + if figEnd != 0 { + beg += figEnd + break + } + + // copy the current line + end := skipUntilChar(data, beg, '\n') + 1 + + // did we reach the end of the buffer without a closing marker? + if end >= len(data) { + return 0 + } + + // verbatim copy to the working buffer + if doRender { + raw.Write(data[beg:end]) + } + beg = end + } + + if !doRender { + return beg + } + + figure := &ast.CaptionFigure{} + p.addBlock(figure) + p.block(raw.Bytes()) + + defer p.finalize(figure) + + if captionContent, id, consumed := p.caption(data[beg:], []byte("Figure: ")); consumed > 0 { + caption := &ast.Caption{} + p.Inline(caption, captionContent) + + figure.HeadingID = id + + p.addChild(caption) + + beg += consumed + } + + p.finalize(figure) + return beg +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/include.go b/vendor/github.com/gomarkdown/markdown/parser/include.go new file mode 100644 index 000000000..2448a6854 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/include.go @@ -0,0 +1,129 @@ +package parser + +import ( + "bytes" + "path" + "path/filepath" +) + +// isInclude parses {{...}}[...], that contains a path between the {{, the [...] syntax contains +// an address to select which lines to include. It is treated as an opaque string and just given +// to readInclude. +func (p *Parser) isInclude(data []byte) (filename string, address []byte, consumed int) { + i := skipCharN(data, 0, ' ', 3) // start with up to 3 spaces + if len(data[i:]) < 3 { + return "", nil, 0 + } + if data[i] != '{' || data[i+1] != '{' { + return "", nil, 0 + } + start := i + 2 + + // find the end delimiter + i = skipUntilChar(data, i, '}') + if i+1 >= len(data) { + return "", nil, 0 + } + end := i + i++ + if data[i] != '}' { + return "", nil, 0 + } + filename = string(data[start:end]) + + if i+1 < len(data) && data[i+1] == '[' { // potential address specification + start := i + 2 + + end = skipUntilChar(data, start, ']') + if end >= len(data) { + return "", nil, 0 + } + address = data[start:end] + return filename, address, end + 1 + } + + return filename, address, i + 1 +} + +func (p *Parser) readInclude(from, file string, address []byte) []byte { + if p.Opts.ReadIncludeFn != nil { + return p.Opts.ReadIncludeFn(from, file, address) + } + + return nil +} + +// isCodeInclude parses <{{...}} which is similar to isInclude the returned bytes are, however wrapped in a code block. +func (p *Parser) isCodeInclude(data []byte) (filename string, address []byte, consumed int) { + i := skipCharN(data, 0, ' ', 3) // start with up to 3 spaces + if len(data[i:]) < 3 { + return "", nil, 0 + } + if data[i] != '<' { + return "", nil, 0 + } + start := i + + filename, address, consumed = p.isInclude(data[i+1:]) + if consumed == 0 { + return "", nil, 0 + } + return filename, address, start + consumed + 1 +} + +// readCodeInclude acts like include except the returned bytes are wrapped in a fenced code block. +func (p *Parser) readCodeInclude(from, file string, address []byte) []byte { + data := p.readInclude(from, file, address) + if data == nil { + return nil + } + ext := path.Ext(file) + buf := &bytes.Buffer{} + buf.Write([]byte("```")) + if ext != "" { // starts with a dot + buf.WriteString(" " + ext[1:] + "\n") + } else { + buf.WriteByte('\n') + } + buf.Write(data) + buf.WriteString("```\n") + return buf.Bytes() +} + +// incStack hold the current stack of chained includes. Each value is the containing +// path of the file being parsed. +type incStack struct { + stack []string +} + +func newIncStack() *incStack { + return &incStack{stack: []string{}} +} + +// Push updates i with new. +func (i *incStack) Push(new string) { + if path.IsAbs(new) { + i.stack = append(i.stack, path.Dir(new)) + return + } + last := "" + if len(i.stack) > 0 { + last = i.stack[len(i.stack)-1] + } + i.stack = append(i.stack, path.Dir(filepath.Join(last, new))) +} + +// Pop pops the last value. +func (i *incStack) Pop() { + if len(i.stack) == 0 { + return + } + i.stack = i.stack[:len(i.stack)-1] +} + +func (i *incStack) Last() string { + if len(i.stack) == 0 { + return "" + } + return i.stack[len(i.stack)-1] +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/inline.go b/vendor/github.com/gomarkdown/markdown/parser/inline.go new file mode 100644 index 000000000..1c416617d --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/inline.go @@ -0,0 +1,1238 @@ +package parser + +import ( + "bytes" + "regexp" + "strconv" + + "github.com/gomarkdown/markdown/ast" +) + +// Parsing of inline elements + +var ( + urlRe = `((https?|ftp):\/\/|\/)[-A-Za-z0-9+&@#\/%?=~_|!:,.;\(\)]+` + anchorRe = regexp.MustCompile(`^(]+")?\s?>` + urlRe + `<\/a>)`) + + // TODO: improve this regexp to catch all possible entities: + htmlEntityRe = regexp.MustCompile(`&[a-z]{2,5};`) +) + +// Inline parses text within a block. +// Each function returns the number of consumed chars. +func (p *Parser) Inline(currBlock ast.Node, data []byte) { + // handlers might call us recursively: enforce a maximum depth + if len(data) == 0 { + return + } + if p.nesting >= p.maxNesting { + ast.AppendChild(currBlock, newTextNode(data)) + return + } + p.nesting++ + beg, end := 0, 0 + + n := len(data) + for end < n { + handler := p.inlineCallback[data[end]] + if handler == nil { + end++ + continue + } + consumed, node := handler(p, data, end) + if consumed == 0 { + // no action from the callback + end++ + continue + } + // copy inactive chars into the output + ast.AppendChild(currBlock, newTextNode(data[beg:end])) + if node != nil { + ast.AppendChild(currBlock, node) + } + beg = end + consumed + end = beg + } + + if beg < n { + if data[end-1] == '\n' { + end-- + } + ast.AppendChild(currBlock, newTextNode(data[beg:end])) + } + p.nesting-- +} + +func statusTag(p *Parser, data []byte, offset int) (int, ast.Node) { + data = data[offset:] + n := len(data) + + if n == 1 { + return 0, nil + } + + // Space cannot follow tag + if isSpace(data[1]) { + return 0, nil + } + + i := 1 + for i < n { + if isSpace(data[i]) { + break + } + if !isValidStatusTagChar(data[i]) { + return 0, nil + } + i++ + } + + if i == 1 { + return 0, nil + } + + statusTag := &ast.StatusTag{} + statusTag.Literal = data[1:i] + + return i, statusTag +} + +// single and double emphasis parsing +func emphasis(p *Parser, data []byte, offset int) (int, ast.Node) { + data = data[offset:] + c := data[0] + + n := len(data) + if n > 2 && data[1] != c { + // whitespace cannot follow an opening emphasis; + // strikethrough only takes two characters '~~' + if isSpace(data[1]) { + return 0, nil + } + if p.extensions&SuperSubscript != 0 && c == '~' { + // potential subscript, no spaces, except when escaped, helperEmphasis does + // not check that for us, so walk the bytes and check. + ret := skipUntilChar(data[1:], 0, c) + if ret == 0 { + return 0, nil + } + ret++ // we started with data[1:] above. + for i := 1; i < ret; i++ { + if isSpace(data[i]) && !isEscape(data, i) { + return 0, nil + } + } + sub := &ast.Subscript{} + sub.Literal = data[1:ret] + return ret + 1, sub + } + ret, node := helperEmphasis(p, data[1:], c) + if ret == 0 { + return 0, nil + } + + return ret + 1, node + } + + if n > 3 && data[1] == c && data[2] != c { + if isSpace(data[2]) { + return 0, nil + } + ret, node := helperDoubleEmphasis(p, data[2:], c) + if ret == 0 { + return 0, nil + } + + return ret + 2, node + } + + if n > 4 && data[1] == c && data[2] == c && data[3] != c { + if c == '~' || isSpace(data[3]) { + return 0, nil + } + ret, node := helperTripleEmphasis(p, data, 3, c) + if ret == 0 { + return 0, nil + } + + return ret + 3, node + } + + return 0, nil +} + +func codeSpan(p *Parser, data []byte, offset int) (int, ast.Node) { + data = data[offset:] + + // count the number of backticks in the delimiter + nb := skipChar(data, 0, '`') + + // find the next delimiter + i, end := 0, 0 + for end = nb; end < len(data) && i < nb; end++ { + if data[end] == '`' { + i++ + } else { + i = 0 + } + } + + // no matching delimiter? + if i < nb && end >= len(data) { + return 0, nil + } + + // trim outside whitespace + fBegin := nb + for fBegin < end && data[fBegin] == ' ' { + fBegin++ + } + + fEnd := end - nb + for fEnd > fBegin && data[fEnd-1] == ' ' { + fEnd-- + } + + // render the code span + if fBegin != fEnd { + code := &ast.Code{} + code.Literal = data[fBegin:fEnd] + return end, code + } + + return end, nil +} + +// newline preceded by two spaces becomes
+func maybeLineBreak(p *Parser, data []byte, offset int) (int, ast.Node) { + origOffset := offset + offset = skipChar(data, offset, ' ') + + if offset < len(data) && data[offset] == '\n' { + if offset-origOffset >= 2 { + return offset - origOffset + 1, &ast.Hardbreak{} + } + return offset - origOffset, nil + } + return 0, nil +} + +// newline without two spaces works when HardLineBreak is enabled +func lineBreak(p *Parser, data []byte, offset int) (int, ast.Node) { + if p.extensions&HardLineBreak != 0 { + return 1, &ast.Hardbreak{} + } + return 0, nil +} + +type linkType int + +const ( + linkNormal linkType = iota + linkImg + linkDeferredFootnote + linkInlineFootnote + linkCitation +) + +func isReferenceStyleLink(data []byte, pos int, t linkType) bool { + if t == linkDeferredFootnote { + return false + } + return pos < len(data)-1 && data[pos] == '[' && data[pos+1] != '^' +} + +func maybeImage(p *Parser, data []byte, offset int) (int, ast.Node) { + if offset < len(data)-1 && data[offset+1] == '[' { + return link(p, data, offset) + } + return 0, nil +} + +func maybeInlineFootnoteOrSuper(p *Parser, data []byte, offset int) (int, ast.Node) { + if offset < len(data)-1 && data[offset+1] == '[' { + return link(p, data, offset) + } + + if p.extensions&SuperSubscript != 0 { + ret := skipUntilChar(data[offset:], 1, '^') + if ret == 0 { + return 0, nil + } + for i := offset; i < offset+ret; i++ { + if isSpace(data[i]) && !isEscape(data, i) { + return 0, nil + } + } + sup := &ast.Superscript{} + sup.Literal = data[offset+1 : offset+ret] + return offset + ret, sup + } + + return 0, nil +} + +// '[': parse a link or an image or a footnote or a citation +func link(p *Parser, data []byte, offset int) (int, ast.Node) { + // no links allowed inside regular links, footnote, and deferred footnotes + if p.insideLink && (offset > 0 && data[offset-1] == '[' || len(data)-1 > offset && data[offset+1] == '^') { + return 0, nil + } + + var t linkType + switch { + // special case: ![^text] == deferred footnote (that follows something with + // an exclamation point) + case p.extensions&Footnotes != 0 && len(data)-1 > offset && data[offset+1] == '^': + t = linkDeferredFootnote + // ![alt] == image + case offset >= 0 && data[offset] == '!': + t = linkImg + offset++ + // [@citation], [@-citation], [@?citation], [@!citation] + case p.extensions&Mmark != 0 && len(data)-1 > offset && data[offset+1] == '@': + t = linkCitation + // [text] == regular link + // ^[text] == inline footnote + // [^refId] == deferred footnote + case p.extensions&Footnotes != 0: + if offset >= 0 && data[offset] == '^' { + t = linkInlineFootnote + offset++ + } else if len(data)-1 > offset && data[offset+1] == '^' { + t = linkDeferredFootnote + } + default: + t = linkNormal + } + + data = data[offset:] + + if t == linkCitation { + return citation(p, data, 0) + } + + var ( + i = 1 + noteID int + title, link, linkID, altContent []byte + textHasNl = false + ) + + if t == linkDeferredFootnote { + i++ + } + + // look for the matching closing bracket + for level := 1; level > 0 && i < len(data); i++ { + switch { + case data[i] == '\n': + textHasNl = true + + case data[i-1] == '\\': + continue + + case data[i] == '[': + level++ + + case data[i] == ']': + level-- + if level <= 0 { + i-- // compensate for extra i++ in for loop + } + } + } + + if i >= len(data) { + return 0, nil + } + + txtE := i + i++ + var footnoteNode ast.Node + + // skip any amount of whitespace or newline + // (this is much more lax than original markdown syntax) + i = skipSpace(data, i) + + // inline style link + switch { + case i < len(data) && data[i] == '(': + // skip initial whitespace + i++ + + i = skipSpace(data, i) + + linkB := i + + // look for link end: ' " ) + findlinkend: + for i < len(data) { + switch { + case data[i] == '\\': + i += 2 + + case data[i] == ')' || data[i] == '\'' || data[i] == '"': + break findlinkend + + default: + i++ + } + } + + if i >= len(data) { + return 0, nil + } + linkE := i + + // look for title end if present + titleB, titleE := 0, 0 + if data[i] == '\'' || data[i] == '"' { + i++ + titleB = i + + findtitleend: + for i < len(data) { + switch { + case data[i] == '\\': + i += 2 + + case data[i] == ')': + break findtitleend + + default: + i++ + } + } + + if i >= len(data) { + return 0, nil + } + + // skip whitespace after title + titleE = i - 1 + for titleE > titleB && isSpace(data[titleE]) { + titleE-- + } + + // check for closing quote presence + if data[titleE] != '\'' && data[titleE] != '"' { + titleB, titleE = 0, 0 + linkE = i + } + } + + // remove whitespace at the end of the link + for linkE > linkB && isSpace(data[linkE-1]) { + linkE-- + } + + // remove optional angle brackets around the link + if data[linkB] == '<' { + linkB++ + } + if data[linkE-1] == '>' { + linkE-- + } + + // build escaped link and title + if linkE > linkB { + link = data[linkB:linkE] + } + + if titleE > titleB { + title = data[titleB:titleE] + } + + i++ + + // reference style link + case isReferenceStyleLink(data, i, t): + var id []byte + altContentConsidered := false + + // look for the id + i++ + linkB := i + i = skipUntilChar(data, i, ']') + + if i >= len(data) { + return 0, nil + } + linkE := i + + // find the reference + if linkB == linkE { + if textHasNl { + var b bytes.Buffer + + for j := 1; j < txtE; j++ { + switch { + case data[j] != '\n': + b.WriteByte(data[j]) + case data[j-1] != ' ': + b.WriteByte(' ') + } + } + + id = b.Bytes() + } else { + id = data[1:txtE] + altContentConsidered = true + } + } else { + id = data[linkB:linkE] + } + + // find the reference with matching id + lr, ok := p.getRef(string(id)) + if !ok { + return 0, nil + } + + // keep link and title from reference + linkID = id + link = lr.link + title = lr.title + if altContentConsidered { + altContent = lr.text + } + i++ + + // shortcut reference style link or reference or inline footnote + default: + var id []byte + + // craft the id + if textHasNl { + var b bytes.Buffer + + for j := 1; j < txtE; j++ { + switch { + case data[j] != '\n': + b.WriteByte(data[j]) + case data[j-1] != ' ': + b.WriteByte(' ') + } + } + + id = b.Bytes() + } else { + if t == linkDeferredFootnote { + id = data[2:txtE] // get rid of the ^ + } else { + id = data[1:txtE] + } + } + + footnoteNode = &ast.ListItem{} + if t == linkInlineFootnote { + // create a new reference + noteID = len(p.notes) + 1 + + var fragment []byte + if len(id) > 0 { + if len(id) < 16 { + fragment = make([]byte, len(id)) + } else { + fragment = make([]byte, 16) + } + copy(fragment, slugify(id)) + } else { + fragment = append([]byte("footnote-"), []byte(strconv.Itoa(noteID))...) + } + + ref := &reference{ + noteID: noteID, + hasBlock: false, + link: fragment, + title: id, + footnote: footnoteNode, + } + + p.notes = append(p.notes, ref) + p.refsRecord[string(ref.link)] = struct{}{} + + link = ref.link + title = ref.title + } else { + // find the reference with matching id + lr, ok := p.getRef(string(id)) + if !ok { + return 0, nil + } + + if t == linkDeferredFootnote && !p.isFootnote(lr) { + lr.noteID = len(p.notes) + 1 + lr.footnote = footnoteNode + p.notes = append(p.notes, lr) + p.refsRecord[string(lr.link)] = struct{}{} + } + + // keep link and title from reference + link = lr.link + // if inline footnote, title == footnote contents + title = lr.title + noteID = lr.noteID + } + + // rewind the whitespace + i = txtE + 1 + } + + var uLink []byte + if t == linkNormal || t == linkImg { + if len(link) > 0 { + var uLinkBuf bytes.Buffer + unescapeText(&uLinkBuf, link) + uLink = uLinkBuf.Bytes() + } + + // links need something to click on and somewhere to go + if len(uLink) == 0 || (t == linkNormal && txtE <= 1) { + return 0, nil + } + } + + // call the relevant rendering function + switch t { + case linkNormal: + link := &ast.Link{ + Destination: normalizeURI(uLink), + Title: title, + DeferredID: linkID, + } + if len(altContent) > 0 { + ast.AppendChild(link, newTextNode(altContent)) + } else { + // links cannot contain other links, so turn off link parsing + // temporarily and recurse + insideLink := p.insideLink + p.insideLink = true + p.Inline(link, data[1:txtE]) + p.insideLink = insideLink + } + return i, link + + case linkImg: + image := &ast.Image{ + Destination: uLink, + Title: title, + } + ast.AppendChild(image, newTextNode(data[1:txtE])) + return i + 1, image + + case linkInlineFootnote, linkDeferredFootnote: + link := &ast.Link{ + Destination: link, + Title: title, + NoteID: noteID, + Footnote: footnoteNode, + } + if t == linkDeferredFootnote { + link.DeferredID = data[2:txtE] + } + if t == linkInlineFootnote { + i++ + } + return i, link + + default: + return 0, nil + } +} + +func stripMailto(link []byte) []byte { + if bytes.HasPrefix(link, []byte("mailto://")) { + return link[9:] + } else if bytes.HasPrefix(link, []byte("mailto:")) { + return link[7:] + } else { + return link + } +} + +// autolinkType specifies a kind of autolink that gets detected. +type autolinkType int + +// These are the possible flag values for the autolink renderer. +const ( + notAutolink autolinkType = iota + normalAutolink + emailAutolink +) + +// '\\' backslash escape +var escapeChars = []byte("\\`*_{}[]()#+-.!:|&<>~") + +func escape(p *Parser, data []byte, offset int) (int, ast.Node) { + data = data[offset:] + + if len(data) <= 1 { + return 2, nil + } + + if p.extensions&NonBlockingSpace != 0 && data[1] == ' ' { + return 2, &ast.NonBlockingSpace{} + } + + if p.extensions&BackslashLineBreak != 0 && data[1] == '\n' { + return 2, &ast.Hardbreak{} + } + + if bytes.IndexByte(escapeChars, data[1]) < 0 { + return 0, nil + } + + return 2, newTextNode(data[1:2]) +} + +func unescapeText(ob *bytes.Buffer, src []byte) { + i := 0 + for i < len(src) { + org := i + for i < len(src) && src[i] != '\\' { + i++ + } + + if i > org { + ob.Write(src[org:i]) + } + + if i+1 >= len(src) { + break + } + + ob.WriteByte(src[i+1]) + i += 2 + } +} + +func linkEndsWithEntity(data []byte, linkEnd int) bool { + entityRanges := htmlEntityRe.FindAllIndex(data[:linkEnd], -1) + return entityRanges != nil && entityRanges[len(entityRanges)-1][1] == linkEnd +} + +// hasPrefixCaseInsensitive is a custom implementation of +// strings.HasPrefix(strings.ToLower(s), prefix) +// we rolled our own because ToLower pulls in a huge machinery of lowercasing +// anything from Unicode and that's very slow. Since this func will only be +// used on ASCII protocol prefixes, we can take shortcuts. +func hasPrefixCaseInsensitive(s, prefix []byte) bool { + if len(s) < len(prefix) { + return false + } + delta := byte('a' - 'A') + for i, b := range prefix { + if b != s[i] && b != s[i]+delta { + return false + } + } + return true +} + +var protocolPrefixes = [][]byte{ + []byte("http://"), + []byte("https://"), + []byte("ftp://"), + []byte("file://"), + []byte("mailto:"), +} + +const shortestPrefix = 6 // len("ftp://"), the shortest of the above + +func maybeAutoLink(p *Parser, data []byte, offset int) (int, ast.Node) { + // quick check to rule out most false hits + if p.insideLink || len(data) < offset+shortestPrefix { + return 0, nil + } + for _, prefix := range protocolPrefixes { + endOfHead := offset + 8 // 8 is the len() of the longest prefix + if endOfHead > len(data) { + endOfHead = len(data) + } + if hasPrefixCaseInsensitive(data[offset:endOfHead], prefix) { + return autoLink(p, data, offset) + } + } + return 0, nil +} + +func autoLink(p *Parser, data []byte, offset int) (int, ast.Node) { + // Now a more expensive check to see if we're not inside an anchor element + anchorStart := offset + offsetFromAnchor := 0 + for anchorStart > 0 && data[anchorStart] != '<' { + anchorStart-- + offsetFromAnchor++ + } + + anchorStr := anchorRe.Find(data[anchorStart:]) + if anchorStr != nil { + anchorClose := &ast.HTMLSpan{} + anchorClose.Literal = anchorStr[offsetFromAnchor:] + return len(anchorStr) - offsetFromAnchor, anchorClose + } + + // scan backward for a word boundary + rewind := 0 + for offset-rewind > 0 && rewind <= 7 && isLetter(data[offset-rewind-1]) { + rewind++ + } + if rewind > 6 { // longest supported protocol is "mailto" which has 6 letters + return 0, nil + } + + origData := data + data = data[offset-rewind:] + + if !isSafeLink(data) { + return 0, nil + } + + linkEnd := 0 + for linkEnd < len(data) && !isEndOfLink(data[linkEnd]) { + linkEnd++ + } + + // Skip punctuation at the end of the link + if (data[linkEnd-1] == '.' || data[linkEnd-1] == ',') && data[linkEnd-2] != '\\' { + linkEnd-- + } + + // But don't skip semicolon if it's a part of escaped entity: + if data[linkEnd-1] == ';' && data[linkEnd-2] != '\\' && !linkEndsWithEntity(data, linkEnd) { + linkEnd-- + } + + // See if the link finishes with a punctuation sign that can be closed. + var copen byte + switch data[linkEnd-1] { + case '"': + copen = '"' + case '\'': + copen = '\'' + case ')': + copen = '(' + case ']': + copen = '[' + case '}': + copen = '{' + default: + copen = 0 + } + + if copen != 0 { + bufEnd := offset - rewind + linkEnd - 2 + + openDelim := 1 + + /* Try to close the final punctuation sign in this same line; + * if we managed to close it outside of the URL, that means that it's + * not part of the URL. If it closes inside the URL, that means it + * is part of the URL. + * + * Examples: + * + * foo http://www.pokemon.com/Pikachu_(Electric) bar + * => http://www.pokemon.com/Pikachu_(Electric) + * + * foo (http://www.pokemon.com/Pikachu_(Electric)) bar + * => http://www.pokemon.com/Pikachu_(Electric) + * + * foo http://www.pokemon.com/Pikachu_(Electric)) bar + * => http://www.pokemon.com/Pikachu_(Electric)) + * + * (foo http://www.pokemon.com/Pikachu_(Electric)) bar + * => foo http://www.pokemon.com/Pikachu_(Electric) + */ + + for bufEnd >= 0 && origData[bufEnd] != '\n' && openDelim != 0 { + if origData[bufEnd] == data[linkEnd-1] { + openDelim++ + } + + if origData[bufEnd] == copen { + openDelim-- + } + + bufEnd-- + } + + if openDelim == 0 { + linkEnd-- + } + } + + var uLink bytes.Buffer + unescapeText(&uLink, data[:linkEnd]) + + if uLink.Len() > 0 { + node := &ast.Link{ + Destination: uLink.Bytes(), + } + ast.AppendChild(node, newTextNode(uLink.Bytes())) + return linkEnd, node + } + + return linkEnd, nil +} + +func isEndOfLink(char byte) bool { + return isSpace(char) || char == '<' +} + +var validUris = [][]byte{[]byte("http://"), []byte("https://"), []byte("ftp://"), []byte("mailto://")} +var validPaths = [][]byte{[]byte("/"), []byte("./"), []byte("../")} + +func isSafeLink(link []byte) bool { + nLink := len(link) + for _, path := range validPaths { + nPath := len(path) + linkPrefix := link[:nPath] + if nLink >= nPath && bytes.Equal(linkPrefix, path) { + if nLink == nPath { + return true + } else if isAlnum(link[nPath]) { + return true + } + } + } + + for _, prefix := range validUris { + // TODO: handle unicode here + // case-insensitive prefix test + nPrefix := len(prefix) + if nLink > nPrefix { + linkPrefix := bytes.ToLower(link[:nPrefix]) + if bytes.Equal(linkPrefix, prefix) && isAlnum(link[nPrefix]) { + return true + } + } + } + + return false +} + +// return the length of the given tag, or 0 is it's not valid +func tagLength(data []byte) (autolink autolinkType, end int) { + var i, j int + + // a valid tag can't be shorter than 3 chars + if len(data) < 3 { + return notAutolink, 0 + } + + // begins with a '<' optionally followed by '/', followed by letter or number + if data[0] != '<' { + return notAutolink, 0 + } + if data[1] == '/' { + i = 2 + } else { + i = 1 + } + + if !isAlnum(data[i]) { + return notAutolink, 0 + } + + // scheme test + autolink = notAutolink + + // try to find the beginning of an URI + for i < len(data) && (isAlnum(data[i]) || data[i] == '.' || data[i] == '+' || data[i] == '-') { + i++ + } + + if i > 1 && i < len(data) && data[i] == '@' { + if j = isMailtoAutoLink(data[i:]); j != 0 { + return emailAutolink, i + j + } + } + + if i > 2 && i < len(data) && data[i] == ':' { + autolink = normalAutolink + i++ + } + + // complete autolink test: no whitespace or ' or " + switch { + case i >= len(data): + autolink = notAutolink + case autolink != notAutolink: + j = i + + for i < len(data) { + if data[i] == '\\' { + i += 2 + } else if data[i] == '>' || data[i] == '\'' || data[i] == '"' || isSpace(data[i]) { + break + } else { + i++ + } + + } + + if i >= len(data) { + return autolink, 0 + } + if i > j && data[i] == '>' { + return autolink, i + 1 + } + + // one of the forbidden chars has been found + autolink = notAutolink + } + i += bytes.IndexByte(data[i:], '>') + if i < 0 { + return autolink, 0 + } + return autolink, i + 1 +} + +// look for the address part of a mail autolink and '>' +// this is less strict than the original markdown e-mail address matching +func isMailtoAutoLink(data []byte) int { + nb := 0 + + // address is assumed to be: [-@._a-zA-Z0-9]+ with exactly one '@' + for i, c := range data { + if isAlnum(c) { + continue + } + + switch c { + case '@': + nb++ + + case '-', '.', '_': + break + + case '>': + if nb == 1 { + return i + 1 + } + return 0 + default: + return 0 + } + } + + return 0 +} + +// look for the next emph char, skipping other constructs +func helperFindEmphChar(data []byte, c byte) int { + i := 0 + + for i < len(data) { + for i < len(data) && data[i] != c && data[i] != '`' && data[i] != '[' { + i++ + } + if i >= len(data) { + return 0 + } + // do not count escaped chars + if i != 0 && data[i-1] == '\\' { + i++ + continue + } + if data[i] == c { + return i + } + + if data[i] == '`' { + // skip a code span + tmpI := 0 + i++ + for i < len(data) && data[i] != '`' { + if tmpI == 0 && data[i] == c { + tmpI = i + } + i++ + } + if i >= len(data) { + return tmpI + } + i++ + } else if data[i] == '[' { + // skip a link + tmpI := 0 + i++ + for i < len(data) && data[i] != ']' { + if tmpI == 0 && data[i] == c { + tmpI = i + } + i++ + } + i++ + for i < len(data) && (data[i] == ' ' || data[i] == '\n') { + i++ + } + if i >= len(data) { + return tmpI + } + if data[i] != '[' && data[i] != '(' { // not a link + if tmpI > 0 { + return tmpI + } + continue + } + cc := data[i] + i++ + for i < len(data) && data[i] != cc { + if tmpI == 0 && data[i] == c { + return i + } + i++ + } + if i >= len(data) { + return tmpI + } + i++ + } + } + return 0 +} + +func helperEmphasis(p *Parser, data []byte, c byte) (int, ast.Node) { + i := 0 + + // skip one symbol if coming from emph3 + if len(data) > 1 && data[0] == c && data[1] == c { + i = 1 + } + + for i < len(data) { + length := helperFindEmphChar(data[i:], c) + if length == 0 { + return 0, nil + } + i += length + if i >= len(data) { + return 0, nil + } + + if i+1 < len(data) && data[i+1] == c { + i++ + continue + } + + if data[i] == c && !isSpace(data[i-1]) { + + if p.extensions&NoIntraEmphasis != 0 { + if !(i+1 == len(data) || isSpace(data[i+1]) || isPunctuation(data[i+1])) { + continue + } + } + + emph := &ast.Emph{} + emph.Literal = data[:i] + return i + 1, emph + } + } + + return 0, nil +} + +func helperDoubleEmphasis(p *Parser, data []byte, c byte) (int, ast.Node) { + i := 0 + + for i < len(data) { + length := helperFindEmphChar(data[i:], c) + if length == 0 { + return 0, nil + } + i += length + + if i+1 < len(data) && data[i] == c && data[i+1] == c && i > 0 && !isSpace(data[i-1]) { + strong := &ast.Strong{} + strong.Literal = data[:i] + var node ast.Node = strong + if c == '~' { + del := &ast.Del{} + del.Literal = data[:i] + node = del + } + return i + 2, node + } + i++ + } + return 0, nil +} + +func helperTripleEmphasis(p *Parser, data []byte, offset int, c byte) (int, ast.Node) { + i := 0 + origData := data + data = data[offset:] + + for i < len(data) { + length := helperFindEmphChar(data[i:], c) + if length == 0 { + return 0, nil + } + i += length + + // skip whitespace preceded symbols + if data[i] != c || isSpace(data[i-1]) { + continue + } + + switch { + case i+2 < len(data) && data[i+1] == c && data[i+2] == c: + // triple symbol found + strong := &ast.Strong{} + strong.Literal = data[:i] + return i + 3, strong + case i+1 < len(data) && data[i+1] == c: + // double symbol found, hand over to emph1 + length, node := helperEmphasis(p, origData[offset-2:], c) + if length == 0 { + return 0, nil + } + return length - 2, node + default: + // single symbol found, hand over to emph2 + length, node := helperDoubleEmphasis(p, origData[offset-1:], c) + if length == 0 { + return 0, nil + } + return length - 1, node + } + } + return 0, nil +} + +// math handle inline math wrapped with '$' +func math(p *Parser, data []byte, offset int) (int, ast.Node) { + data = data[offset:] + + // too short, or block math + if len(data) <= 2 || data[1] == '$' { + return 0, nil + } + + // find next '$' + var end int + for end = 1; end < len(data) && data[end] != '$'; end++ { + } + + // $ not match + if end == len(data) { + return 0, nil + } + + // create inline math node + math := &ast.Math{} + math.Literal = data[1:end] + return end + 1, math +} + +func newTextNode(d []byte) *ast.Text { + return &ast.Text{ast.Leaf{Literal: d}} +} + +func normalizeURI(s []byte) []byte { + return s // TODO: implement +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/matter.go b/vendor/github.com/gomarkdown/markdown/parser/matter.go new file mode 100644 index 000000000..926863572 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/matter.go @@ -0,0 +1,36 @@ +package parser + +import ( + "bytes" + + "github.com/gomarkdown/markdown/ast" +) + +func (p *Parser) documentMatter(data []byte) int { + if data[0] != '{' { + return 0 + } + + consumed := 0 + matter := ast.DocumentMatterNone + if bytes.HasPrefix(data, []byte("{frontmatter}")) { + consumed = len("{frontmatter}") + matter = ast.DocumentMatterFront + } + if bytes.HasPrefix(data, []byte("{mainmatter}")) { + consumed = len("{mainmatter}") + matter = ast.DocumentMatterMain + } + if bytes.HasPrefix(data, []byte("{backmatter}")) { + consumed = len("{backmatter}") + matter = ast.DocumentMatterBack + } + if consumed == 0 { + return 0 + } + node := &ast.DocumentMatter{Matter: matter} + p.addBlock(node) + p.finalize(node) + + return consumed +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/options.go b/vendor/github.com/gomarkdown/markdown/parser/options.go new file mode 100644 index 000000000..d3d0c0887 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/options.go @@ -0,0 +1,32 @@ +package parser + +import ( + "github.com/gomarkdown/markdown/ast" +) + +// Flags control optional behavior of parser. +type Flags int + +// Options is a collection of supplementary parameters tweaking the behavior of various parts of the parser. +type Options struct { + ParserHook BlockFunc + ReadIncludeFn ReadIncludeFunc + + Flags Flags // Flags allow customizing parser's behavior +} + +// Parser renderer configuration options. +const ( + FlagsNone Flags = 0 + SkipFootnoteList Flags = 1 << iota // Skip adding the footnote list (regardless if they are parsed) +) + +// BlockFunc allows to registration of a parser function. If successful it +// returns an ast.Node, a buffer that should be parsed as a block and the the number of bytes consumed. +type BlockFunc func(data []byte) (ast.Node, []byte, int) + +// ReadIncludeFunc should read the file under path and returns the read bytes, +// from will be set to the name of the current file being parsed. Initially +// this will be empty. address is the optional address specifier of which lines +// of the file to return. If this function is not set no data will be read. +type ReadIncludeFunc func(from, path string, address []byte) []byte diff --git a/vendor/github.com/gomarkdown/markdown/parser/parser.go b/vendor/github.com/gomarkdown/markdown/parser/parser.go new file mode 100644 index 000000000..b87d5b605 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/parser.go @@ -0,0 +1,803 @@ +/* +Package parser implements parser for markdown text that generates AST (abstract syntax tree). +*/ +package parser + +import ( + "bytes" + "fmt" + "strings" + "unicode/utf8" + + "github.com/gomarkdown/markdown/ast" +) + +// Extensions is a bitmask of enabled parser extensions. +type Extensions int + +// Bit flags representing markdown parsing extensions. +// Use | (or) to specify multiple extensions. +const ( + NoExtensions Extensions = 0 + NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words + Tables // Parse tables + FencedCode // Parse fenced code blocks + Autolink // Detect embedded URLs that are not explicitly marked + Strikethrough // Strikethrough text using ~~test~~ + LaxHTMLBlocks // Loosen up HTML block parsing rules + SpaceHeadings // Be strict about prefix heading rules + HardLineBreak // Translate newlines into line breaks + NonBlockingSpace // Translate backspace spaces into line non-blocking spaces + TabSizeEight // Expand tabs to eight spaces instead of four + Footnotes // Pandoc-style footnotes + NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block + HeadingIDs // specify heading IDs with {#id} + Titleblock // Titleblock ala pandoc + AutoHeadingIDs // Create the heading ID from the text + BackslashLineBreak // Translate trailing backslashes into line breaks + DefinitionLists // Parse definition lists + MathJax // Parse MathJax + OrderedListStart // Keep track of the first number used when starting an ordered list. + Attributes // Block Attributes + SuperSubscript // Super- and subscript support: 2^10^, H~2~O. + EmptyLinesBreakList // 2 empty lines break out of list + Includes // Support including other files. + Mmark // Support Mmark syntax, see https://mmark.nl/syntax + + CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode | + Autolink | Strikethrough | SpaceHeadings | HeadingIDs | + BackslashLineBreak | DefinitionLists | MathJax +) + +// The size of a tab stop. +const ( + tabSizeDefault = 4 + tabSizeDouble = 8 +) + +// for each character that triggers a response when parsing inline data. +type inlineParser func(p *Parser, data []byte, offset int) (int, ast.Node) + +// ReferenceOverrideFunc is expected to be called with a reference string and +// return either a valid Reference type that the reference string maps to or +// nil. If overridden is false, the default reference logic will be executed. +// See the documentation in Options for more details on use-case. +type ReferenceOverrideFunc func(reference string) (ref *Reference, overridden bool) + +// Parser is a type that holds extensions and the runtime state used by +// Parse, and the renderer. You can not use it directly, construct it with New. +type Parser struct { + + // ReferenceOverride is an optional function callback that is called every + // time a reference is resolved. It can be set before starting parsing. + // + // In Markdown, the link reference syntax can be made to resolve a link to + // a reference instead of an inline URL, in one of the following ways: + // + // * [link text][refid] + // * [refid][] + // + // Usually, the refid is defined at the bottom of the Markdown document. If + // this override function is provided, the refid is passed to the override + // function first, before consulting the defined refids at the bottom. If + // the override function indicates an override did not occur, the refids at + // the bottom will be used to fill in the link details. + ReferenceOverride ReferenceOverrideFunc + + Opts Options + + // after parsing, this is AST root of parsed markdown text + Doc ast.Node + + extensions Extensions + + refs map[string]*reference + refsRecord map[string]struct{} + inlineCallback [256]inlineParser + nesting int + maxNesting int + insideLink bool + indexCnt int // incremented after every index + + // Footnotes need to be ordered as well as available to quickly check for + // presence. If a ref is also a footnote, it's stored both in refs and here + // in notes. Slice is nil if footnotes not enabled. + notes []*reference + + tip ast.Node // = doc + oldTip ast.Node + lastMatchedContainer ast.Node // = doc + allClosed bool + + // Attributes are attached to block level elements. + attr *ast.Attribute + + includeStack *incStack +} + +// New creates a markdown parser with CommonExtensions. +// +// You can then call `doc := p.Parse(markdown)` to parse markdown document +// and `markdown.Render(doc, renderer)` to convert it to another format with +// a renderer. +func New() *Parser { + return NewWithExtensions(CommonExtensions) +} + +// NewWithExtensions creates a markdown parser with given extensions. +func NewWithExtensions(extension Extensions) *Parser { + p := Parser{ + refs: make(map[string]*reference), + refsRecord: make(map[string]struct{}), + maxNesting: 1, + insideLink: false, + Doc: &ast.Document{}, + extensions: extension, + allClosed: true, + includeStack: newIncStack(), + } + p.tip = p.Doc + p.oldTip = p.Doc + p.lastMatchedContainer = p.Doc + + p.inlineCallback[' '] = maybeLineBreak + p.inlineCallback['*'] = emphasis + p.inlineCallback['#'] = statusTag + p.inlineCallback['_'] = emphasis + if p.extensions&Strikethrough != 0 { + p.inlineCallback['~'] = emphasis + } + p.inlineCallback['`'] = codeSpan + p.inlineCallback['\n'] = lineBreak + p.inlineCallback['\\'] = escape + if p.extensions&Autolink != 0 { + p.inlineCallback['h'] = maybeAutoLink + p.inlineCallback['m'] = maybeAutoLink + p.inlineCallback['f'] = maybeAutoLink + p.inlineCallback['H'] = maybeAutoLink + p.inlineCallback['M'] = maybeAutoLink + p.inlineCallback['F'] = maybeAutoLink + } + + return &p +} + +func (p *Parser) getRef(refid string) (ref *reference, found bool) { + if p.ReferenceOverride != nil { + r, overridden := p.ReferenceOverride(refid) + if overridden { + if r == nil { + return nil, false + } + return &reference{ + link: []byte(r.Link), + title: []byte(r.Title), + noteID: 0, + hasBlock: false, + text: []byte(r.Text)}, true + } + } + // refs are case insensitive + ref, found = p.refs[strings.ToLower(refid)] + return ref, found +} + +func (p *Parser) isFootnote(ref *reference) bool { + _, ok := p.refsRecord[string(ref.link)] + return ok +} + +func (p *Parser) finalize(block ast.Node) { + p.tip = block.GetParent() +} + +func (p *Parser) addChild(node ast.Node) ast.Node { + for !canNodeContain(p.tip, node) { + p.finalize(p.tip) + } + ast.AppendChild(p.tip, node) + p.tip = node + return node +} + +func canNodeContain(n ast.Node, v ast.Node) bool { + switch n.(type) { + case *ast.List: + return isListItem(v) + case *ast.Document, *ast.BlockQuote, *ast.Aside, *ast.ListItem, *ast.CaptionFigure: + return !isListItem(v) + case *ast.Table: + switch v.(type) { + case *ast.TableHeader, *ast.TableBody, *ast.TableFooter: + return true + default: + return false + } + case *ast.TableHeader, *ast.TableBody, *ast.TableFooter: + _, ok := v.(*ast.TableRow) + return ok + case *ast.TableRow: + _, ok := v.(*ast.TableCell) + return ok + } + return false +} + +func (p *Parser) closeUnmatchedBlocks() { + if p.allClosed { + return + } + for p.oldTip != p.lastMatchedContainer { + parent := p.oldTip.GetParent() + p.finalize(p.oldTip) + p.oldTip = parent + } + p.allClosed = true +} + +// Reference represents the details of a link. +// See the documentation in Options for more details on use-case. +type Reference struct { + // Link is usually the URL the reference points to. + Link string + // Title is the alternate text describing the link in more detail. + Title string + // Text is the optional text to override the ref with if the syntax used was + // [refid][] + Text string +} + +// Parse generates AST (abstract syntax tree) representing markdown document. +// +// The result is a root of the tree whose underlying type is *ast.Document +// +// You can then convert AST to html using html.Renderer, to some other format +// using a custom renderer or transform the tree. +func (p *Parser) Parse(input []byte) ast.Node { + p.block(input) + // Walk the tree and finish up some of unfinished blocks + for p.tip != nil { + p.finalize(p.tip) + } + // Walk the tree again and process inline markdown in each block + ast.WalkFunc(p.Doc, func(node ast.Node, entering bool) ast.WalkStatus { + switch node.(type) { + case *ast.Paragraph: + p.Inline(node, node.AsContainer().Content) + node.AsContainer().Content = nil + } + return ast.GoToNext + }) + + return p.Doc +} + +func (p *Parser) parseRefsToAST() { + if p.extensions&Footnotes == 0 || len(p.notes) == 0 { + return + } + p.tip = p.Doc + list := &ast.List{ + IsFootnotesList: true, + ListFlags: ast.ListTypeOrdered, + } + p.addBlock(&ast.Footnotes{}) + block := p.addBlock(list) + flags := ast.ListItemBeginningOfList + // Note: this loop is intentionally explicit, not range-form. This is + // because the body of the loop will append nested footnotes to p.notes and + // we need to process those late additions. Range form would only walk over + // the fixed initial set. + for i := 0; i < len(p.notes); i++ { + ref := p.notes[i] + p.addChild(ref.footnote) + block := ref.footnote + listItem := block.(*ast.ListItem) + listItem.ListFlags = flags | ast.ListTypeOrdered + listItem.RefLink = ref.link + if ref.hasBlock { + flags |= ast.ListItemContainsBlock + p.block(ref.title) + } else { + p.Inline(block, ref.title) + } + flags &^= ast.ListItemBeginningOfList | ast.ListItemContainsBlock + } + above := list.Parent + finalizeList(list) + p.tip = above + + ast.WalkFunc(block, func(node ast.Node, entering bool) ast.WalkStatus { + switch node.(type) { + case *ast.Paragraph, *ast.Heading: + p.Inline(node, node.AsContainer().Content) + node.AsContainer().Content = nil + } + return ast.GoToNext + }) +} + +// +// Link references +// +// This section implements support for references that (usually) appear +// as footnotes in a document, and can be referenced anywhere in the document. +// The basic format is: +// +// [1]: http://www.google.com/ "Google" +// [2]: http://www.github.com/ "Github" +// +// Anywhere in the document, the reference can be linked by referring to its +// label, i.e., 1 and 2 in this example, as in: +// +// This library is hosted on [Github][2], a git hosting site. +// +// Actual footnotes as specified in Pandoc and supported by some other Markdown +// libraries such as php-markdown are also taken care of. They look like this: +// +// This sentence needs a bit of further explanation.[^note] +// +// [^note]: This is the explanation. +// +// Footnotes should be placed at the end of the document in an ordered list. +// Inline footnotes such as: +// +// Inline footnotes^[Not supported.] also exist. +// +// are not yet supported. + +// reference holds all information necessary for a reference-style links or +// footnotes. +// +// Consider this markdown with reference-style links: +// +// [link][ref] +// +// [ref]: /url/ "tooltip title" +// +// It will be ultimately converted to this HTML: +// +//

link

+// +// And a reference structure will be populated as follows: +// +// p.refs["ref"] = &reference{ +// link: "/url/", +// title: "tooltip title", +// } +// +// Alternatively, reference can contain information about a footnote. Consider +// this markdown: +// +// Text needing a footnote.[^a] +// +// [^a]: This is the note +// +// A reference structure will be populated as follows: +// +// p.refs["a"] = &reference{ +// link: "a", +// title: "This is the note", +// noteID: , +// } +// +// TODO: As you can see, it begs for splitting into two dedicated structures +// for refs and for footnotes. +type reference struct { + link []byte + title []byte + noteID int // 0 if not a footnote ref + hasBlock bool + footnote ast.Node // a link to the Item node within a list of footnotes + + text []byte // only gets populated by refOverride feature with Reference.Text +} + +func (r *reference) String() string { + return fmt.Sprintf("{link: %q, title: %q, text: %q, noteID: %d, hasBlock: %v}", + r.link, r.title, r.text, r.noteID, r.hasBlock) +} + +// Check whether or not data starts with a reference link. +// If so, it is parsed and stored in the list of references +// (in the render struct). +// Returns the number of bytes to skip to move past it, +// or zero if the first line is not a reference. +func isReference(p *Parser, data []byte, tabSize int) int { + // up to 3 optional leading spaces + if len(data) < 4 { + return 0 + } + i := 0 + for i < 3 && data[i] == ' ' { + i++ + } + + noteID := 0 + + // id part: anything but a newline between brackets + if data[i] != '[' { + return 0 + } + i++ + if p.extensions&Footnotes != 0 { + if i < len(data) && data[i] == '^' { + // we can set it to anything here because the proper noteIds will + // be assigned later during the second pass. It just has to be != 0 + noteID = 1 + i++ + } + } + idOffset := i + for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' { + i++ + } + if i >= len(data) || data[i] != ']' { + return 0 + } + idEnd := i + // footnotes can have empty ID, like this: [^], but a reference can not be + // empty like this: []. Break early if it's not a footnote and there's no ID + if noteID == 0 && idOffset == idEnd { + return 0 + } + // spacer: colon (space | tab)* newline? (space | tab)* + i++ + if i >= len(data) || data[i] != ':' { + return 0 + } + i++ + for i < len(data) && (data[i] == ' ' || data[i] == '\t') { + i++ + } + if i < len(data) && (data[i] == '\n' || data[i] == '\r') { + i++ + if i < len(data) && data[i] == '\n' && data[i-1] == '\r' { + i++ + } + } + for i < len(data) && (data[i] == ' ' || data[i] == '\t') { + i++ + } + if i >= len(data) { + return 0 + } + + var ( + linkOffset, linkEnd int + titleOffset, titleEnd int + lineEnd int + raw []byte + hasBlock bool + ) + + if p.extensions&Footnotes != 0 && noteID != 0 { + linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize) + lineEnd = linkEnd + } else { + linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i) + } + if lineEnd == 0 { + return 0 + } + + // a valid ref has been found + + ref := &reference{ + noteID: noteID, + hasBlock: hasBlock, + } + + if noteID > 0 { + // reusing the link field for the id since footnotes don't have links + ref.link = data[idOffset:idEnd] + // if footnote, it's not really a title, it's the contained text + ref.title = raw + } else { + ref.link = data[linkOffset:linkEnd] + ref.title = data[titleOffset:titleEnd] + } + + // id matches are case-insensitive + id := string(bytes.ToLower(data[idOffset:idEnd])) + + p.refs[id] = ref + + return lineEnd +} + +func scanLinkRef(p *Parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) { + // link: whitespace-free sequence, optionally between angle brackets + if data[i] == '<' { + i++ + } + linkOffset = i + for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' { + i++ + } + linkEnd = i + if linkEnd < len(data) && data[linkOffset] == '<' && data[linkEnd-1] == '>' { + linkOffset++ + linkEnd-- + } + + // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' ) + for i < len(data) && (data[i] == ' ' || data[i] == '\t') { + i++ + } + if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' { + return + } + + // compute end-of-line + if i >= len(data) || data[i] == '\r' || data[i] == '\n' { + lineEnd = i + } + if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' { + lineEnd++ + } + + // optional (space|tab)* spacer after a newline + if lineEnd > 0 { + i = lineEnd + 1 + for i < len(data) && (data[i] == ' ' || data[i] == '\t') { + i++ + } + } + + // optional title: any non-newline sequence enclosed in '"() alone on its line + if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') { + i++ + titleOffset = i + + // look for EOL + for i < len(data) && data[i] != '\n' && data[i] != '\r' { + i++ + } + if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' { + titleEnd = i + 1 + } else { + titleEnd = i + } + + // step back + i-- + for i > titleOffset && (data[i] == ' ' || data[i] == '\t') { + i-- + } + if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') { + lineEnd = titleEnd + titleEnd = i + } + } + + return +} + +// The first bit of this logic is the same as Parser.listItem, but the rest +// is much simpler. This function simply finds the entire block and shifts it +// over by one tab if it is indeed a block (just returns the line if it's not). +// blockEnd is the end of the section in the input buffer, and contents is the +// extracted text that was shifted over one tab. It will need to be rendered at +// the end of the document. +func scanFootnote(p *Parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) { + if i == 0 || len(data) == 0 { + return + } + + // skip leading whitespace on first line + for i < len(data) && data[i] == ' ' { + i++ + } + + blockStart = i + + // find the end of the line + blockEnd = i + for i < len(data) && data[i-1] != '\n' { + i++ + } + + // get working buffer + var raw bytes.Buffer + + // put the first line into the working buffer + raw.Write(data[blockEnd:i]) + blockEnd = i + + // process the following lines + containsBlankLine := false + +gatherLines: + for blockEnd < len(data) { + i++ + + // find the end of this line + for i < len(data) && data[i-1] != '\n' { + i++ + } + + // if it is an empty line, guess that it is part of this item + // and move on to the next line + if p.isEmpty(data[blockEnd:i]) > 0 { + containsBlankLine = true + blockEnd = i + continue + } + + n := 0 + if n = isIndented(data[blockEnd:i], indentSize); n == 0 { + // this is the end of the block. + // we don't want to include this last line in the index. + break gatherLines + } + + // if there were blank lines before this one, insert a new one now + if containsBlankLine { + raw.WriteByte('\n') + containsBlankLine = false + } + + // get rid of that first tab, write to buffer + raw.Write(data[blockEnd+n : i]) + hasBlock = true + + blockEnd = i + } + + if data[blockEnd-1] != '\n' { + raw.WriteByte('\n') + } + + contents = raw.Bytes() + + return +} + +// isPunctuation returns true if c is a punctuation symbol. +func isPunctuation(c byte) bool { + for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") { + if c == r { + return true + } + } + return false +} + +// isSpace returns true if c is a white-space charactr +func isSpace(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v' +} + +// isLetter returns true if c is ascii letter +func isLetter(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +// isAlnum returns true if c is a digit or letter +// TODO: check when this is looking for ASCII alnum and when it should use unicode +func isAlnum(c byte) bool { + return (c >= '0' && c <= '9') || isLetter(c) +} + +func isValidStatusTagChar(c byte) bool { + return isAlnum(c) || c == '-' +} + +// TODO: this is not used +// Replace tab characters with spaces, aligning to the next TAB_SIZE column. +// always ends output with a newline +func expandTabs(out *bytes.Buffer, line []byte, tabSize int) { + // first, check for common cases: no tabs, or only tabs at beginning of line + i, prefix := 0, 0 + slowcase := false + for i = 0; i < len(line); i++ { + if line[i] == '\t' { + if prefix == i { + prefix++ + } else { + slowcase = true + break + } + } + } + + // no need to decode runes if all tabs are at the beginning of the line + if !slowcase { + for i = 0; i < prefix*tabSize; i++ { + out.WriteByte(' ') + } + out.Write(line[prefix:]) + return + } + + // the slow case: we need to count runes to figure out how + // many spaces to insert for each tab + column := 0 + i = 0 + for i < len(line) { + start := i + for i < len(line) && line[i] != '\t' { + _, size := utf8.DecodeRune(line[i:]) + i += size + column++ + } + + if i > start { + out.Write(line[start:i]) + } + + if i >= len(line) { + break + } + + for { + out.WriteByte(' ') + column++ + if column%tabSize == 0 { + break + } + } + + i++ + } +} + +// Find if a line counts as indented or not. +// Returns number of characters the indent is (0 = not indented). +func isIndented(data []byte, indentSize int) int { + if len(data) == 0 { + return 0 + } + if data[0] == '\t' { + return 1 + } + if len(data) < indentSize { + return 0 + } + for i := 0; i < indentSize; i++ { + if data[i] != ' ' { + return 0 + } + } + return indentSize +} + +// Create a url-safe slug for fragments +func slugify(in []byte) []byte { + if len(in) == 0 { + return in + } + out := make([]byte, 0, len(in)) + sym := false + + for _, ch := range in { + if isAlnum(ch) { + sym = false + out = append(out, ch) + } else if sym { + continue + } else { + out = append(out, '-') + sym = true + } + } + var a, b int + var ch byte + for a, ch = range out { + if ch != '-' { + break + } + } + for b = len(out) - 1; b > 0; b-- { + if out[b] != '-' { + break + } + } + return out[a : b+1] +} + +func isListItem(d ast.Node) bool { + _, ok := d.(*ast.ListItem) + return ok +} diff --git a/vendor/github.com/gomarkdown/markdown/parser/ref.go b/vendor/github.com/gomarkdown/markdown/parser/ref.go new file mode 100644 index 000000000..0b59a196d --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/parser/ref.go @@ -0,0 +1,89 @@ +package parser + +import ( + "bytes" + "fmt" + + "github.com/gomarkdown/markdown/ast" +) + +// parse '(#r)', where r does not contain spaces. Or. +// (!item) (!item, subitem), for an index, (!!item) signals primary. +func maybeShortRefOrIndex(p *Parser, data []byte, offset int) (int, ast.Node) { + if len(data[offset:]) < 4 { + return 0, nil + } + // short ref first + data = data[offset:] + i := 1 + switch data[i] { + case '#': // cross ref + i++ + Loop: + for i < len(data) { + c := data[i] + switch { + case c == ')': + break Loop + case !isAlnum(c): + if c == '_' || c == '-' || c == ':' { + i++ + continue + } + i = 0 + break Loop + } + i++ + } + if i >= len(data) { + return 0, nil + } + if data[i] != ')' { + return 0, nil + } + + id := data[2:i] + node := &ast.CrossReference{} + node.Destination = id + + return i + 1, node + + case '!': // index + i++ + start := i + i = skipUntilChar(data, start, ')') + + // did we reach the end of the buffer without a closing marker? + if i >= len(data) { + return 0, nil + } + + if len(data[start:i]) < 1 { + return 0, nil + } + + idx := &ast.Index{} + + idx.ID = fmt.Sprintf("idxref:%d", p.indexCnt) + p.indexCnt++ + + idx.Primary = data[start] == '!' + buf := data[start:i] + + if idx.Primary { + buf = buf[1:] + } + items := bytes.Split(buf, []byte(",")) + switch len(items) { + case 1: + idx.Item = bytes.TrimSpace(items[0]) + return i + 1, idx + case 2: + idx.Item = bytes.TrimSpace(items[0]) + idx.Subitem = bytes.TrimSpace(items[1]) + return i + 1, idx + } + } + + return 0, nil +} diff --git a/vendor/github.com/gomarkdown/markdown/todo.md b/vendor/github.com/gomarkdown/markdown/todo.md new file mode 100644 index 000000000..be8bb55c6 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/todo.md @@ -0,0 +1,7 @@ +# Things to do + +[ ] docs: add examples like https://godoc.org/github.com/dgrijalva/jwt-go (put in foo_example_test.go). Or see https://github.com/garyburd/redigo/blob/master/redis/zpop_example_test.go#L5 / https://godoc.org/github.com/garyburd/redigo/redis or https://godoc.org/github.com/go-redis/redis + +[ ] figure out expandTabs and parser.TabSizeEight. Are those used? + +[ ] SoftbreakData is not used diff --git a/vendor/github.com/gomarkdown/markdown/tracking-perf.md b/vendor/github.com/gomarkdown/markdown/tracking-perf.md new file mode 100644 index 000000000..40b951830 --- /dev/null +++ b/vendor/github.com/gomarkdown/markdown/tracking-perf.md @@ -0,0 +1,189 @@ +## Tracking perf changes + +Initial performance: +``` +goos: darwin +goarch: amd64 +pkg: github.com/gomarkdown/markdown +BenchmarkEscapeHTML-8 2000000 823 ns/op 0 B/op 0 allocs/op +BenchmarkSmartDoubleQuotes-8 300000 5033 ns/op 9872 B/op 56 allocs/op +BenchmarkReferenceAmps-8 100000 19538 ns/op 26776 B/op 150 allocs/op +BenchmarkReferenceAutoLinks-8 100000 17574 ns/op 24544 B/op 132 allocs/op +BenchmarkReferenceBackslashEscapes-8 30000 50977 ns/op 76752 B/op 243 allocs/op +BenchmarkReferenceBlockquotesWithCodeBlocks-8 200000 8546 ns/op 12864 B/op 65 allocs/op +BenchmarkReferenceCodeBlocks-8 200000 9000 ns/op 14912 B/op 70 allocs/op +BenchmarkReferenceCodeSpans-8 200000 8856 ns/op 14992 B/op 69 allocs/op +BenchmarkReferenceHardWrappedPara-8 200000 6599 ns/op 11312 B/op 57 allocs/op +BenchmarkReferenceHorizontalRules-8 100000 15483 ns/op 23536 B/op 98 allocs/op +BenchmarkReferenceInlineHTMLAdvances-8 200000 6839 ns/op 12150 B/op 62 allocs/op +BenchmarkReferenceInlineHTMLSimple-8 100000 19940 ns/op 28488 B/op 117 allocs/op +BenchmarkReferenceInlineHTMLComments-8 200000 7455 ns/op 13440 B/op 64 allocs/op +BenchmarkReferenceLinksInline-8 100000 16425 ns/op 23664 B/op 147 allocs/op +BenchmarkReferenceLinksReference-8 30000 54895 ns/op 66464 B/op 416 allocs/op +BenchmarkReferenceLinksShortcut-8 100000 17647 ns/op 23776 B/op 158 allocs/op +BenchmarkReferenceLiterQuotesInTitles-8 200000 9367 ns/op 14832 B/op 95 allocs/op +BenchmarkReferenceMarkdownBasics-8 10000 129772 ns/op 130848 B/op 378 allocs/op +BenchmarkReferenceMarkdownSyntax-8 3000 502365 ns/op 461411 B/op 1411 allocs/op +BenchmarkReferenceNestedBlockquotes-8 200000 7028 ns/op 12688 B/op 64 allocs/op +BenchmarkReferenceOrderedAndUnorderedLists-8 20000 79686 ns/op 107520 B/op 374 allocs/op +BenchmarkReferenceStrongAndEm-8 200000 10020 ns/op 17792 B/op 78 allocs/op +BenchmarkReferenceTabs-8 200000 12025 ns/op 18224 B/op 81 allocs/op +BenchmarkReferenceTidyness-8 200000 8985 ns/op 14432 B/op 71 allocs/op +PASS +ok github.com/gomarkdown/markdown 45.375s +``` + +After switching to using interface{} for Node.Data: +``` +BenchmarkEscapeHTML-8 2000000 929 ns/op 0 B/op 0 allocs/op +BenchmarkSmartDoubleQuotes-8 300000 5126 ns/op 9248 B/op 56 allocs/op +BenchmarkReferenceAmps-8 100000 19927 ns/op 17880 B/op 154 allocs/op +BenchmarkReferenceAutoLinks-8 100000 20732 ns/op 17360 B/op 141 allocs/op +BenchmarkReferenceBackslashEscapes-8 30000 50267 ns/op 38128 B/op 244 allocs/op +BenchmarkReferenceBlockquotesWithCodeBlocks-8 200000 8988 ns/op 10912 B/op 67 allocs/op +BenchmarkReferenceCodeBlocks-8 200000 8611 ns/op 12256 B/op 74 allocs/op +BenchmarkReferenceCodeSpans-8 200000 8256 ns/op 11248 B/op 69 allocs/op +BenchmarkReferenceHardWrappedPara-8 200000 6739 ns/op 9856 B/op 57 allocs/op +BenchmarkReferenceHorizontalRules-8 100000 15503 ns/op 15600 B/op 104 allocs/op +BenchmarkReferenceInlineHTMLAdvances-8 200000 6874 ns/op 10278 B/op 62 allocs/op +BenchmarkReferenceInlineHTMLSimple-8 100000 22271 ns/op 18552 B/op 121 allocs/op +BenchmarkReferenceInlineHTMLComments-8 200000 8315 ns/op 10736 B/op 64 allocs/op +BenchmarkReferenceLinksInline-8 100000 16155 ns/op 16912 B/op 152 allocs/op +BenchmarkReferenceLinksReference-8 30000 52387 ns/op 38192 B/op 445 allocs/op +BenchmarkReferenceLinksShortcut-8 100000 17111 ns/op 16592 B/op 167 allocs/op +BenchmarkReferenceLiterQuotesInTitles-8 200000 9164 ns/op 12048 B/op 97 allocs/op +BenchmarkReferenceMarkdownBasics-8 10000 129262 ns/op 87264 B/op 416 allocs/op +BenchmarkReferenceMarkdownSyntax-8 3000 496873 ns/op 293906 B/op 1559 allocs/op +BenchmarkReferenceNestedBlockquotes-8 200000 6854 ns/op 10192 B/op 64 allocs/op +BenchmarkReferenceOrderedAndUnorderedLists-8 20000 79633 ns/op 55024 B/op 447 allocs/op +BenchmarkReferenceStrongAndEm-8 200000 9637 ns/op 12176 B/op 78 allocs/op +BenchmarkReferenceTabs-8 100000 12164 ns/op 13776 B/op 87 allocs/op +BenchmarkReferenceTidyness-8 200000 8677 ns/op 11296 B/op 75 allocs/op +``` + +Not necessarily faster, but uses less bytes per op (but sometimes more allocs). + +After tweaking the API: +``` +$ ./s/run-bench.sh + +go test -bench=. -test.benchmem +goos: darwin +goarch: amd64 +pkg: github.com/gomarkdown/markdown +BenchmarkEscapeHTML-8 2000000 834 ns/op 0 B/op 0 allocs/op +BenchmarkSmartDoubleQuotes-8 300000 3486 ns/op 6160 B/op 27 allocs/op +BenchmarkReferenceAmps-8 100000 18158 ns/op 14792 B/op 125 allocs/op +BenchmarkReferenceAutoLinks-8 100000 16824 ns/op 14272 B/op 112 allocs/op +BenchmarkReferenceBackslashEscapes-8 30000 44066 ns/op 35040 B/op 215 allocs/op +BenchmarkReferenceBlockquotesWithCodeBlocks-8 200000 6868 ns/op 7824 B/op 38 allocs/op +BenchmarkReferenceCodeBlocks-8 200000 7157 ns/op 9168 B/op 45 allocs/op +BenchmarkReferenceCodeSpans-8 200000 6663 ns/op 8160 B/op 40 allocs/op +BenchmarkReferenceHardWrappedPara-8 300000 4821 ns/op 6768 B/op 28 allocs/op +BenchmarkReferenceHorizontalRules-8 100000 13033 ns/op 12512 B/op 75 allocs/op +BenchmarkReferenceInlineHTMLAdvances-8 300000 4998 ns/op 7190 B/op 33 allocs/op +BenchmarkReferenceInlineHTMLSimple-8 100000 17696 ns/op 15464 B/op 92 allocs/op +BenchmarkReferenceInlineHTMLComments-8 300000 5506 ns/op 7648 B/op 35 allocs/op +BenchmarkReferenceLinksInline-8 100000 14450 ns/op 13824 B/op 123 allocs/op +BenchmarkReferenceLinksReference-8 30000 52561 ns/op 35104 B/op 416 allocs/op +BenchmarkReferenceLinksShortcut-8 100000 15616 ns/op 13504 B/op 138 allocs/op +BenchmarkReferenceLiterQuotesInTitles-8 200000 7772 ns/op 8960 B/op 68 allocs/op +BenchmarkReferenceMarkdownBasics-8 10000 121436 ns/op 84176 B/op 387 allocs/op +BenchmarkReferenceMarkdownSyntax-8 3000 487404 ns/op 290818 B/op 1530 allocs/op +BenchmarkReferenceNestedBlockquotes-8 300000 5098 ns/op 7104 B/op 35 allocs/op +BenchmarkReferenceOrderedAndUnorderedLists-8 20000 74422 ns/op 51936 B/op 418 allocs/op +BenchmarkReferenceStrongAndEm-8 200000 7888 ns/op 9088 B/op 49 allocs/op +BenchmarkReferenceTabs-8 200000 10061 ns/op 10688 B/op 58 allocs/op +BenchmarkReferenceTidyness-8 200000 7152 ns/op 8208 B/op 46 allocs/op +ok github.com/gomarkdown/markdown 40.809s +``` + +After refactoring Renderer: +``` +BenchmarkEscapeHTML-8 2000000 883 ns/op 0 B/op 0 allocs/op +BenchmarkSmartDoubleQuotes-8 300000 3717 ns/op 6208 B/op 29 allocs/op +BenchmarkReferenceAmps-8 100000 19135 ns/op 14680 B/op 123 allocs/op +BenchmarkReferenceAutoLinks-8 100000 17142 ns/op 14176 B/op 110 allocs/op +BenchmarkReferenceBackslashEscapes-8 30000 54616 ns/op 35088 B/op 217 allocs/op +BenchmarkReferenceBlockquotesWithCodeBlocks-8 200000 7993 ns/op 7872 B/op 40 allocs/op +BenchmarkReferenceCodeBlocks-8 200000 8285 ns/op 9216 B/op 47 allocs/op +BenchmarkReferenceCodeSpans-8 200000 7684 ns/op 8208 B/op 42 allocs/op +BenchmarkReferenceHardWrappedPara-8 200000 5595 ns/op 6816 B/op 30 allocs/op +BenchmarkReferenceHorizontalRules-8 100000 16444 ns/op 12560 B/op 77 allocs/op +BenchmarkReferenceInlineHTMLAdvances-8 200000 5415 ns/op 7238 B/op 35 allocs/op +BenchmarkReferenceInlineHTMLSimple-8 100000 19867 ns/op 15512 B/op 94 allocs/op +BenchmarkReferenceInlineHTMLComments-8 200000 6026 ns/op 7696 B/op 37 allocs/op +BenchmarkReferenceLinksInline-8 100000 14864 ns/op 13664 B/op 120 allocs/op +BenchmarkReferenceLinksReference-8 30000 52479 ns/op 34816 B/op 401 allocs/op +BenchmarkReferenceLinksShortcut-8 100000 15812 ns/op 13472 B/op 135 allocs/op +BenchmarkReferenceLiterQuotesInTitles-8 200000 7767 ns/op 8880 B/op 68 allocs/op +BenchmarkReferenceMarkdownBasics-8 10000 131065 ns/op 84048 B/op 386 allocs/op +BenchmarkReferenceMarkdownSyntax-8 2000 515604 ns/op 289953 B/op 1501 allocs/op +BenchmarkReferenceNestedBlockquotes-8 200000 5655 ns/op 7152 B/op 37 allocs/op +BenchmarkReferenceOrderedAndUnorderedLists-8 20000 84188 ns/op 51984 B/op 420 allocs/op +BenchmarkReferenceStrongAndEm-8 200000 8664 ns/op 9136 B/op 51 allocs/op +BenchmarkReferenceTabs-8 100000 11110 ns/op 10736 B/op 60 allocs/op +BenchmarkReferenceTidyness-8 200000 7628 ns/op 8256 B/op 48 allocs/op +ok github.com/gomarkdown/markdown 40.841s +``` + +After Node refactor to have Children array: +``` +BenchmarkEscapeHTML-8 2000000 901 ns/op 0 B/op 0 allocs/op +BenchmarkSmartDoubleQuotes-8 300000 3905 ns/op 6224 B/op 31 allocs/op +BenchmarkReferenceAmps-8 100000 22216 ns/op 15560 B/op 157 allocs/op +BenchmarkReferenceAutoLinks-8 100000 20335 ns/op 14824 B/op 146 allocs/op +BenchmarkReferenceBackslashEscapes-8 20000 69174 ns/op 37392 B/op 316 allocs/op +BenchmarkReferenceBlockquotesWithCodeBlocks-8 200000 8443 ns/op 7968 B/op 48 allocs/op +BenchmarkReferenceCodeBlocks-8 200000 9250 ns/op 9392 B/op 58 allocs/op +BenchmarkReferenceCodeSpans-8 200000 8515 ns/op 8432 B/op 54 allocs/op +BenchmarkReferenceHardWrappedPara-8 200000 5738 ns/op 6856 B/op 34 allocs/op +BenchmarkReferenceHorizontalRules-8 100000 20864 ns/op 13648 B/op 93 allocs/op +BenchmarkReferenceInlineHTMLAdvances-8 200000 6187 ns/op 7310 B/op 40 allocs/op +BenchmarkReferenceInlineHTMLSimple-8 50000 23793 ns/op 16128 B/op 114 allocs/op +BenchmarkReferenceInlineHTMLComments-8 200000 7060 ns/op 7840 B/op 44 allocs/op +BenchmarkReferenceLinksInline-8 100000 18432 ns/op 14496 B/op 153 allocs/op +BenchmarkReferenceLinksReference-8 20000 67666 ns/op 37136 B/op 502 allocs/op +BenchmarkReferenceLinksShortcut-8 100000 19324 ns/op 13984 B/op 162 allocs/op +BenchmarkReferenceLiterQuotesInTitles-8 200000 8998 ns/op 9320 B/op 83 allocs/op +BenchmarkReferenceMarkdownBasics-8 10000 160908 ns/op 88152 B/op 518 allocs/op +BenchmarkReferenceMarkdownSyntax-8 2000 707160 ns/op 303801 B/op 2044 allocs/op +BenchmarkReferenceNestedBlockquotes-8 200000 6740 ns/op 7248 B/op 45 allocs/op +BenchmarkReferenceOrderedAndUnorderedLists-8 10000 115808 ns/op 55052 B/op 626 allocs/op +BenchmarkReferenceStrongAndEm-8 100000 10540 ns/op 9416 B/op 72 allocs/op +BenchmarkReferenceTabs-8 100000 13171 ns/op 10968 B/op 77 allocs/op +BenchmarkReferenceTidyness-8 200000 8903 ns/op 8404 B/op 62 allocs/op +PASS +ok github.com/gomarkdown/markdown 43.477s +``` +It's slower (but opens up possibilities for further improvements). + +After refactoring to make ast.Node a top-level thing. +``` +BenchmarkEscapeHTML-8 2000000 829 ns/op 0 B/op 0 allocs/op +BenchmarkSmartDoubleQuotes-8 300000 3998 ns/op 6192 B/op 31 allocs/op +BenchmarkReferenceAmps-8 50000 27389 ns/op 15480 B/op 153 allocs/op +BenchmarkReferenceAutoLinks-8 50000 23106 ns/op 14656 B/op 137 allocs/op +BenchmarkReferenceBackslashEscapes-8 10000 112435 ns/op 36696 B/op 315 allocs/op +BenchmarkReferenceBlockquotesWithCodeBlocks-8 200000 9227 ns/op 7856 B/op 46 allocs/op +BenchmarkReferenceCodeBlocks-8 200000 10469 ns/op 9248 B/op 54 allocs/op +BenchmarkReferenceCodeSpans-8 200000 10522 ns/op 8368 B/op 54 allocs/op +BenchmarkReferenceHardWrappedPara-8 200000 6354 ns/op 6784 B/op 34 allocs/op +BenchmarkReferenceHorizontalRules-8 50000 32393 ns/op 13952 B/op 87 allocs/op +BenchmarkReferenceInlineHTMLAdvances-8 200000 6894 ns/op 7238 B/op 40 allocs/op +BenchmarkReferenceInlineHTMLSimple-8 50000 32942 ns/op 15864 B/op 110 allocs/op +BenchmarkReferenceInlineHTMLComments-8 200000 8181 ns/op 7776 B/op 44 allocs/op +BenchmarkReferenceLinksInline-8 100000 21679 ns/op 14400 B/op 148 allocs/op +BenchmarkReferenceLinksReference-8 20000 83928 ns/op 36688 B/op 473 allocs/op +BenchmarkReferenceLinksShortcut-8 100000 22053 ns/op 13872 B/op 153 allocs/op +BenchmarkReferenceLiterQuotesInTitles-8 100000 10784 ns/op 9296 B/op 81 allocs/op +BenchmarkReferenceMarkdownBasics-8 5000 237097 ns/op 87760 B/op 480 allocs/op +BenchmarkReferenceMarkdownSyntax-8 1000 1465402 ns/op 300769 B/op 1896 allocs/op +BenchmarkReferenceNestedBlockquotes-8 200000 7461 ns/op 7152 B/op 45 allocs/op +BenchmarkReferenceOrderedAndUnorderedLists-8 5000 212256 ns/op 53724 B/op 553 allocs/op +BenchmarkReferenceStrongAndEm-8 100000 13018 ns/op 9264 B/op 72 allocs/op +BenchmarkReferenceTabs-8 100000 15005 ns/op 10752 B/op 71 allocs/op +BenchmarkReferenceTidyness-8 200000 10308 ns/op 8292 B/op 58 allocs/op +PASS +ok github.com/gomarkdown/markdown 42.176s +``` diff --git a/vendor/github.com/status-im/status-protocol-go/datasync/datasync.go b/vendor/github.com/status-im/status-protocol-go/datasync/datasync.go index 1e6a089ec..71e6e20bc 100644 --- a/vendor/github.com/status-im/status-protocol-go/datasync/datasync.go +++ b/vendor/github.com/status-im/status-protocol-go/datasync/datasync.go @@ -13,7 +13,7 @@ import ( type DataSync struct { *datasyncnode.Node - // DataSyncNodeTransport is the implemntation of the datasync transport interface + // DataSyncNodeTransport is the implementation of the datasync transport interface. *DataSyncNodeTransport logger *zap.Logger sendingEnabled bool @@ -23,14 +23,6 @@ func New(node *datasyncnode.Node, transport *DataSyncNodeTransport, sendingEnabl return &DataSync{Node: node, DataSyncNodeTransport: transport, sendingEnabled: sendingEnabled, logger: logger} } -func (d *DataSync) Add(publicKey *ecdsa.PublicKey, datasyncMessage datasyncproto.Payload) { - packet := datasynctransport.Packet{ - Sender: datasyncpeer.PublicKeyToPeerID(*publicKey), - Payload: datasyncMessage, - } - d.DataSyncNodeTransport.AddPacket(packet) -} - func (d *DataSync) Handle(sender *ecdsa.PublicKey, payload []byte) [][]byte { var payloads [][]byte logger := d.logger.With(zap.String("site", "Handle")) @@ -48,18 +40,26 @@ func (d *DataSync) Handle(sender *ecdsa.PublicKey, payload []byte) [][]byte { payloads = append(payloads, message.Body) } if d.sendingEnabled { - d.Add(sender, datasyncMessage) + d.add(sender, datasyncMessage) } } return payloads } +func (d *DataSync) Stop() { + d.Node.Stop() +} + +func (d *DataSync) add(publicKey *ecdsa.PublicKey, datasyncMessage datasyncproto.Payload) { + packet := datasynctransport.Packet{ + Sender: datasyncpeer.PublicKeyToPeerID(*publicKey), + Payload: datasyncMessage, + } + d.DataSyncNodeTransport.AddPacket(packet) +} + func unwrap(payload []byte) (datasyncPayload datasyncproto.Payload, err error) { err = proto.Unmarshal(payload, &datasyncPayload) return } - -func (d *DataSync) Stop() { - d.Node.Stop() -} diff --git a/vendor/github.com/status-im/status-protocol-go/go.mod b/vendor/github.com/status-im/status-protocol-go/go.mod index 40e65475b..0bb1f102b 100644 --- a/vendor/github.com/status-im/status-protocol-go/go.mod +++ b/vendor/github.com/status-im/status-protocol-go/go.mod @@ -7,6 +7,7 @@ require ( github.com/cenkalti/backoff/v3 v3.0.0 github.com/ethereum/go-ethereum v1.9.5 github.com/golang/protobuf v1.3.2 + github.com/gomarkdown/markdown v0.0.0-20191113114344-af599402d015 github.com/google/uuid v1.1.1 github.com/jinzhu/copier v0.0.0-20190625015134-976e0346caa8 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 // indirect @@ -32,3 +33,5 @@ require ( ) replace github.com/ethereum/go-ethereum v1.9.5 => github.com/status-im/go-ethereum v1.9.5-status.5 + +replace github.com/gomarkdown/markdown => github.com/status-im/markdown v0.0.0-20191113114344-af599402d015 diff --git a/vendor/github.com/status-im/status-protocol-go/go.sum b/vendor/github.com/status-im/status-protocol-go/go.sum index 93d25a0e8..fc2acdac7 100644 --- a/vendor/github.com/status-im/status-protocol-go/go.sum +++ b/vendor/github.com/status-im/status-protocol-go/go.sum @@ -8,6 +8,7 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -85,6 +86,7 @@ github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -95,6 +97,7 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB github.com/elastic/gosigar v0.0.0-20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo= github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/ethereum/go-ethereum v1.9.2/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -283,6 +286,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= @@ -310,6 +314,8 @@ github.com/status-im/go-ethereum v1.9.5-status.5 h1:d2RJC6ltNZJM2mrAW6kDWYdzewF8 github.com/status-im/go-ethereum v1.9.5-status.5/go.mod h1:g2+E89NWtyA+55p6XEl5Sdt7Mtez3V0T3+Y7mJNb+tI= github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw= github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/status-im/markdown v0.0.0-20191113114344-af599402d015 h1:ijC73VP0hucsy/MRn4cmtoQVB1mKdLcvurMYPvmPc4Y= +github.com/status-im/markdown v0.0.0-20191113114344-af599402d015/go.mod h1:tmG2bxyvZ2EItDO5JewbdFvV45j13IYQgvnMJ3+qAaE= github.com/status-im/migrate/v4 v4.6.2-status.2 h1:SdC+sMDl/aI7vUlwD2qj2p7KsK4T60IS9z4/rYCCbI8= github.com/status-im/migrate/v4 v4.6.2-status.2/go.mod h1:c/kc90n47GZu/58nnz1OMLTf7uE4Da4gZP5qmU+A/v8= github.com/status-im/whisper v1.5.2 h1:26NgiKusmPic38eQdtXnaY+iaQ/LuQ3Dh0kCGYT/Uxs= @@ -318,6 +324,7 @@ github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -358,6 +365,7 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -386,6 +394,7 @@ golang.org/x/net v0.0.0-20190225153610-fe579d43d832/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3 h1:6KET3Sqa7fkVfD63QnAM81ZeYg5n4HwApOJkufONnHA= golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -426,6 +435,8 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425222832-ad9eeb80039a/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191109212701-97ad0ed33101/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/vendor/github.com/status-im/status-protocol-go/messenger.go b/vendor/github.com/status-im/status-protocol-go/messenger.go index 9c1aef49b..4591fbae0 100644 --- a/vendor/github.com/status-im/status-protocol-go/messenger.go +++ b/vendor/github.com/status-im/status-protocol-go/messenger.go @@ -811,6 +811,16 @@ func (m *Messenger) RetrieveRawAll() (map[transport.Filter][]*protocol.StatusMes continue } + for _, msg := range statusMessages { + if msg.ParsedMessage != nil { + if textMessage, ok := msg.ParsedMessage.(protocol.Message); ok { + textMessage.Content = protocol.PrepareContent(textMessage.Content) + msg.ParsedMessage = textMessage + } + } + + } + result[chat] = append(result[chat], statusMessages...) } } @@ -929,6 +939,7 @@ type postProcessor struct { type postProcessorConfig struct { MatchChat bool // match each messages to a chat; may result in a new chat creation Persist bool // if true, all sent and received user messages will be persisted + Parse bool // if true, it will parse the content } func newPostProcessor(m *Messenger, config postProcessorConfig) *postProcessor { @@ -951,6 +962,9 @@ func (p *postProcessor) Run(messages []*protocol.Message) ([]*protocol.Message, if p.config.MatchChat { fns = append(fns, p.matchMessages) } + if p.config.Parse { + fns = append(fns, p.parseMessages) + } if p.config.Persist { fns = append(fns, p.saveMessages) } @@ -973,6 +987,14 @@ func (p *postProcessor) saveMessages(messages []*protocol.Message) ([]*protocol. return messages, nil } +func (p *postProcessor) parseMessages(messages []*protocol.Message) ([]*protocol.Message, error) { + for _, m := range messages { + m.Content = protocol.PrepareContent(m.Content) + } + + return messages, nil +} + func (p *postProcessor) matchMessages(messages []*protocol.Message) ([]*protocol.Message, error) { chats, err := p.persistence.Chats() if err != nil { @@ -1107,3 +1129,8 @@ func (m *Messenger) VerifyENSNames(rpcEndpoint, contractAddress string, ensDetai func GenerateAlias(id string) (string, error) { return alias.GenerateFromPublicKeyString(id) } + +// PrepareContent parses the content of a message and returns the parsed version +func (m *Messenger) PrepareContent(content protocol.Content) protocol.Content { + return protocol.PrepareContent(content) +} diff --git a/vendor/github.com/status-im/status-protocol-go/persistence.go b/vendor/github.com/status-im/status-protocol-go/persistence.go index bf3316e13..8e14f2916 100644 --- a/vendor/github.com/status-im/status-protocol-go/persistence.go +++ b/vendor/github.com/status-im/status-protocol-go/persistence.go @@ -14,10 +14,6 @@ import ( protocol "github.com/status-im/status-protocol-go/v1" ) -const ( - uniqueIDContstraint = "UNIQUE constraint failed: user_messages.id" -) - var ( // ErrMsgAlreadyExist returned if msg already exist. ErrMsgAlreadyExist = errors.New("message with given ID already exist") @@ -117,62 +113,61 @@ func (db sqlitePersistence) Chats() ([]*Chat, error) { return db.chats(nil) } -func (db sqlitePersistence) chats(tx *sql.Tx) ([]*Chat, error) { - var err error - +func (db sqlitePersistence) chats(tx *sql.Tx) (chats []*Chat, err error) { if tx == nil { tx, err = db.db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { - return nil, err + return } defer func() { if err == nil { err = tx.Commit() return - } // don't shadow original error _ = tx.Rollback() }() } - rows, err := tx.Query(`SELECT - id, - name, - color, - active, - type, - timestamp, - deleted_at_clock_value, - public_key, - unviewed_message_count, - last_clock_value, - last_message_content_type, - last_message_content, - last_message_timestamp, - last_message_clock_value, - members, - membership_updates - FROM chats - ORDER BY chats.timestamp DESC`) + rows, err := tx.Query(` + SELECT + id, + name, + color, + active, + type, + timestamp, + deleted_at_clock_value, + public_key, + unviewed_message_count, + last_clock_value, + last_message_content_type, + last_message_content, + last_message_timestamp, + last_message_clock_value, + members, + membership_updates + FROM chats + ORDER BY chats.timestamp DESC + `) if err != nil { - return nil, err + return } defer rows.Close() - var response []*Chat - for rows.Next() { - var lastMessageContentType sql.NullString - var lastMessageContent sql.NullString - var lastMessageTimestamp sql.NullInt64 - var lastMessageClockValue sql.NullInt64 + var ( + lastMessageContentType sql.NullString + lastMessageContent sql.NullString + lastMessageTimestamp sql.NullInt64 + lastMessageClockValue sql.NullInt64 - chat := &Chat{} - encodedMembers := []byte{} - encodedMembershipUpdates := []byte{} - pkey := []byte{} - err := rows.Scan( + chat Chat + encodedMembers []byte + encodedMembershipUpdates []byte + pkey []byte + ) + err = rows.Scan( &chat.ID, &chat.Name, &chat.Color, @@ -191,7 +186,7 @@ func (db sqlitePersistence) chats(tx *sql.Tx) ([]*Chat, error) { &encodedMembershipUpdates, ) if err != nil { - return nil, err + return } chat.LastMessageContent = lastMessageContent.String chat.LastMessageContentType = lastMessageContentType.String @@ -200,44 +195,47 @@ func (db sqlitePersistence) chats(tx *sql.Tx) ([]*Chat, error) { // Restore members membersDecoder := gob.NewDecoder(bytes.NewBuffer(encodedMembers)) - if err := membersDecoder.Decode(&chat.Members); err != nil { - return nil, err + err = membersDecoder.Decode(&chat.Members) + if err != nil { + return } // Restore membership updates membershipUpdatesDecoder := gob.NewDecoder(bytes.NewBuffer(encodedMembershipUpdates)) - if err := membershipUpdatesDecoder.Decode(&chat.MembershipUpdates); err != nil { - return nil, err + err = membershipUpdatesDecoder.Decode(&chat.MembershipUpdates) + if err != nil { + return } if len(pkey) != 0 { chat.PublicKey, err = crypto.UnmarshalPubkey(pkey) if err != nil { - return nil, err + return } } - response = append(response, chat) + chats = append(chats, &chat) } - return response, nil + return } func (db sqlitePersistence) Contacts() ([]*Contact, error) { - rows, err := db.db.Query(`SELECT - id, - address, - name, - alias, - identicon, - photo, - last_updated, - system_tags, - device_info, - ens_verified, - ens_verified_at, - tribute_to_talk - FROM contacts`) - + rows, err := db.db.Query(` + SELECT + id, + address, + name, + alias, + identicon, + photo, + last_updated, + system_tags, + device_info, + ens_verified, + ens_verified_at, + tribute_to_talk + FROM contacts + `) if err != nil { return nil, err } @@ -246,9 +244,11 @@ func (db sqlitePersistence) Contacts() ([]*Contact, error) { var response []*Contact for rows.Next() { - contact := &Contact{} - encodedDeviceInfo := []byte{} - encodedSystemTags := []byte{} + var ( + contact Contact + encodedDeviceInfo []byte + encodedSystemTags []byte + ) err := rows.Scan( &contact.ID, &contact.Address, @@ -283,7 +283,7 @@ func (db sqlitePersistence) Contacts() ([]*Contact, error) { } } - response = append(response, contact) + response = append(response, &contact) } return response, nil @@ -298,7 +298,6 @@ func (db sqlitePersistence) SetContactsENSData(contacts []Contact) error { if err == nil { err = tx.Commit() return - } // don't shadow original error _ = tx.Rollback() @@ -324,8 +323,7 @@ func (db sqlitePersistence) SetContactsENSData(contacts []Contact) error { // SetContactsGeneratedData sets a contact generated data if not existing already // in the database -func (db sqlitePersistence) SetContactsGeneratedData(contacts []Contact, tx *sql.Tx) error { - var err error +func (db sqlitePersistence) SetContactsGeneratedData(contacts []Contact, tx *sql.Tx) (err error) { if tx == nil { tx, err = db.db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { @@ -343,36 +341,35 @@ func (db sqlitePersistence) SetContactsGeneratedData(contacts []Contact, tx *sql } for _, contact := range contacts { - _, err := tx.Exec(`INSERT OR IGNORE INTO contacts( - id, - address, - name, - alias, - identicon, - photo, - last_updated, - tribute_to_talk) - VALUES (?, ?, "", ?, ?, "", 0, "")`, + _, err = tx.Exec(` + INSERT OR IGNORE INTO contacts( + id, + address, + name, + alias, + identicon, + photo, + last_updated, + tribute_to_talk + ) VALUES (?, ?, "", ?, ?, "", 0, "")`, contact.ID, contact.Address, contact.Alias, contact.Identicon, ) if err != nil { - return err + return } } - return nil + return } -func (db sqlitePersistence) SaveContact(contact Contact, tx *sql.Tx) error { - var err error - +func (db sqlitePersistence) SaveContact(contact Contact, tx *sql.Tx) (err error) { if tx == nil { tx, err = db.db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { - return err + return } defer func() { if err == nil { @@ -387,37 +384,38 @@ func (db sqlitePersistence) SaveContact(contact Contact, tx *sql.Tx) error { // Encode device info var encodedDeviceInfo bytes.Buffer deviceInfoEncoder := gob.NewEncoder(&encodedDeviceInfo) - - if err := deviceInfoEncoder.Encode(contact.DeviceInfo); err != nil { - return err + err = deviceInfoEncoder.Encode(contact.DeviceInfo) + if err != nil { + return } // Encoded system tags var encodedSystemTags bytes.Buffer systemTagsEncoder := gob.NewEncoder(&encodedSystemTags) - - if err := systemTagsEncoder.Encode(contact.SystemTags); err != nil { - return err + err = systemTagsEncoder.Encode(contact.SystemTags) + if err != nil { + return } // Insert record - stmt, err := tx.Prepare(`INSERT INTO contacts( - id, - address, - name, - alias, - identicon, - photo, - last_updated, - system_tags, - device_info, - ens_verified, - ens_verified_at, - tribute_to_talk - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + stmt, err := tx.Prepare(` + INSERT INTO contacts( + id, + address, + name, + alias, + identicon, + photo, + last_updated, + system_tags, + device_info, + ens_verified, + ens_verified_at, + tribute_to_talk + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) if err != nil { - return err + return } defer stmt.Close() @@ -435,12 +433,13 @@ func (db sqlitePersistence) SaveContact(contact Contact, tx *sql.Tx) error { contact.ENSVerifiedAt, contact.TributeToTalk, ) - return err + return } // Messages returns messages for a given contact, in a given period. Ordered by a timestamp. func (db sqlitePersistence) Messages(from, to time.Time) (result []*protocol.Message, err error) { - rows, err := db.db.Query(`SELECT + rows, err := db.db.Query(` + SELECT id, chat_id, content_type, @@ -452,47 +451,42 @@ func (db sqlitePersistence) Messages(from, to time.Time) (result []*protocol.Mes content_text, public_key, flags - FROM user_messages + FROM user_messages WHERE timestamp >= ? AND timestamp <= ? ORDER BY timestamp`, protocol.TimestampInMsFromTime(from), protocol.TimestampInMsFromTime(to), ) if err != nil { - return nil, err + return } defer rows.Close() - var rst []*protocol.Message for rows.Next() { msg := protocol.Message{ Content: protocol.Content{}, } - pkey := []byte{} + var pkey []byte err = rows.Scan( &msg.ID, &msg.ChatID, &msg.ContentT, &msg.MessageT, &msg.Text, &msg.Clock, &msg.Timestamp, &msg.Content.ChatID, &msg.Content.Text, &pkey, &msg.Flags, ) if err != nil { - return nil, err + return } if len(pkey) != 0 { msg.SigPubKey, err = crypto.UnmarshalPubkey(pkey) if err != nil { - return nil, err + return } } - rst = append(rst, &msg) + result = append(result, &msg) } - return rst, nil + return } func (db sqlitePersistence) SaveMessages(messages []*protocol.Message) (last int64, err error) { - var ( - tx *sql.Tx - stmt *sql.Stmt - ) - tx, err = db.db.BeginTx(context.Background(), &sql.TxOptions{}) + tx, err := db.db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { return } @@ -505,8 +499,8 @@ func (db sqlitePersistence) SaveMessages(messages []*protocol.Message) (last int _ = tx.Rollback() }() - stmt, err = tx.Prepare(`INSERT INTO - user_messages( + stmt, err := tx.Prepare(` + INSERT OR IGNORE INTO user_messages( id, chat_id, content_type, @@ -536,11 +530,6 @@ func (db sqlitePersistence) SaveMessages(messages []*protocol.Message) (last int msg.Content.ChatID, msg.Content.Text, pkey, msg.Flags, ) if err != nil { - if err.Error() == uniqueIDContstraint { - // skip duplicated messages - err = nil - continue - } return } @@ -549,5 +538,6 @@ func (db sqlitePersistence) SaveMessages(messages []*protocol.Message) (last int return } } + return } diff --git a/vendor/github.com/status-im/status-protocol-go/persistence_legacy.go b/vendor/github.com/status-im/status-protocol-go/persistence_legacy.go index a8c041b13..b7a864252 100644 --- a/vendor/github.com/status-im/status-protocol-go/persistence_legacy.go +++ b/vendor/github.com/status-im/status-protocol-go/persistence_legacy.go @@ -267,21 +267,15 @@ func (db sqlitePersistence) MessageByChatID(chatID string, currCursor string, li return result, newCursor, nil } -func (db sqlitePersistence) SaveMessagesLegacy(messages []*Message) error { - var ( - tx *sql.Tx - stmt *sql.Stmt - err error - ) - tx, err = db.db.BeginTx(context.Background(), &sql.TxOptions{}) +func (db sqlitePersistence) SaveMessagesLegacy(messages []*Message) (err error) { + tx, err := db.db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { - return err + return } defer func() { if err == nil { err = tx.Commit() return - } // don't shadow original error _ = tx.Rollback() @@ -289,21 +283,19 @@ func (db sqlitePersistence) SaveMessagesLegacy(messages []*Message) error { allFields := db.tableUserMessagesLegacyAllFields() valuesVector := strings.Repeat("?, ", db.tableUserMessagesLegacyAllFieldsCount()-1) + "?" - query := fmt.Sprintf(`INSERT INTO user_messages_legacy(%s) VALUES (%s)`, allFields, valuesVector) - - stmt, err = tx.Prepare(query) + stmt, err := tx.Prepare(query) if err != nil { - return err + return } for _, msg := range messages { - _, err := stmt.Exec(db.tableUserMessagesLegacyAllValues(msg)...) + _, err = stmt.Exec(db.tableUserMessagesLegacyAllValues(msg)...) if err != nil { - return err + return } } - return err + return } func (db sqlitePersistence) DeleteMessage(id string) error { @@ -343,20 +335,15 @@ func (db sqlitePersistence) UpdateMessageOutgoingStatus(id string, newOutgoingSt } // BlockContact updates a contact, deletes all the messages and 1-to-1 chat, updates the unread messages count and returns a map with the new count -func (db sqlitePersistence) BlockContact(contact Contact) ([]*Chat, error) { - var ( - tx *sql.Tx - err error - ) - tx, err = db.db.BeginTx(context.Background(), &sql.TxOptions{}) +func (db sqlitePersistence) BlockContact(contact Contact) (chats []*Chat, err error) { + tx, err := db.db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { - return nil, err + return } defer func() { if err == nil { err = tx.Commit() return - } // don't shadow original error _ = tx.Rollback() @@ -370,34 +357,36 @@ func (db sqlitePersistence) BlockContact(contact Contact) ([]*Chat, error) { contact.ID, ) if err != nil { - return nil, err + return } // Update contact err = db.SaveContact(contact, tx) if err != nil { - return nil, err + return } // Delete one-to-one chat _, err = tx.Exec("DELETE FROM chats WHERE id = ?", contact.ID) if err != nil { - return nil, err + return } // Recalculate denormalized fields _, err = tx.Exec(` - UPDATE chats - SET - unviewed_message_count = (SELECT COUNT(1) FROM user_messages_legacy WHERE seen = 0 AND chat_id = chats.id), - last_message_content = (SELECT content from user_messages_legacy WHERE chat_id = chats.id ORDER BY clock_value DESC LIMIT 1), - last_message_timestamp = (SELECT timestamp from user_messages_legacy WHERE chat_id = chats.id ORDER BY clock_value DESC LIMIT 1), - last_message_clock_value = (SELECT clock_value from user_messages_legacy WHERE chat_id = chats.id ORDER BY clock_value DESC LIMIT 1), - last_message_content_type = (SELECT content_type from user_messages_legacy WHERE chat_id = chats.id ORDER BY clock_value DESC LIMIT 1)`) + UPDATE chats + SET + unviewed_message_count = (SELECT COUNT(1) FROM user_messages_legacy WHERE seen = 0 AND chat_id = chats.id), + last_message_content = (SELECT content from user_messages_legacy WHERE chat_id = chats.id ORDER BY clock_value DESC LIMIT 1), + last_message_timestamp = (SELECT timestamp from user_messages_legacy WHERE chat_id = chats.id ORDER BY clock_value DESC LIMIT 1), + last_message_clock_value = (SELECT clock_value from user_messages_legacy WHERE chat_id = chats.id ORDER BY clock_value DESC LIMIT 1), + last_message_content_type = (SELECT content_type from user_messages_legacy WHERE chat_id = chats.id ORDER BY clock_value DESC LIMIT 1) + `) if err != nil { - return nil, err + return } // return the updated chats - return db.chats(tx) + chats, err = db.chats(tx) + return } diff --git a/vendor/github.com/status-im/status-protocol-go/v1/content.go b/vendor/github.com/status-im/status-protocol-go/v1/content.go new file mode 100644 index 000000000..7ba86a642 --- /dev/null +++ b/vendor/github.com/status-im/status-protocol-go/v1/content.go @@ -0,0 +1,38 @@ +package statusproto + +import ( + "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/ast" + "strings" + "unicode" + "unicode/utf8" +) + +// Content contains the chat ID and the actual text of a message. +type Content struct { + ChatID string `json:"chat_id"` + Text string `json:"text"` + ResponseTo string `json:"response-to"` + Name string `json:"name"` // the ENS name of the sender + ParsedText ast.Node `json:"parsedText"` + LineCount int `json:"lineCount"` + RTL bool `json:"rtl"` +} + +// Check if the first character is Hebrew or Arabic or the RTL character +func isRTL(s string) bool { + first, _ := utf8.DecodeRuneInString(s) + return unicode.Is(unicode.Hebrew, first) || + unicode.Is(unicode.Arabic, first) || + // RTL character + first == '\u200f' +} + +// PrepareContent return the parsed content of the message, the line-count and whether +// is a right-to-left message +func PrepareContent(content Content) Content { + content.ParsedText = markdown.Parse([]byte(content.Text), nil) + content.LineCount = strings.Count(content.Text, "\n") + content.RTL = isRTL(content.Text) + return content +} diff --git a/vendor/github.com/status-im/status-protocol-go/v1/message.go b/vendor/github.com/status-im/status-protocol-go/v1/message.go index 71eedacc9..c1c91620c 100644 --- a/vendor/github.com/status-im/status-protocol-go/v1/message.go +++ b/vendor/github.com/status-im/status-protocol-go/v1/message.go @@ -32,14 +32,6 @@ var ( ErrInvalidDecodedValue = errors.New("invalid decoded value type") ) -// Content contains the chat ID and the actual text of a message. -type Content struct { - ChatID string `json:"chat_id"` - Text string `json:"text"` - ResponseTo string `json:"response-to"` - Name string `json:"name"` // the ENS name of the sender -} - // TimestampInMs is a timestamp in milliseconds. type TimestampInMs int64 diff --git a/vendor/modules.txt b/vendor/modules.txt index f2778122f..3c1e89757 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -133,6 +133,10 @@ github.com/golang/protobuf/proto github.com/golang/protobuf/protoc-gen-go/descriptor # github.com/golang/snappy v0.0.1 github.com/golang/snappy +# github.com/gomarkdown/markdown v0.0.0-20191113114344-af599402d015 => github.com/status-im/markdown v0.0.0-20191113114344-af599402d015 +github.com/gomarkdown/markdown +github.com/gomarkdown/markdown/ast +github.com/gomarkdown/markdown/parser # github.com/google/uuid v1.1.1 github.com/google/uuid # github.com/gorilla/websocket v1.4.1 @@ -365,7 +369,7 @@ github.com/status-im/migrate/v4/source/go_bindata github.com/status-im/rendezvous github.com/status-im/rendezvous/protocol github.com/status-im/rendezvous/server -# github.com/status-im/status-protocol-go v0.4.5-0.20191107122821-775d17008edf +# github.com/status-im/status-protocol-go v0.5.1 github.com/status-im/status-protocol-go github.com/status-im/status-protocol-go/applicationmetadata github.com/status-im/status-protocol-go/bridge/geth