From 7d22880556a86eeee1c145253098a84de5dc82a5 Mon Sep 17 00:00:00 2001
From: Alex Yatskov <FooSoft@users.noreply.github.com>
Date: Tue, 17 Nov 2015 12:58:23 +0900
Subject: [PATCH 01/42] Adding a link to Md2Vim.

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 7b97970..dd8eb16 100644
--- a/README.md
+++ b/README.md
@@ -242,6 +242,8 @@ are a few of note:
     point. In particular, it does not do any inline escaping, so input
     that happens to look like LaTeX code will be passed through without
     modification.
+    
+*   [Md2Vim](https://github.com/FooSoft/md2vim): transforms markdown files into vimdoc format.
 
 
 Todo

From 1622f8f31262f18a4a3d99a6f5d30e900b224cec Mon Sep 17 00:00:00 2001
From: Nat Welch <nat@natwelch.com>
Date: Thu, 26 Nov 2015 22:11:25 +0000
Subject: [PATCH 02/42] Apply @miekg's patch

From https://github.com/miekg/mmark/commit/99ce7134f6bf7b5b919d18ce5cc5099339b27e10.patch
---
 inline.go      | 20 +++++++++++++++++---
 inline_test.go | 14 ++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/inline.go b/inline.go
index a28bb66..0e41a3c 100644
--- a/inline.go
+++ b/inline.go
@@ -260,7 +260,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 		}
 	}
 
-	if i >= len(data) {
+	if i >= len(data) || brace > 0 {
 		return 0
 	}
 
@@ -285,14 +285,28 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 
 		linkB := i
 
-		// look for link end: ' " )
+		// look for link end: ' " ), check for new openning
+		// braces and take this into account, this may lead
+		// for overshooting and probably will require some
+		// finetuning.
 	findlinkend:
 		for i < len(data) {
 			switch {
 			case data[i] == '\\':
 				i += 2
 
-			case data[i] == ')' || data[i] == '\'' || data[i] == '"':
+			case data[i] == '(':
+				brace++
+				i++
+
+			case data[i] == ')':
+				if brace <= 0 {
+					break findlinkend
+				}
+				brace--
+				i++
+
+			case data[i] == '\'' || data[i] == '"':
 				break findlinkend
 
 			default:
diff --git a/inline_test.go b/inline_test.go
index d793e01..6227914 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -541,6 +541,20 @@ func TestInlineLink(t *testing.T) {
 
 		"[link](<../>)\n",
 		"<p><a href=\"../\">link</a></p>\n",
+
+		// Issue 116 in blackfriday
+		"![](http://www.broadgate.co.uk/Content/Upload/DetailImages/Cyclus700(1).jpg)",
+		"<p><img src=\"http://www.broadgate.co.uk/Content/Upload/DetailImages/Cyclus700(1).jpg\" alt=\"\" />\n</p>\n",
+
+		// no closing ), autolinking detects the url next
+		"[disambiguation](http://en.wikipedia.org/wiki/Disambiguation_(disambiguation) is the",
+		"<p>[disambiguation](<a href=\"http://en.wikipedia.org/wiki/Disambiguation_(disambiguation\">http://en.wikipedia.org/wiki/Disambiguation_(disambiguation</a>) is the</p>\n",
+
+		"[disambiguation](http://en.wikipedia.org/wiki/Disambiguation_(disambiguation)) is the",
+		"<p><a href=\"http://en.wikipedia.org/wiki/Disambiguation_(disambiguation)\">disambiguation</a> is the</p>\n",
+
+		"[disambiguation](http://en.wikipedia.org/wiki/Disambiguation_(disambiguation))",
+		"<p><a href=\"http://en.wikipedia.org/wiki/Disambiguation_(disambiguation)\">disambiguation</a></p>\n",
 	}
 	doLinkTestsInline(t, tests)
 

From 45f5800b758cf5de7d0639ecdf072f2279c635e0 Mon Sep 17 00:00:00 2001
From: Nat Welch <nat@natwelch.com>
Date: Sun, 29 Nov 2015 00:08:48 +0000
Subject: [PATCH 03/42] Get tests to pass

---
 inline.go      | 2 ++
 inline_test.go | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/inline.go b/inline.go
index 0e41a3c..27bcc80 100644
--- a/inline.go
+++ b/inline.go
@@ -240,6 +240,8 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 		i++
 	}
 
+	brace := 0
+
 	// look for the matching closing bracket
 	for level := 1; level > 0 && i < len(data); i++ {
 		switch {
diff --git a/inline_test.go b/inline_test.go
index 6227914..b608a0a 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -544,7 +544,7 @@ func TestInlineLink(t *testing.T) {
 
 		// Issue 116 in blackfriday
 		"![](http://www.broadgate.co.uk/Content/Upload/DetailImages/Cyclus700(1).jpg)",
-		"<p><img src=\"http://www.broadgate.co.uk/Content/Upload/DetailImages/Cyclus700(1).jpg\" alt=\"\" />\n</p>\n",
+		"<p><img src=\"http://www.broadgate.co.uk/Content/Upload/DetailImages/Cyclus700(1).jpg\" alt=\"\" /></p>\n",
 
 		// no closing ), autolinking detects the url next
 		"[disambiguation](http://en.wikipedia.org/wiki/Disambiguation_(disambiguation) is the",

From c1d4a77100551b12c946bfd8f3006083d84771d9 Mon Sep 17 00:00:00 2001
From: Nat Welch <nat@natwelch.com>
Date: Sat, 5 Dec 2015 02:00:23 +0000
Subject: [PATCH 04/42] Fix comment formatting and spelling

---
 inline.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/inline.go b/inline.go
index 27bcc80..0a773da 100644
--- a/inline.go
+++ b/inline.go
@@ -287,10 +287,9 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 
 		linkB := i
 
-		// look for link end: ' " ), check for new openning
-		// braces and take this into account, this may lead
-		// for overshooting and probably will require some
-		// finetuning.
+		// look for link end: ' " ), check for new opening braces and take this
+		// into account, this may lead for overshooting and probably will require
+		// some fine-tuning.
 	findlinkend:
 		for i < len(data) {
 			switch {

From 1bb1fa9e9579400cf9e6f581cc9e7159b73ad20d Mon Sep 17 00:00:00 2001
From: Nat Welch <nat@natwelch.com>
Date: Sat, 5 Dec 2015 02:00:36 +0000
Subject: [PATCH 05/42] remove check that brace is > 0

---
 inline.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inline.go b/inline.go
index 0a773da..6a0eda6 100644
--- a/inline.go
+++ b/inline.go
@@ -262,7 +262,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 		}
 	}
 
-	if i >= len(data) || brace > 0 {
+	if i >= len(data) {
 		return 0
 	}
 

From e3cbadc5c960894017d53ecf49dba4c361dacddf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Sun, 6 Dec 2015 19:26:14 +0200
Subject: [PATCH 06/42] Bump version

---
 markdown.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/markdown.go b/markdown.go
index 746905b..ed6f794 100644
--- a/markdown.go
+++ b/markdown.go
@@ -25,7 +25,7 @@ import (
 	"unicode/utf8"
 )
 
-const VERSION = "1.4"
+const VERSION = "1.5"
 
 // These are the supported markdown parsing extensions.
 // OR these values together to select multiple extensions.

From bcd6dd8711ba8ea44cbfa17ea3c9b0ae0a4559e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Thu, 10 Dec 2015 20:52:47 +0200
Subject: [PATCH 07/42] Don't require the input to end with a newline

Technically, not ending a line with a newline is wrong, but this blunder
is so pervasive that we have no other choice but to live with it.

Fixes #180.
---
 inline_test.go | 3 +++
 markdown.go    | 3 ---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/inline_test.go b/inline_test.go
index b608a0a..626db11 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -695,6 +695,9 @@ func TestReferenceLink(t *testing.T) {
 
 		"[ref]\n   [ref]: ../url/ \"title\"\n",
 		"<p><a href=\"../url/\" title=\"title\">ref</a></p>\n",
+
+		"[link][ref]\n   [ref]: /url/",
+		"<p><a href=\"/url/\">link</a></p>\n",
 	}
 	doLinkTestsInline(t, tests)
 }
diff --git a/markdown.go b/markdown.go
index ed6f794..f4a55a8 100644
--- a/markdown.go
+++ b/markdown.go
@@ -638,9 +638,6 @@ func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffse
 	for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
 		i++
 	}
-	if i == len(data) {
-		return
-	}
 	linkEnd = i
 	if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
 		linkOffset++

From fc740701cff64485115bb8030c1307af988d49e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Thu, 10 Dec 2015 21:03:21 +0200
Subject: [PATCH 08/42] Cosmetics: fix comment placement and import order

---
 inline.go      | 2 +-
 inline_test.go | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/inline.go b/inline.go
index 6a0eda6..044ae70 100644
--- a/inline.go
+++ b/inline.go
@@ -275,8 +275,8 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 		i++
 	}
 
-	// inline style link
 	switch {
+	// inline style link
 	case i < len(data) && data[i] == '(':
 		// skip initial whitespace
 		i++
diff --git a/inline_test.go b/inline_test.go
index 626db11..1806669 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -15,9 +15,8 @@ package blackfriday
 
 import (
 	"regexp"
-	"testing"
-
 	"strings"
+	"testing"
 )
 
 func runMarkdownInline(input string, opts Options, htmlFlags int, params HtmlRendererParameters) string {

From d18b67ae0afd61dae240896eae1785f00709aa31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Sat, 12 Dec 2015 19:18:59 +0200
Subject: [PATCH 09/42] Add more tests for emphasised text

Including EXTENSION_NO_INTRA_EMPHASIS. These nail down a fix for #178
that was a side effect of cc3cc10.
---
 inline_test.go | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/inline_test.go b/inline_test.go
index 1806669..3ae2561 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -155,10 +155,35 @@ func TestEmphasis(t *testing.T) {
 
 		"*What is A\\* algorithm?*\n",
 		"<p><em>What is A* algorithm?</em></p>\n",
+
+		"some para_graph with _emphasised_ text.\n",
+		"<p>some para_graph with <em>emphasised</em> text.</p>\n",
+
+		"some paragraph with _emphasised_ te_xt.\n",
+		"<p>some paragraph with <em>emphasised</em> te_xt.</p>\n",
+
+		"some paragraph with t_wo bi_ts of _emphasised_ text.\n",
+		"<p>some paragraph with t<em>wo bi</em>ts of <em>emphasised</em> text.</p>\n",
+
+		"un*frigging*believable\n",
+		"<p>un<em>frigging</em>believable</p>\n",
 	}
 	doTestsInline(t, tests)
 }
 
+func TestNoIntraEmphasis(t *testing.T) {
+	tests := []string{
+		"some para_graph with _emphasised_ text.\n",
+		"<p>some para_graph with <em>emphasised</em> text.</p>\n",
+
+		"un*frigging*believable\n",
+		"<p>un*frigging*believable</p>\n",
+	}
+	doTestsInlineParam(t, tests, Options{
+		Extensions: EXTENSION_NO_INTRA_EMPHASIS},
+		0, HtmlRendererParameters{})
+}
+
 func TestReferenceOverride(t *testing.T) {
 	var tests = []string{
 		"test [ref1][]\n",

From 05a79218edc4c95d46c7be8f5af8b17f752d5061 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Mon, 14 Dec 2015 20:23:49 +0200
Subject: [PATCH 10/42] Implement support for CDATA section

Fixes #165.
---
 block.go      | 58 ++++++++++++++++++++++++++++++++++-----------------
 block_test.go | 17 +++++++++++++++
 2 files changed, 56 insertions(+), 19 deletions(-)

diff --git a/block.go b/block.go
index b5b0841..4a3b68e 100644
--- a/block.go
+++ b/block.go
@@ -320,6 +320,11 @@ func (p *parser) html(out *bytes.Buffer, data []byte, doRender bool) int {
 			return size
 		}
 
+		// check for HTML CDATA
+		if size := p.htmlCDATA(out, data, doRender); size > 0 {
+			return size
+		}
+
 		// no special case recognized
 		return 0
 	}
@@ -397,12 +402,10 @@ func (p *parser) html(out *bytes.Buffer, data []byte, doRender bool) int {
 	return i
 }
 
-// HTML comment, lax form
-func (p *parser) htmlComment(out *bytes.Buffer, data []byte, doRender bool) int {
-	i := p.inlineHtmlComment(out, data)
-	// needs to end with a blank line
-	if j := p.isEmpty(data[i:]); j > 0 {
-		size := i + j
+func (p *parser) renderHtmlBlock(out *bytes.Buffer, data []byte, start int, doRender bool) int {
+	// html block needs to end with a blank line
+	if i := p.isEmpty(data[start:]); i > 0 {
+		size := start + i
 		if doRender {
 			// trim trailing newlines
 			end := size
@@ -416,6 +419,35 @@ func (p *parser) htmlComment(out *bytes.Buffer, data []byte, doRender bool) int
 	return 0
 }
 
+// HTML comment, lax form
+func (p *parser) htmlComment(out *bytes.Buffer, data []byte, doRender bool) int {
+	i := p.inlineHtmlComment(out, data)
+	return p.renderHtmlBlock(out, data, i, doRender)
+}
+
+// HTML CDATA section
+func (p *parser) htmlCDATA(out *bytes.Buffer, data []byte, doRender bool) int {
+	cdataTag := "<![cdata["
+	cdataTagLen := len(cdataTag)
+	if len(data) < cdataTagLen+1 {
+		return 0
+	}
+	if !bytes.Equal(bytes.ToLower(data[:cdataTagLen]), []byte(cdataTag)) {
+		return 0
+	}
+	i := cdataTagLen
+	// scan for an end-of-comment marker, across lines if necessary
+	for i < len(data) && !(data[i-2] == ']' && data[i-1] == ']' && data[i] == '>') {
+		i++
+	}
+	i++
+	// no end-of-comment marker
+	if i >= len(data) {
+		return 0
+	}
+	return p.renderHtmlBlock(out, data, i, doRender)
+}
+
 // HR, which is the only self-closing block tag considered
 func (p *parser) htmlHr(out *bytes.Buffer, data []byte, doRender bool) int {
 	if data[0] != '<' || (data[1] != 'h' && data[1] != 'H') || (data[2] != 'r' && data[2] != 'R') {
@@ -432,19 +464,7 @@ func (p *parser) htmlHr(out *bytes.Buffer, data []byte, doRender bool) int {
 	}
 
 	if data[i] == '>' {
-		i++
-		if j := p.isEmpty(data[i:]); j > 0 {
-			size := i + j
-			if doRender {
-				// trim newlines
-				end := size
-				for end > 0 && data[end-1] == '\n' {
-					end--
-				}
-				p.r.BlockHtml(out, data[:end])
-			}
-			return size
-		}
+		return p.renderHtmlBlock(out, data, i+1, doRender)
 	}
 
 	return 0
diff --git a/block_test.go b/block_test.go
index b33c257..b95edc2 100644
--- a/block_test.go
+++ b/block_test.go
@@ -1530,3 +1530,20 @@ func TestBlockComments(t *testing.T) {
 	}
 	doTestsBlock(t, tests, 0)
 }
+
+func TestCDATA(t *testing.T) {
+	var tests = []string{
+		"Some text\n\n<![CDATA[foo]]>\n",
+		"<p>Some text</p>\n\n<![CDATA[foo]]>\n",
+
+		"CDATA ]]\n\n<![CDATA[]]]]>\n",
+		"<p>CDATA ]]</p>\n\n<![CDATA[]]]]>\n",
+
+		"CDATA >\n\n<![CDATA[>]]>\n",
+		"<p>CDATA &gt;</p>\n\n<![CDATA[>]]>\n",
+
+		"Lots of text\n\n<![CDATA[lots of te><t\non\nseveral\nlines]]>\n",
+		"<p>Lots of text</p>\n\n<![CDATA[lots of te><t\non\nseveral\nlines]]>\n",
+	}
+	doTestsBlock(t, tests, 0)
+}

From 594d923645085ba482d4341bff4d17ee7d9fab02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Wed, 16 Dec 2015 20:51:44 +0200
Subject: [PATCH 11/42] Convert constant variables into consts

---
 block.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/block.go b/block.go
index 4a3b68e..9ccffaf 100644
--- a/block.go
+++ b/block.go
@@ -427,8 +427,8 @@ func (p *parser) htmlComment(out *bytes.Buffer, data []byte, doRender bool) int
 
 // HTML CDATA section
 func (p *parser) htmlCDATA(out *bytes.Buffer, data []byte, doRender bool) int {
-	cdataTag := "<![cdata["
-	cdataTagLen := len(cdataTag)
+	const cdataTag = "<![cdata["
+	const cdataTagLen = len(cdataTag)
 	if len(data) < cdataTagLen+1 {
 		return 0
 	}

From f5ac2ddf7a72485b82fe3821c0f4e88b84035f09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Wed, 16 Dec 2015 20:53:40 +0200
Subject: [PATCH 12/42] Add a few more CDATA tests

---
 block_test.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/block_test.go b/block_test.go
index b95edc2..60da6ca 100644
--- a/block_test.go
+++ b/block_test.go
@@ -1544,6 +1544,27 @@ func TestCDATA(t *testing.T) {
 
 		"Lots of text\n\n<![CDATA[lots of te><t\non\nseveral\nlines]]>\n",
 		"<p>Lots of text</p>\n\n<![CDATA[lots of te><t\non\nseveral\nlines]]>\n",
+
+		"<![CDATA[>]]>\n",
+		"<![CDATA[>]]>\n",
 	}
 	doTestsBlock(t, tests, 0)
+	doTestsBlock(t, []string{
+		"``` html\n<![CDATA[foo]]>\n```\n",
+		"<pre><code class=\"language-html\">&lt;![CDATA[foo]]&gt;\n</code></pre>\n",
+
+		"<![CDATA[\n``` python\ndef func():\n    pass\n```\n]]>\n",
+		"<![CDATA[\n``` python\ndef func():\n    pass\n```\n]]>\n",
+
+		`<![CDATA[
+> def func():
+>     pass
+]]>
+`,
+		`<![CDATA[
+> def func():
+>     pass
+]]>
+`,
+	}, EXTENSION_FENCED_CODE)
 }

From ee63ffd3e233a02a3d4ab121bc6d5897d67df38e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Fri, 25 Dec 2015 13:04:56 +0200
Subject: [PATCH 13/42] Fix initialisms in function names

---
 block.go  | 10 +++++-----
 inline.go |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/block.go b/block.go
index 9ccffaf..7fb472e 100644
--- a/block.go
+++ b/block.go
@@ -402,7 +402,7 @@ func (p *parser) html(out *bytes.Buffer, data []byte, doRender bool) int {
 	return i
 }
 
-func (p *parser) renderHtmlBlock(out *bytes.Buffer, data []byte, start int, doRender bool) int {
+func (p *parser) renderHTMLBlock(out *bytes.Buffer, data []byte, start int, doRender bool) int {
 	// html block needs to end with a blank line
 	if i := p.isEmpty(data[start:]); i > 0 {
 		size := start + i
@@ -421,8 +421,8 @@ func (p *parser) renderHtmlBlock(out *bytes.Buffer, data []byte, start int, doRe
 
 // HTML comment, lax form
 func (p *parser) htmlComment(out *bytes.Buffer, data []byte, doRender bool) int {
-	i := p.inlineHtmlComment(out, data)
-	return p.renderHtmlBlock(out, data, i, doRender)
+	i := p.inlineHTMLComment(out, data)
+	return p.renderHTMLBlock(out, data, i, doRender)
 }
 
 // HTML CDATA section
@@ -445,7 +445,7 @@ func (p *parser) htmlCDATA(out *bytes.Buffer, data []byte, doRender bool) int {
 	if i >= len(data) {
 		return 0
 	}
-	return p.renderHtmlBlock(out, data, i, doRender)
+	return p.renderHTMLBlock(out, data, i, doRender)
 }
 
 // HR, which is the only self-closing block tag considered
@@ -464,7 +464,7 @@ func (p *parser) htmlHr(out *bytes.Buffer, data []byte, doRender bool) int {
 	}
 
 	if data[i] == '>' {
-		return p.renderHtmlBlock(out, data, i+1, doRender)
+		return p.renderHTMLBlock(out, data, i+1, doRender)
 	}
 
 	return 0
diff --git a/inline.go b/inline.go
index 044ae70..7dac6b8 100644
--- a/inline.go
+++ b/inline.go
@@ -575,7 +575,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 	return i
 }
 
-func (p *parser) inlineHtmlComment(out *bytes.Buffer, data []byte) int {
+func (p *parser) inlineHTMLComment(out *bytes.Buffer, data []byte) int {
 	if len(data) < 5 {
 		return 0
 	}
@@ -599,7 +599,7 @@ func leftAngle(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 	data = data[offset:]
 	altype := LINK_TYPE_NOT_AUTOLINK
 	end := tagLength(data, &altype)
-	if size := p.inlineHtmlComment(out, data); size > 0 {
+	if size := p.inlineHTMLComment(out, data); size > 0 {
 		end = size
 	}
 	if end > 2 {

From fc6236fc55676063c27d56d02aeab6caa1ef07f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Sat, 26 Dec 2015 20:18:22 +0200
Subject: [PATCH 14/42] Fix bug with gathering list item lines

Instead of swallowing an empty line and then reintroducing it back again
in certain cases, collect the list item body in an unaltered form and
let the recursive parsing call sort things out.

Fixes issue #228.
---
 block.go      | 10 ++--------
 block_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 8 deletions(-)

diff --git a/block.go b/block.go
index 7fb472e..740ad46 100644
--- a/block.go
+++ b/block.go
@@ -1153,6 +1153,7 @@ gatherlines:
 		// and move on to the next line
 		if p.isEmpty(data[line:i]) > 0 {
 			containsBlankLine = true
+			raw.Write(data[line:i])
 			line = i
 			continue
 		}
@@ -1220,17 +1221,10 @@ gatherlines:
 
 		// a blank line means this should be parsed as a block
 		case containsBlankLine:
-			raw.WriteByte('\n')
 			*flags |= LIST_ITEM_CONTAINS_BLOCK
 		}
 
-		// if this line was preceeded by one or more blanks,
-		// re-introduce the blank into the buffer
-		if containsBlankLine {
-			containsBlankLine = false
-			raw.WriteByte('\n')
-
-		}
+		containsBlankLine = false
 
 		// add the line into the working buffer without prefix
 		raw.Write(data[line+indent : i])
diff --git a/block_test.go b/block_test.go
index 60da6ca..f59268e 100644
--- a/block_test.go
+++ b/block_test.go
@@ -702,10 +702,41 @@ func TestUnorderedList(t *testing.T) {
 
 		"* List\n\n    * sublist\n\n    normal text\n\n    * another sublist\n",
 		"<ul>\n<li><p>List</p>\n\n<ul>\n<li>sublist</li>\n</ul>\n\n<p>normal text</p>\n\n<ul>\n<li>another sublist</li>\n</ul></li>\n</ul>\n",
+
+		`* Foo
+
+        bar
+
+        qux
+`,
+		`<ul>
+<li><p>Foo</p>
+
+<pre><code>bar
+
+qux
+</code></pre></li>
+</ul>
+`,
 	}
 	doTestsBlock(t, tests, 0)
 }
 
+func TestFencedCodeBlockWithinList(t *testing.T) {
+	doTestsBlock(t, []string{
+		"* Foo\n\n    ```\n    bar\n\n    qux\n    ```\n",
+		`<ul>
+<li><p>Foo</p>
+
+<pre><code>bar
+
+qux
+</code></pre></li>
+</ul>
+`,
+	}, EXTENSION_FENCED_CODE)
+}
+
 func TestOrderedList(t *testing.T) {
 	var tests = []string{
 		"1. Hello\n",
@@ -798,6 +829,26 @@ func TestOrderedList(t *testing.T) {
 
 		"1. numbers\n1. are ignored\n",
 		"<ol>\n<li>numbers</li>\n<li>are ignored</li>\n</ol>\n",
+
+		`1. Foo
+
+        bar
+
+
+
+        qux
+`,
+		`<ol>
+<li><p>Foo</p>
+
+<pre><code>bar
+
+
+
+qux
+</code></pre></li>
+</ol>
+`,
 	}
 	doTestsBlock(t, tests, 0)
 }

From 2ee51ae2a0819c1a21a0b86398013cbc605fc9c7 Mon Sep 17 00:00:00 2001
From: shawnps <shawn@signalsciences.com>
Date: Fri, 22 Jan 2016 16:04:49 -0800
Subject: [PATCH 15/42] gofmt -s

---
 markdown.go | 76 ++++++++++++++++++++++++++---------------------------
 1 file changed, 38 insertions(+), 38 deletions(-)

diff --git a/markdown.go b/markdown.go
index f4a55a8..ed438de 100644
--- a/markdown.go
+++ b/markdown.go
@@ -105,46 +105,46 @@ const (
 // blockTags is a set of tags that are recognized as HTML block tags.
 // Any of these can be included in markdown text without special escaping.
 var blockTags = map[string]struct{}{
-	"blockquote": struct{}{},
-	"del":        struct{}{},
-	"div":        struct{}{},
-	"dl":         struct{}{},
-	"fieldset":   struct{}{},
-	"form":       struct{}{},
-	"h1":         struct{}{},
-	"h2":         struct{}{},
-	"h3":         struct{}{},
-	"h4":         struct{}{},
-	"h5":         struct{}{},
-	"h6":         struct{}{},
-	"iframe":     struct{}{},
-	"ins":        struct{}{},
-	"math":       struct{}{},
-	"noscript":   struct{}{},
-	"ol":         struct{}{},
-	"pre":        struct{}{},
-	"p":          struct{}{},
-	"script":     struct{}{},
-	"style":      struct{}{},
-	"table":      struct{}{},
-	"ul":         struct{}{},
+	"blockquote": {},
+	"del":        {},
+	"div":        {},
+	"dl":         {},
+	"fieldset":   {},
+	"form":       {},
+	"h1":         {},
+	"h2":         {},
+	"h3":         {},
+	"h4":         {},
+	"h5":         {},
+	"h6":         {},
+	"iframe":     {},
+	"ins":        {},
+	"math":       {},
+	"noscript":   {},
+	"ol":         {},
+	"pre":        {},
+	"p":          {},
+	"script":     {},
+	"style":      {},
+	"table":      {},
+	"ul":         {},
 
 	// HTML5
-	"address":    struct{}{},
-	"article":    struct{}{},
-	"aside":      struct{}{},
-	"canvas":     struct{}{},
-	"figcaption": struct{}{},
-	"figure":     struct{}{},
-	"footer":     struct{}{},
-	"header":     struct{}{},
-	"hgroup":     struct{}{},
-	"main":       struct{}{},
-	"nav":        struct{}{},
-	"output":     struct{}{},
-	"progress":   struct{}{},
-	"section":    struct{}{},
-	"video":      struct{}{},
+	"address":    {},
+	"article":    {},
+	"aside":      {},
+	"canvas":     {},
+	"figcaption": {},
+	"figure":     {},
+	"footer":     {},
+	"header":     {},
+	"hgroup":     {},
+	"main":       {},
+	"nav":        {},
+	"output":     {},
+	"progress":   {},
+	"section":    {},
+	"video":      {},
 }
 
 // Renderer is the rendering interface.

From 1d94959fea02ce16c17107ed8e04726e10a25f3c Mon Sep 17 00:00:00 2001
From: Shawn Smith <shawnpsmith@gmail.com>
Date: Wed, 16 Mar 2016 15:21:50 +0900
Subject: [PATCH 16/42] fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index dd8eb16..7650ce4 100644
--- a/README.md
+++ b/README.md
@@ -114,7 +114,7 @@ All features of Sundown are supported, including:
     know and send me the input that does it.
 
     NOTE: "safety" in this context means *runtime safety only*. In order to
-    protect yourself agains JavaScript injection in untrusted content, see
+    protect yourself against JavaScript injection in untrusted content, see
     [this example](https://github.com/russross/blackfriday#sanitize-untrusted-content).
 
 *   **Fast processing**. It is fast enough to render on-demand in

From 232d06cf99dd8b1236b4f816b8e5b95e004d5cf2 Mon Sep 17 00:00:00 2001
From: Tom Kwok <contact@tomkwok.com>
Date: Sat, 2 Apr 2016 22:11:14 +0800
Subject: [PATCH 17/42] Fix regression: index out of range panic in reference
 link (#172, #173)

---
 markdown.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/markdown.go b/markdown.go
index ed438de..aea997a 100644
--- a/markdown.go
+++ b/markdown.go
@@ -635,6 +635,9 @@ func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffse
 		i++
 	}
 	linkOffset = i
+	if i == len(data) {
+		return
+	}
 	for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
 		i++
 	}

From 69f51afe42974676426fd208d692b01f0bf58d3c Mon Sep 17 00:00:00 2001
From: Tom Kwok <contact@tomkwok.com>
Date: Sun, 3 Apr 2016 21:08:32 +0800
Subject: [PATCH 18/42] Add test for issue 172

---
 inline_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/inline_test.go b/inline_test.go
index 3ae2561..9fb937b 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -722,6 +722,10 @@ func TestReferenceLink(t *testing.T) {
 
 		"[link][ref]\n   [ref]: /url/",
 		"<p><a href=\"/url/\">link</a></p>\n",
+
+		// Issue 172 in blackfriday
+		"[]:<",
+		"<p>[]:&lt;</p>\n",
 	}
 	doLinkTestsInline(t, tests)
 }

From fc997ac2ba5819bbc83936fc6b10fb90adc5fa10 Mon Sep 17 00:00:00 2001
From: Dmitri Shuralyov <shurcooL@gmail.com>
Date: Thu, 28 Apr 2016 22:47:04 -0700
Subject: [PATCH 19/42] Travis: Run gofmt -s, go vet, go test -race, add Go
 1.6.

Also make tip a fast-finish allowed failure. That way, if CI fails on
tip due to a temporary issue with tip, it will not break build status.
However, it's still possible to see tip build status by looking at CI
details page.

Do not run go vet with Go 1.4 or older since it's not included in the
standard library, and it's no longer available in external standard
library.

Add godoc badge to README.md.
---
 .travis.yml | 40 ++++++++++++++++++++++++++--------------
 README.md   |  2 +-
 2 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 208fd25..a1687f1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,18 +1,30 @@
-# Travis CI (http://travis-ci.org/) is a continuous integration service for
-# open source projects. This file configures it to run unit tests for
-# blackfriday.
-
+sudo: false
 language: go
-
 go:
-    - 1.2
-    - 1.3
-    - 1.4
-    - 1.5
-
+  - 1.5.4
+  - 1.6.2
+  - tip
+matrix:
+  include:
+    - go: 1.2.2
+      script:
+        - go get -t -v ./...
+        - go test -v -race ./...
+    - go: 1.3.3
+      script:
+        - go get -t -v ./...
+        - go test -v -race ./...
+    - go: 1.4.3
+      script:
+        - go get -t -v ./...
+        - go test -v -race ./...
+  allow_failures:
+    - go: tip
+  fast_finish: true
 install:
-    - go get -d -t -v ./...
-    - go build -v ./...
-
+  - # Do nothing. This is needed to prevent default install action "go get -t -v ./..." from happening here (we want it to happen inside script step).
 script:
-    - go test -v ./...
+  - go get -t -v ./...
+  - diff -u <(echo -n) <(gofmt -d -s .)
+  - go tool vet .
+  - go test -v -race ./...
diff --git a/README.md b/README.md
index 7650ce4..1879ce7 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-Blackfriday [![Build Status](https://travis-ci.org/russross/blackfriday.svg?branch=master)](https://travis-ci.org/russross/blackfriday)
+Blackfriday [![Build Status](https://travis-ci.org/russross/blackfriday.svg?branch=master)](https://travis-ci.org/russross/blackfriday) [![GoDoc](https://godoc.org/github.com/russross/blackfriday?status.svg)](https://godoc.org/github.com/russross/blackfriday)
 ===========
 
 Blackfriday is a [Markdown][1] processor implemented in [Go][2]. It

From 0529888f557077c1621b9edb5bf9aa0ee623c00c Mon Sep 17 00:00:00 2001
From: Cameron Moore <moorereason@gmail.com>
Date: Mon, 2 May 2016 08:54:19 -0500
Subject: [PATCH 20/42] Fix adjacent list merging

Prevent adjacent lists of differing types from being merged into a
single list.  No options are provided to enable the previous behavior.

Fixes #235
---
 block.go      | 7 +++++++
 block_test.go | 9 +++++++++
 2 files changed, 16 insertions(+)

diff --git a/block.go b/block.go
index 740ad46..f9ce9e3 100644
--- a/block.go
+++ b/block.go
@@ -1174,6 +1174,13 @@ gatherlines:
 			p.dliPrefix(chunk) > 0:
 
 			if containsBlankLine {
+				// end the list if the type changed after a blank line
+				if (*flags&LIST_TYPE_ORDERED != 0 && p.uliPrefix(chunk) > 0) ||
+					(*flags&LIST_TYPE_ORDERED == 0 && p.oliPrefix(chunk) > 0) {
+
+					*flags |= LIST_ITEM_END_OF_LIST
+					break gatherlines
+				}
 				*flags |= LIST_ITEM_CONTAINS_BLOCK
 			}
 
diff --git a/block_test.go b/block_test.go
index f59268e..bb3a50b 100644
--- a/block_test.go
+++ b/block_test.go
@@ -657,6 +657,9 @@ func TestUnorderedList(t *testing.T) {
 		"Paragraph\n\n* Linebreak\n",
 		"<p>Paragraph</p>\n\n<ul>\n<li>Linebreak</li>\n</ul>\n",
 
+		"*   List\n\n1. Spacer Mixed listing\n",
+		"<ul>\n<li>List</li>\n</ul>\n\n<ol>\n<li>Spacer Mixed listing</li>\n</ol>\n",
+
 		"*   List\n    * Nested list\n",
 		"<ul>\n<li>List\n\n<ul>\n<li>Nested list</li>\n</ul></li>\n</ul>\n",
 
@@ -815,6 +818,12 @@ func TestOrderedList(t *testing.T) {
 		"1. List\n\n          code block with spaces\n",
 		"<ol>\n<li><p>List</p>\n\n<pre><code>  code block with spaces\n</code></pre></li>\n</ol>\n",
 
+		"1. List\n\n* Spacer Mixed listing\n",
+		"<ol>\n<li>List</li>\n</ol>\n\n<ul>\n<li>Spacer Mixed listing</li>\n</ul>\n",
+
+		"1. List\n* Mixed listing\n",
+		"<ol>\n<li>List</li>\n<li>Mixed listing</li>\n</ol>\n",
+
 		"1. List\n    * Mixted list\n",
 		"<ol>\n<li>List\n\n<ul>\n<li>Mixted list</li>\n</ul></li>\n</ol>\n",
 

From acc07e6144689596cc6c5003e407dced0c27b92e Mon Sep 17 00:00:00 2001
From: Cameron Moore <moorereason@gmail.com>
Date: Fri, 6 May 2016 13:27:55 -0500
Subject: [PATCH 21/42] Fix definition lists that contain other lists

Fixes #263
---
 block.go      | 5 +++--
 block_test.go | 8 ++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/block.go b/block.go
index f9ce9e3..3f4af78 100644
--- a/block.go
+++ b/block.go
@@ -1175,8 +1175,9 @@ gatherlines:
 
 			if containsBlankLine {
 				// end the list if the type changed after a blank line
-				if (*flags&LIST_TYPE_ORDERED != 0 && p.uliPrefix(chunk) > 0) ||
-					(*flags&LIST_TYPE_ORDERED == 0 && p.oliPrefix(chunk) > 0) {
+				if indent <= itemIndent &&
+					((*flags&LIST_TYPE_ORDERED != 0 && p.uliPrefix(chunk) > 0) ||
+						(*flags&LIST_TYPE_ORDERED == 0 && p.oliPrefix(chunk) > 0)) {
 
 					*flags |= LIST_ITEM_END_OF_LIST
 					break gatherlines
diff --git a/block_test.go b/block_test.go
index bb3a50b..f3618fe 100644
--- a/block_test.go
+++ b/block_test.go
@@ -960,6 +960,14 @@ func TestDefinitionList(t *testing.T) {
 			"<dd><p>Definition b</p></dd>\n" +
 			"</dl>\n" +
 			"\n<p>Text 2</p>\n",
+
+		"Term 1\n:   Definition a\n\n    Text 1\n\n    1. First\n    2. Second",
+		"<dl>\n" +
+			"<dt>Term 1</dt>\n" +
+			"<dd><p>Definition a</p>\n\n" +
+			"<p>Text 1</p>\n\n" +
+			"<ol>\n<li>First</li>\n<li>Second</li>\n</ol></dd>\n" +
+			"</dl>\n",
 	}
 	doTestsBlock(t, tests, EXTENSION_DEFINITION_LISTS)
 }

From b88a9bd458157a51de7629412f806d2891b735d3 Mon Sep 17 00:00:00 2001
From: Dmitri Shuralyov <shurcooL@gmail.com>
Date: Tue, 24 May 2016 21:59:52 -0700
Subject: [PATCH 22/42] Avoid unneeded break statement.

This is purely a style change with no behavior difference.

In Go (unlike most other languages), case statements in a switch
don't need an explicit break statement, it happens by default. Adding
it explicitly is possible, but has no effect.

In this case, having the break statement hurts readability because
it's hard to tell if it's a mistake, and the break was intended to
break out of the outer for loop, rather than do nothing for the switch
statement. So, remove it, to make it more clear that there is no
bug here.
---
 inline.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inline.go b/inline.go
index 7dac6b8..cb00ed6 100644
--- a/inline.go
+++ b/inline.go
@@ -938,7 +938,7 @@ func isMailtoAutoLink(data []byte) int {
 			nb++
 
 		case '-', '.', '_':
-			break
+			// Do nothing.
 
 		case '>':
 			if nb == 1 {

From 96537c6eaa663cfa2a46ddf3e380acd163152601 Mon Sep 17 00:00:00 2001
From: Dmitri Shuralyov <shurcooL@gmail.com>
Date: Fri, 15 Jul 2016 13:52:01 -0400
Subject: [PATCH 23/42] Create a place for document-level unit tests.

These will be helpful for catching regressions or changes in behavior
to edge cases such as empty input, or specifically crafted inputs that
may cause panics, etc.

Move test for issue #172 there since it's a document-level test, not an
inline one.

Add test for issue #173.

Make some things more consistent.

Don't use a named receiver in methods that don't use it. This makes the
code more readable since one can more quickly tell the inputs to the
method.
---
 block.go         |  4 +--
 inline_test.go   | 16 ++++-------
 markdown_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++
 ref_test.go      |  2 +-
 4 files changed, 83 insertions(+), 14 deletions(-)
 create mode 100644 markdown_test.go

diff --git a/block.go b/block.go
index 3f4af78..3fa1e92 100644
--- a/block.go
+++ b/block.go
@@ -515,7 +515,7 @@ func (p *parser) htmlFindEnd(tag string, data []byte) int {
 	return i + skip
 }
 
-func (p *parser) isEmpty(data []byte) int {
+func (*parser) isEmpty(data []byte) int {
 	// it is okay to call isEmpty on an empty buffer
 	if len(data) == 0 {
 		return 0
@@ -530,7 +530,7 @@ func (p *parser) isEmpty(data []byte) int {
 	return i + 1
 }
 
-func (p *parser) isHRule(data []byte) bool {
+func (*parser) isHRule(data []byte) bool {
 	i := 0
 
 	// skip up to three spaces
diff --git a/inline_test.go b/inline_test.go
index 9fb937b..bcaba09 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -59,13 +59,11 @@ func doTestsInlineParam(t *testing.T, tests []string, opts Options, htmlFlags in
 	params HtmlRendererParameters) {
 	// catch and report panics
 	var candidate string
-	/*
-		defer func() {
-			if err := recover(); err != nil {
-				t.Errorf("\npanic while processing [%#v] (%v)\n", candidate, err)
-			}
-		}()
-	*/
+	defer func() {
+		if err := recover(); err != nil {
+			t.Errorf("\npanic while processing [%#v]: %s\n", candidate, err)
+		}
+	}()
 
 	for i := 0; i+1 < len(tests); i += 2 {
 		input := tests[i]
@@ -722,10 +720,6 @@ func TestReferenceLink(t *testing.T) {
 
 		"[link][ref]\n   [ref]: /url/",
 		"<p><a href=\"/url/\">link</a></p>\n",
-
-		// Issue 172 in blackfriday
-		"[]:<",
-		"<p>[]:&lt;</p>\n",
 	}
 	doLinkTestsInline(t, tests)
 }
diff --git a/markdown_test.go b/markdown_test.go
new file mode 100644
index 0000000..d897644
--- /dev/null
+++ b/markdown_test.go
@@ -0,0 +1,75 @@
+//
+// Blackfriday Markdown Processor
+// Available at http://github.com/russross/blackfriday
+//
+// Copyright © 2011 Russ Ross <russ@russross.com>.
+// Distributed under the Simplified BSD License.
+// See README.md for details.
+//
+
+//
+// Unit tests for full document parsing and rendering
+//
+
+package blackfriday
+
+import (
+	"testing"
+)
+
+func runMarkdown(input string) string {
+	return string(MarkdownCommon([]byte(input)))
+}
+
+func doTests(t *testing.T, tests []string) {
+	// catch and report panics
+	var candidate string
+	defer func() {
+		if err := recover(); err != nil {
+			t.Errorf("\npanic while processing [%#v]: %s\n", candidate, err)
+		}
+	}()
+
+	for i := 0; i+1 < len(tests); i += 2 {
+		input := tests[i]
+		candidate = input
+		expected := tests[i+1]
+		actual := runMarkdown(candidate)
+		if actual != expected {
+			t.Errorf("\nInput   [%#v]\nExpected[%#v]\nActual  [%#v]",
+				candidate, expected, actual)
+		}
+
+		// now test every substring to stress test bounds checking
+		if !testing.Short() {
+			for start := 0; start < len(input); start++ {
+				for end := start + 1; end <= len(input); end++ {
+					candidate = input[start:end]
+					_ = runMarkdown(candidate)
+				}
+			}
+		}
+	}
+}
+
+func TestDocument(t *testing.T) {
+	var tests = []string{
+		// Empty document.
+		"",
+		"",
+
+		" ",
+		"",
+
+		// This shouldn't panic.
+		// https://github.com/russross/blackfriday/issues/172
+		"[]:<",
+		"<p>[]:&lt;</p>\n",
+
+		// This shouldn't panic.
+		// https://github.com/russross/blackfriday/issues/173
+		"   [",
+		"<p>[</p>\n",
+	}
+	doTests(t, tests)
+}
diff --git a/ref_test.go b/ref_test.go
index 770439c..733a63a 100644
--- a/ref_test.go
+++ b/ref_test.go
@@ -29,7 +29,7 @@ func doTestsReference(t *testing.T, files []string, flag int) {
 	var candidate string
 	defer func() {
 		if err := recover(); err != nil {
-			t.Errorf("\npanic while processing [%#v]\n", candidate)
+			t.Errorf("\npanic while processing [%#v]: %s\n", candidate, err)
 		}
 	}()
 

From 00496765999eff4cae439cf36e45b661bdb971ad Mon Sep 17 00:00:00 2001
From: Dmitri Shuralyov <shurcooL@gmail.com>
Date: Fri, 15 Jul 2016 14:41:27 -0400
Subject: [PATCH 24/42] Improve fence line detection.

Rename isFenceCode to isFenceLine, document it, add tests.

Add support for making newline optional, this will be needed in future
commits.
---
 block.go      | 54 +++++++++++++++++++++---------------------
 block_test.go | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 27 deletions(-)

diff --git a/block.go b/block.go
index 3fa1e92..021a54d 100644
--- a/block.go
+++ b/block.go
@@ -559,21 +559,24 @@ func (*parser) isHRule(data []byte) bool {
 	return n >= 3
 }
 
-func (p *parser) isFencedCode(data []byte, syntax **string, oldmarker string) (skip int, marker string) {
+// isFenceLine checks if there's a fence line (e.g., ``` or ``` go) at the beginning of data,
+// and returns the end index if so, or 0 otherwise. It also returns the marker found.
+// If syntax is not nil, it gets set to the syntax specified in the fence line.
+// A final newline is mandatory to recognize the fence line, unless newlineOptional is true.
+func isFenceLine(data []byte, syntax *string, oldmarker string, newlineOptional bool) (end int, marker string) {
 	i, size := 0, 0
-	skip = 0
 
 	// skip up to three spaces
 	for i < len(data) && i < 3 && data[i] == ' ' {
 		i++
 	}
-	if i >= len(data) {
-		return
-	}
 
 	// check for the marker characters: ~ or `
+	if i >= len(data) {
+		return 0, ""
+	}
 	if data[i] != '~' && data[i] != '`' {
-		return
+		return 0, ""
 	}
 
 	c := data[i]
@@ -584,27 +587,28 @@ func (p *parser) isFencedCode(data []byte, syntax **string, oldmarker string) (s
 		i++
 	}
 
-	if i >= len(data) {
-		return
-	}
-
 	// the marker char must occur at least 3 times
 	if size < 3 {
-		return
+		return 0, ""
 	}
 	marker = string(data[i-size : i])
 
 	// if this is the end marker, it must match the beginning marker
 	if oldmarker != "" && marker != oldmarker {
-		return
+		return 0, ""
 	}
 
+	// TODO(shurcooL): It's probably a good idea to simplify the 2 code paths here
+	// into one, always get the syntax, and discard it if the caller doesn't care.
 	if syntax != nil {
 		syn := 0
 		i = skipChar(data, i, ' ')
 
 		if i >= len(data) {
-			return
+			if newlineOptional && i == len(data) {
+				return i, marker
+			}
+			return 0, ""
 		}
 
 		syntaxStart := i
@@ -619,7 +623,7 @@ func (p *parser) isFencedCode(data []byte, syntax **string, oldmarker string) (s
 			}
 
 			if i >= len(data) || data[i] != '}' {
-				return
+				return 0, ""
 			}
 
 			// strip all whitespace at the beginning and the end
@@ -641,22 +645,23 @@ func (p *parser) isFencedCode(data []byte, syntax **string, oldmarker string) (s
 			}
 		}
 
-		language := string(data[syntaxStart : syntaxStart+syn])
-		*syntax = &language
+		*syntax = string(data[syntaxStart : syntaxStart+syn])
 	}
 
 	i = skipChar(data, i, ' ')
 	if i >= len(data) || data[i] != '\n' {
-		return
+		if newlineOptional && i == len(data) {
+			return i, marker
+		}
+		return 0, ""
 	}
 
-	skip = i + 1
-	return
+	return i + 1, marker // Take newline into account.
 }
 
 func (p *parser) fencedCode(out *bytes.Buffer, data []byte, doRender bool) int {
-	var lang *string
-	beg, marker := p.isFencedCode(data, &lang, "")
+	var syntax string
+	beg, marker := isFenceLine(data, &syntax, "", true)
 	if beg == 0 || beg >= len(data) {
 		return 0
 	}
@@ -667,7 +672,7 @@ func (p *parser) fencedCode(out *bytes.Buffer, data []byte, doRender bool) int {
 		// safe to assume beg < len(data)
 
 		// check for the end of the code block
-		fenceEnd, _ := p.isFencedCode(data[beg:], nil, marker)
+		fenceEnd, _ := isFenceLine(data[beg:], nil, marker, true)
 		if fenceEnd != 0 {
 			beg += fenceEnd
 			break
@@ -688,11 +693,6 @@ func (p *parser) fencedCode(out *bytes.Buffer, data []byte, doRender bool) int {
 		beg = end
 	}
 
-	syntax := ""
-	if lang != nil {
-		syntax = *lang
-	}
-
 	if doRender {
 		p.r.BlockCode(out, work.Bytes(), syntax)
 	}
diff --git a/block_test.go b/block_test.go
index f3618fe..a20f5df 100644
--- a/block_test.go
+++ b/block_test.go
@@ -1636,3 +1636,68 @@ func TestCDATA(t *testing.T) {
 `,
 	}, EXTENSION_FENCED_CODE)
 }
+
+func TestIsFenceLine(t *testing.T) {
+	tests := []struct {
+		data            []byte
+		syntaxRequested bool
+		newlineOptional bool
+		wantEnd         int
+		wantMarker      string
+		wantSyntax      string
+	}{
+		{
+			data:    []byte("```"),
+			wantEnd: 0,
+		},
+		{
+			data:       []byte("```\nstuff here\n"),
+			wantEnd:    4,
+			wantMarker: "```",
+		},
+		{
+			data:    []byte("stuff here\n```\n"),
+			wantEnd: 0,
+		},
+		{
+			data:            []byte("```"),
+			newlineOptional: true,
+			wantEnd:         3,
+			wantMarker:      "```",
+		},
+		{
+			data:            []byte("```"),
+			syntaxRequested: true,
+			newlineOptional: true,
+			wantEnd:         3,
+			wantMarker:      "```",
+		},
+		{
+			data:            []byte("``` go"),
+			syntaxRequested: true,
+			newlineOptional: true,
+			wantEnd:         6,
+			wantMarker:      "```",
+			wantSyntax:      "go",
+		},
+	}
+
+	for _, test := range tests {
+		var syntax *string
+		if test.syntaxRequested {
+			syntax = new(string)
+		}
+		end, marker := isFenceLine(test.data, syntax, "```", test.newlineOptional)
+		if got, want := end, test.wantEnd; got != want {
+			t.Errorf("got end %v, want %v", got, want)
+		}
+		if got, want := marker, test.wantMarker; got != want {
+			t.Errorf("got marker %q, want %q", got, want)
+		}
+		if test.syntaxRequested {
+			if got, want := *syntax, test.wantSyntax; got != want {
+				t.Errorf("got syntax %q, want %q", got, want)
+			}
+		}
+	}
+}

From a5812bb8f2987abe2294beaf65c08baaceb2969f Mon Sep 17 00:00:00 2001
From: Dmitri Shuralyov <shurcooL@gmail.com>
Date: Fri, 15 Jul 2016 14:51:15 -0400
Subject: [PATCH 25/42] Improve fenced code block detection for first pass.

In first pass, there may not be a trailing newline after a fenced code
block yet. Make newline optional in isFenceLine when calling
fencedCodeBlock to detect the fenced code block it anyway. This is more
complex, but it avoids creating temporary buffers or modifying input in
order to maintain performance (see #148).

Document and rename fencedCode to fencedCodeBlock.

Add regression tests.

Fixes #279.
---
 block.go      | 14 +++++----
 block_test.go |  6 ++++
 markdown.go   | 80 +++++++++++++++++++++++++--------------------------
 3 files changed, 55 insertions(+), 45 deletions(-)

diff --git a/block.go b/block.go
index 021a54d..b237c7e 100644
--- a/block.go
+++ b/block.go
@@ -102,7 +102,7 @@ func (p *parser) block(out *bytes.Buffer, data []byte) {
 		// }
 		// ```
 		if p.flags&EXTENSION_FENCED_CODE != 0 {
-			if i := p.fencedCode(out, data, true); i > 0 {
+			if i := p.fencedCodeBlock(out, data, true); i > 0 {
 				data = data[i:]
 				continue
 			}
@@ -659,7 +659,10 @@ func isFenceLine(data []byte, syntax *string, oldmarker string, newlineOptional
 	return i + 1, marker // Take newline into account.
 }
 
-func (p *parser) fencedCode(out *bytes.Buffer, data []byte, doRender bool) int {
+// fencedCodeBlock returns the end index if data contains a fenced code block at the beginning,
+// or 0 otherwise. It writes to out if doRender is true, otherwise it has no side effects.
+// If doRender is true, a final newline is mandatory to recognize the fenced code block.
+func (p *parser) fencedCodeBlock(out *bytes.Buffer, data []byte, doRender bool) int {
 	var syntax string
 	beg, marker := isFenceLine(data, &syntax, "", true)
 	if beg == 0 || beg >= len(data) {
@@ -672,7 +675,8 @@ func (p *parser) fencedCode(out *bytes.Buffer, data []byte, doRender bool) int {
 		// safe to assume beg < len(data)
 
 		// check for the end of the code block
-		fenceEnd, _ := isFenceLine(data[beg:], nil, marker, true)
+		newlineOptional := !doRender
+		fenceEnd, _ := isFenceLine(data[beg:], nil, marker, newlineOptional)
 		if fenceEnd != 0 {
 			beg += fenceEnd
 			break
@@ -934,7 +938,7 @@ func (p *parser) quote(out *bytes.Buffer, data []byte) int {
 		// irregardless of any contents inside it
 		for data[end] != '\n' {
 			if p.flags&EXTENSION_FENCED_CODE != 0 {
-				if i := p.fencedCode(out, data[end:], false); i > 0 {
+				if i := p.fencedCodeBlock(out, data[end:], false); i > 0 {
 					// -1 to compensate for the extra end++ after the loop:
 					end += i - 1
 					break
@@ -1384,7 +1388,7 @@ func (p *parser) paragraph(out *bytes.Buffer, data []byte) int {
 
 		// if there's a fenced code block, paragraph is over
 		if p.flags&EXTENSION_FENCED_CODE != 0 {
-			if p.fencedCode(out, current, false) > 0 {
+			if p.fencedCodeBlock(out, current, false) > 0 {
 				p.renderParagraph(out, data[:i])
 				return i
 			}
diff --git a/block_test.go b/block_test.go
index a20f5df..6170e56 100644
--- a/block_test.go
+++ b/block_test.go
@@ -1130,6 +1130,12 @@ func TestFencedCodeBlock(t *testing.T) {
 
 		"Some text before a fenced code block\n``` oz\ncode blocks breakup paragraphs\n```\nSome text in between\n``` oz\nmultiple code blocks work okay\n```\nAnd some text after a fenced code block",
 		"<p>Some text before a fenced code block</p>\n\n<pre><code class=\"language-oz\">code blocks breakup paragraphs\n</code></pre>\n\n<p>Some text in between</p>\n\n<pre><code class=\"language-oz\">multiple code blocks work okay\n</code></pre>\n\n<p>And some text after a fenced code block</p>\n",
+
+		"```\n[]:()\n```\n",
+		"<pre><code>[]:()\n</code></pre>\n",
+
+		"```\n[]:()\n[]:)\n[]:(\n[]:x\n[]:testing\n[:testing\n\n[]:\nlinebreak\n[]()\n\n[]:\n[]()\n```",
+		"<pre><code>[]:()\n[]:)\n[]:(\n[]:x\n[]:testing\n[:testing\n\n[]:\nlinebreak\n[]()\n\n[]:\n[]()\n</code></pre>\n",
 	}
 	doTestsBlock(t, tests, EXTENSION_FENCED_CODE)
 }
diff --git a/markdown.go b/markdown.go
index aea997a..58ba68d 100644
--- a/markdown.go
+++ b/markdown.go
@@ -386,9 +386,9 @@ func MarkdownOptions(input []byte, renderer Renderer, opts Options) []byte {
 }
 
 // first pass:
-// - extract references
-// - expand tabs
 // - normalize newlines
+// - extract references (outside of fenced code blocks)
+// - expand tabs (outside of fenced code blocks)
 // - copy everything else
 func firstPass(p *parser, input []byte) []byte {
 	var out bytes.Buffer
@@ -396,46 +396,46 @@ func firstPass(p *parser, input []byte) []byte {
 	if p.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
 		tabSize = TAB_SIZE_EIGHT
 	}
-	beg, end := 0, 0
+	beg := 0
 	lastFencedCodeBlockEnd := 0
-	for beg < len(input) { // iterate over lines
-		if end = isReference(p, input[beg:], tabSize); end > 0 {
-			beg += end
-		} else { // skip to the next line
-			end = beg
-			for end < len(input) && input[end] != '\n' && input[end] != '\r' {
-				end++
-			}
-
-			if p.flags&EXTENSION_FENCED_CODE != 0 {
-				// track fenced code block boundaries to suppress tab expansion
-				// inside them:
-				if beg >= lastFencedCodeBlockEnd {
-					if i := p.fencedCode(&out, input[beg:], false); i > 0 {
-						lastFencedCodeBlockEnd = beg + i
-					}
-				}
-			}
-
-			// add the line body if present
-			if end > beg {
-				if end < lastFencedCodeBlockEnd { // Do not expand tabs while inside fenced code blocks.
-					out.Write(input[beg:end])
-				} else {
-					expandTabs(&out, input[beg:end], tabSize)
-				}
-			}
-			out.WriteByte('\n')
-
-			if end < len(input) && input[end] == '\r' {
-				end++
-			}
-			if end < len(input) && input[end] == '\n' {
-				end++
-			}
-
-			beg = end
+	for beg < len(input) {
+		// Find end of this line, then process the line.
+		end := beg
+		for end < len(input) && input[end] != '\n' && input[end] != '\r' {
+			end++
 		}
+
+		if p.flags&EXTENSION_FENCED_CODE != 0 {
+			// track fenced code block boundaries to suppress tab expansion
+			// and reference extraction inside them:
+			if beg >= lastFencedCodeBlockEnd {
+				if i := p.fencedCodeBlock(&out, input[beg:], false); i > 0 {
+					lastFencedCodeBlockEnd = beg + i
+				}
+			}
+		}
+
+		// add the line body if present
+		if end > beg {
+			if end < lastFencedCodeBlockEnd { // Do not expand tabs while inside fenced code blocks.
+				out.Write(input[beg:end])
+			} else if refEnd := isReference(p, input[beg:], tabSize); refEnd > 0 {
+				beg += refEnd
+				continue
+			} else {
+				expandTabs(&out, input[beg:end], tabSize)
+			}
+		}
+
+		if end < len(input) && input[end] == '\r' {
+			end++
+		}
+		if end < len(input) && input[end] == '\n' {
+			end++
+		}
+		out.WriteByte('\n')
+
+		beg = end
 	}
 
 	// empty input?

From 93622da34e54fb6529bfb7c57e710f37a8d9cbd8 Mon Sep 17 00:00:00 2001
From: Dmitri Shuralyov <shurcooL@gmail.com>
Date: Sat, 16 Jul 2016 11:34:03 -0400
Subject: [PATCH 26/42] Make newline mandatory for opening fence line. (#281)

This was an unintended typo/mistake in #280.

This is stricter, and it's fine. The opening fence line will always need to have a newline.

Add another test for isFenceLine.
---
 block.go      | 2 +-
 block_test.go | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/block.go b/block.go
index b237c7e..0b5510d 100644
--- a/block.go
+++ b/block.go
@@ -664,7 +664,7 @@ func isFenceLine(data []byte, syntax *string, oldmarker string, newlineOptional
 // If doRender is true, a final newline is mandatory to recognize the fenced code block.
 func (p *parser) fencedCodeBlock(out *bytes.Buffer, data []byte, doRender bool) int {
 	var syntax string
-	beg, marker := isFenceLine(data, &syntax, "", true)
+	beg, marker := isFenceLine(data, &syntax, "", false)
 	if beg == 0 || beg >= len(data) {
 		return 0
 	}
diff --git a/block_test.go b/block_test.go
index 6170e56..62d9a47 100644
--- a/block_test.go
+++ b/block_test.go
@@ -1661,6 +1661,12 @@ func TestIsFenceLine(t *testing.T) {
 			wantEnd:    4,
 			wantMarker: "```",
 		},
+		{
+			data:            []byte("```\nstuff here\n"),
+			syntaxRequested: true,
+			wantEnd:         4,
+			wantMarker:      "```",
+		},
 		{
 			data:    []byte("stuff here\n```\n"),
 			wantEnd: 0,

From 35eb537633d9950afc8ae7bdf0edb6134584e9fc Mon Sep 17 00:00:00 2001
From: Dmitri Shuralyov <shurcooL@gmail.com>
Date: Thu, 8 Sep 2016 23:28:54 -0700
Subject: [PATCH 27/42] Set LIST_ITEM_END_OF_LIST when list is at end of
 document. (#305)

The LIST_ITEM_END_OF_LIST flag is an internal flag passed to renderers
when rendering list items. It's normally set for the last item in the
list.

This change fixes the issue where that flag wasn't set in situations
where the Markdown document ends with the list being the last block
level item in it.

The cases above detect and set LIST_ITEM_END_OF_LIST flag when the list
ends because another thing begins, but they miss it when the end of
document is reached.

No tests here because this subtle internal behavior is hard to test and
would require quite a bit of testing/mock infrastructure.

Helps shurcooL/markdownfmt#30.
---
 block.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/block.go b/block.go
index 0b5510d..9cf451f 100644
--- a/block.go
+++ b/block.go
@@ -1244,6 +1244,12 @@ gatherlines:
 		line = i
 	}
 
+	// If reached end of data, the Renderer.ListItem call we're going to make below
+	// is definitely the last in the list.
+	if line >= len(data) {
+		*flags |= LIST_ITEM_END_OF_LIST
+	}
+
 	rawBytes := raw.Bytes()
 
 	// render the contents of the list item

From 5f33e7b7878355cd2b7e6b8eefc48a5472c69f70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?K=C3=A9vin=20Dunglas?= <dunglas@gmail.com>
Date: Mon, 3 Oct 2016 18:27:22 +0200
Subject: [PATCH 28/42] Document using fenced code blocks with bluemonday
 (#309)

Document using fenced code blocks with bluemonday
---
 README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/README.md b/README.md
index 1879ce7..3c113cb 100644
--- a/README.md
+++ b/README.md
@@ -169,6 +169,15 @@ implements the following extensions:
     You can use 3 or more backticks to mark the beginning of the
     block, and the same number to mark the end of the block.
 
+    To preserve classes of fenced code blocks while using the bluemonday
+    HTML sanitizer, use the following policy:
+
+    ``` go
+    p := bluemonday.UGCPolicy()
+    p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
+    html := p.SanitizeBytes(unsafe)
+    ```
+
 *   **Definition lists**. A simple definition list is made of a single-line
     term followed by a colon and the definition for that term.
 

From 3ffe8c7f6b8f7e2026a2aee3477fb63fc33b33ff Mon Sep 17 00:00:00 2001
From: choueric <zhssmail@gmail.com>
Date: Wed, 22 Feb 2017 09:15:46 +0800
Subject: [PATCH 29/42] add extension to join lines

---
 inline.go   | 4 ++++
 markdown.go | 1 +
 2 files changed, 5 insertions(+)

diff --git a/inline.go b/inline.go
index cb00ed6..c1f7475 100644
--- a/inline.go
+++ b/inline.go
@@ -170,6 +170,10 @@ func lineBreak(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 	precededByBackslash := offset >= 1 && data[offset-1] == '\\' // see http://spec.commonmark.org/0.18/#example-527
 	precededByBackslash = precededByBackslash && p.flags&EXTENSION_BACKSLASH_LINE_BREAK != 0
 
+	if p.flags&EXTENSION_JOIN_LINES != 0 {
+		return 1
+	}
+
 	// should there be a hard line break here?
 	if p.flags&EXTENSION_HARD_LINE_BREAK == 0 && !precededByTwoSpaces && !precededByBackslash {
 		return 0
diff --git a/markdown.go b/markdown.go
index 58ba68d..09fc2d7 100644
--- a/markdown.go
+++ b/markdown.go
@@ -46,6 +46,7 @@ const (
 	EXTENSION_AUTO_HEADER_IDS                        // Create the header ID from the text
 	EXTENSION_BACKSLASH_LINE_BREAK                   // translate trailing backslashes into line breaks
 	EXTENSION_DEFINITION_LISTS                       // render definition lists
+	EXTENSION_JOIN_LINES                             // delete newline and join lines
 
 	commonHtmlFlags = 0 |
 		HTML_USE_XHTML |

From 8098dab4eb9c32e0145407e1222b0e7c698f2d76 Mon Sep 17 00:00:00 2001
From: choueric <zhssmail@gmail.com>
Date: Wed, 22 Feb 2017 09:18:21 +0800
Subject: [PATCH 30/42] add testcase for joinLines extension

---
 markdown_test.go          | 30 ++++++++++++++++++++++++++++++
 testdata/zhJoinLines.text |  8 ++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 testdata/zhJoinLines.text

diff --git a/markdown_test.go b/markdown_test.go
index d897644..34d5efe 100644
--- a/markdown_test.go
+++ b/markdown_test.go
@@ -14,6 +14,8 @@
 package blackfriday
 
 import (
+	"io/ioutil"
+	"os"
 	"testing"
 )
 
@@ -73,3 +75,31 @@ func TestDocument(t *testing.T) {
 	}
 	doTests(t, tests)
 }
+
+func TestJoinLines(t *testing.T) {
+	result := `<h1>标题</h1>
+
+<p>第一行文字。</p>
+
+<p>第二行文字。</p>
+`
+
+	file, err := os.Open("testdata/zhJoinLines.text")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	input, err := ioutil.ReadAll(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	opt := Options{Extensions: commonExtensions | EXTENSION_JOIN_LINES}
+	renderer := HtmlRenderer(commonHtmlFlags, "", "")
+	output := MarkdownOptions(input, renderer, opt)
+
+	if string(output) != result {
+		t.Error("output dose not match.")
+	}
+}
diff --git a/testdata/zhJoinLines.text b/testdata/zhJoinLines.text
new file mode 100644
index 0000000..704ff27
--- /dev/null
+++ b/testdata/zhJoinLines.text
@@ -0,0 +1,8 @@
+# 标题
+
+第一
+行文字。
+
+第
+二
+行文字。

From f0bb45f44c5407a0c458d18e0f285126006e1ae7 Mon Sep 17 00:00:00 2001
From: choueric <zhssmail@gmail.com>
Date: Fri, 17 Mar 2017 17:21:30 +0800
Subject: [PATCH 31/42] modify testcase of joinLine.

1. delete testdata 'zhJoinLines.txt'.
2. move the testcase from markdown_test.go into block_test.go.
---
 block_test.go             | 25 +++++++++++++++++++++++++
 markdown_test.go          | 30 ------------------------------
 testdata/zhJoinLines.text |  8 --------
 3 files changed, 25 insertions(+), 38 deletions(-)
 delete mode 100644 testdata/zhJoinLines.text

diff --git a/block_test.go b/block_test.go
index 62d9a47..83ed2e1 100644
--- a/block_test.go
+++ b/block_test.go
@@ -1713,3 +1713,28 @@ func TestIsFenceLine(t *testing.T) {
 		}
 	}
 }
+
+func TestJoinLines(t *testing.T) {
+	input := `# 标题
+
+第一
+行文字。
+
+第
+二
+行文字。
+`
+	result := `<h1>标题</h1>
+
+<p>第一行文字。</p>
+
+<p>第二行文字。</p>
+`
+	opt := Options{Extensions: commonExtensions | EXTENSION_JOIN_LINES}
+	renderer := HtmlRenderer(commonHtmlFlags, "", "")
+	output := MarkdownOptions([]byte(input), renderer, opt)
+
+	if string(output) != result {
+		t.Error("output dose not match.")
+	}
+}
diff --git a/markdown_test.go b/markdown_test.go
index 34d5efe..d897644 100644
--- a/markdown_test.go
+++ b/markdown_test.go
@@ -14,8 +14,6 @@
 package blackfriday
 
 import (
-	"io/ioutil"
-	"os"
 	"testing"
 )
 
@@ -75,31 +73,3 @@ func TestDocument(t *testing.T) {
 	}
 	doTests(t, tests)
 }
-
-func TestJoinLines(t *testing.T) {
-	result := `<h1>标题</h1>
-
-<p>第一行文字。</p>
-
-<p>第二行文字。</p>
-`
-
-	file, err := os.Open("testdata/zhJoinLines.text")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer file.Close()
-
-	input, err := ioutil.ReadAll(file)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	opt := Options{Extensions: commonExtensions | EXTENSION_JOIN_LINES}
-	renderer := HtmlRenderer(commonHtmlFlags, "", "")
-	output := MarkdownOptions(input, renderer, opt)
-
-	if string(output) != result {
-		t.Error("output dose not match.")
-	}
-}
diff --git a/testdata/zhJoinLines.text b/testdata/zhJoinLines.text
deleted file mode 100644
index 704ff27..0000000
--- a/testdata/zhJoinLines.text
+++ /dev/null
@@ -1,8 +0,0 @@
-# 标题
-
-第一
-行文字。
-
-第
-二
-行文字。

From 0ba0f2b6ed7c475a92e4df8641825cb7a11d1fa3 Mon Sep 17 00:00:00 2001
From: Dmitri Shuralyov <shurcooL@gmail.com>
Date: Tue, 9 May 2017 02:07:14 -0400
Subject: [PATCH 32/42] Document SanitizedAnchorName algorithm, copy
 implementation. (#352)

The goal of this change is to reduce number of non-standard library
packages (repositories) that blackfriday imports from 1 to 0, and in
turn, reduce the cost of importing blackfriday into other projects.

Do so by documenting the algorithm of SanitizedAnchorName, and include
a copy of the small function inside blackfriday itself. The same
functionality continues to be available in the original location,
github.com/shurcooL/sanitized_anchor_name.Create. It can be used by
existing users and those that look for a small package, and don't need
all of blackfriday functionality. Existing users of blackfriday can use
the new SanitizedAnchorName function directly and avoid an extra
package import.

Resolves #350.
---
 README.md     | 16 ++++++++++++++++
 block.go      | 28 ++++++++++++++++++++++++----
 block_test.go | 41 +++++++++++++++++++++++++++++++++++++++++
 doc.go        | 32 ++++++++++++++++++++++++++++++++
 markdown.go   |  3 ---
 5 files changed, 113 insertions(+), 7 deletions(-)
 create mode 100644 doc.go

diff --git a/README.md b/README.md
index 3c113cb..0d1ac9a 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,22 @@ installed in `$GOPATH/bin`.  This is a statically-linked binary that
 can be copied to wherever you need it without worrying about
 dependencies and library versions.
 
+### Sanitized anchor names
+
+Blackfriday includes an algorithm for creating sanitized anchor names
+corresponding to a given input text. This algorithm is used to create
+anchors for headings when `EXTENSION_AUTO_HEADER_IDS` is enabled. The
+algorithm has a specification, so that other packages can create
+compatible anchor names and links to those anchors.
+
+The specification is located at https://godoc.org/github.com/russross/blackfriday#hdr-Sanitized_Anchor_Names.
+
+[`SanitizedAnchorName`](https://godoc.org/github.com/russross/blackfriday#SanitizedAnchorName) exposes this functionality, and can be used to
+create compatible links to the anchor names generated by blackfriday.
+This algorithm is also implemented in a small standalone package at
+[`github.com/shurcooL/sanitized_anchor_name`](https://godoc.org/github.com/shurcooL/sanitized_anchor_name). It can be useful for clients
+that want a small package and don't need full functionality of blackfriday.
+
 
 Features
 --------
diff --git a/block.go b/block.go
index 9cf451f..7fc731d 100644
--- a/block.go
+++ b/block.go
@@ -15,8 +15,7 @@ package blackfriday
 
 import (
 	"bytes"
-
-	"github.com/shurcooL/sanitized_anchor_name"
+	"unicode"
 )
 
 // Parse block-level data.
@@ -243,7 +242,7 @@ func (p *parser) prefixHeader(out *bytes.Buffer, data []byte) int {
 	}
 	if end > i {
 		if id == "" && p.flags&EXTENSION_AUTO_HEADER_IDS != 0 {
-			id = sanitized_anchor_name.Create(string(data[i:end]))
+			id = SanitizedAnchorName(string(data[i:end]))
 		}
 		work := func() bool {
 			p.inline(out, data[i:end])
@@ -1364,7 +1363,7 @@ func (p *parser) paragraph(out *bytes.Buffer, data []byte) int {
 
 				id := ""
 				if p.flags&EXTENSION_AUTO_HEADER_IDS != 0 {
-					id = sanitized_anchor_name.Create(string(data[prev:eol]))
+					id = SanitizedAnchorName(string(data[prev:eol]))
 				}
 
 				p.r.Header(out, work, level, id)
@@ -1428,3 +1427,24 @@ func (p *parser) paragraph(out *bytes.Buffer, data []byte) int {
 	p.renderParagraph(out, data[:i])
 	return i
 }
+
+// SanitizedAnchorName returns a sanitized anchor name for the given text.
+//
+// It implements the algorithm specified in the package comment.
+func SanitizedAnchorName(text string) string {
+	var anchorName []rune
+	futureDash := false
+	for _, r := range text {
+		switch {
+		case unicode.IsLetter(r) || unicode.IsNumber(r):
+			if futureDash && len(anchorName) > 0 {
+				anchorName = append(anchorName, '-')
+			}
+			futureDash = false
+			anchorName = append(anchorName, unicode.ToLower(r))
+		default:
+			futureDash = true
+		}
+	}
+	return string(anchorName)
+}
diff --git a/block_test.go b/block_test.go
index 83ed2e1..89d5775 100644
--- a/block_test.go
+++ b/block_test.go
@@ -1738,3 +1738,44 @@ func TestJoinLines(t *testing.T) {
 		t.Error("output dose not match.")
 	}
 }
+
+func TestSanitizedAnchorName(t *testing.T) {
+	tests := []struct {
+		text string
+		want string
+	}{
+		{
+			text: "This is a header",
+			want: "this-is-a-header",
+		},
+		{
+			text: "This is also          a header",
+			want: "this-is-also-a-header",
+		},
+		{
+			text: "main.go",
+			want: "main-go",
+		},
+		{
+			text: "Article 123",
+			want: "article-123",
+		},
+		{
+			text: "<- Let's try this, shall we?",
+			want: "let-s-try-this-shall-we",
+		},
+		{
+			text: "        ",
+			want: "",
+		},
+		{
+			text: "Hello, 世界",
+			want: "hello-世界",
+		},
+	}
+	for _, test := range tests {
+		if got := SanitizedAnchorName(test.text); got != test.want {
+			t.Errorf("SanitizedAnchorName(%q):\ngot %q\nwant %q", test.text, got, test.want)
+		}
+	}
+}
diff --git a/doc.go b/doc.go
new file mode 100644
index 0000000..9656c42
--- /dev/null
+++ b/doc.go
@@ -0,0 +1,32 @@
+// Package blackfriday is a Markdown processor.
+//
+// It translates plain text with simple formatting rules into HTML or LaTeX.
+//
+// Sanitized Anchor Names
+//
+// Blackfriday includes an algorithm for creating sanitized anchor names
+// corresponding to a given input text. This algorithm is used to create
+// anchors for headings when EXTENSION_AUTO_HEADER_IDS is enabled. The
+// algorithm is specified below, so that other packages can create
+// compatible anchor names and links to those anchors.
+//
+// The algorithm iterates over the input text, interpreted as UTF-8,
+// one Unicode code point (rune) at a time. All runes that are letters (category L)
+// or numbers (category N) are considered valid characters. They are mapped to
+// lower case, and included in the output. All other runes are considered
+// invalid characters. Invalid characters that preceed the first valid character,
+// as well as invalid character that follow the last valid character
+// are dropped completely. All other sequences of invalid characters
+// between two valid characters are replaced with a single dash character '-'.
+//
+// SanitizedAnchorName exposes this functionality, and can be used to
+// create compatible links to the anchor names generated by blackfriday.
+// This algorithm is also implemented in a small standalone package at
+// github.com/shurcooL/sanitized_anchor_name. It can be useful for clients
+// that want a small package and don't need full functionality of blackfriday.
+package blackfriday
+
+// NOTE: Keep Sanitized Anchor Name algorithm in sync with package
+//       github.com/shurcooL/sanitized_anchor_name.
+//       Otherwise, users of sanitized_anchor_name will get anchor names
+//       that are incompatible with those generated by blackfriday.
diff --git a/markdown.go b/markdown.go
index 09fc2d7..6d842d3 100644
--- a/markdown.go
+++ b/markdown.go
@@ -13,9 +13,6 @@
 //
 //
 
-// Blackfriday markdown processor.
-//
-// Translates plain text with simple formatting rules into HTML or LaTeX.
 package blackfriday
 
 import (

From a20399916c119a2c9fa4b6642044f7168c022620 Mon Sep 17 00:00:00 2001
From: choueric <zhssmail@gmail.com>
Date: Thu, 8 Jun 2017 14:43:56 +0800
Subject: [PATCH 33/42] fix duplicate and recursive footnotes. (#241)

Fix the infinite loop when there is a self-refer footnote by checking if
the reference is already added into parser.notes.

It also fixes of creating duplicate footnotes through this modification.

Add coresponding testcase.
---
 inline.go      |  2 +-
 inline_test.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 markdown.go    | 10 ++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/inline.go b/inline.go
index c1f7475..eff1383 100644
--- a/inline.go
+++ b/inline.go
@@ -498,7 +498,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 				return 0
 			}
 
-			if t == linkDeferredFootnote {
+			if t == linkDeferredFootnote && !p.isFootnote(lr) {
 				lr.noteId = len(p.notes) + 1
 				p.notes = append(p.notes, lr)
 			}
diff --git a/inline_test.go b/inline_test.go
index bcaba09..4061cfb 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -1023,6 +1023,28 @@ what happens here
 </li>
 </ol>
 </div>
+`,
+	`testing footnotes.[^a]
+
+test footnotes the second.[^b]
+
+[^a]: This is the first note[^a].
+[^b]: this is the second note.[^a]
+`,
+	`<p>testing footnotes.<sup class="footnote-ref" id="fnref:a"><a rel="footnote" href="#fn:a">1</a></sup></p>
+
+<p>test footnotes the second.<sup class="footnote-ref" id="fnref:b"><a rel="footnote" href="#fn:b">2</a></sup></p>
+<div class="footnotes">
+
+<hr />
+
+<ol>
+<li id="fn:a">This is the first note<sup class="footnote-ref" id="fnref:a"><a rel="footnote" href="#fn:a">1</a></sup>.
+</li>
+<li id="fn:b">this is the second note.<sup class="footnote-ref" id="fnref:a"><a rel="footnote" href="#fn:a">1</a></sup>
+</li>
+</ol>
+</div>
 `,
 }
 
@@ -1076,6 +1098,34 @@ func TestNestedFootnotes(t *testing.T) {
 </li>
 </ol>
 </div>
+`,
+		`This uses footnote A.[^A]
+
+This uses footnote C.[^C]
+
+[^A]:
+  A note. use itself.[^A]
+[^B]:
+  B note, uses A to test duplicate.[^A]
+[^C]:
+  C note, uses B.[^B]
+`,
+		`<p>This uses footnote A.<sup class="footnote-ref" id="fnref:A"><a rel="footnote" href="#fn:A">1</a></sup></p>
+
+<p>This uses footnote C.<sup class="footnote-ref" id="fnref:C"><a rel="footnote" href="#fn:C">2</a></sup></p>
+<div class="footnotes">
+
+<hr />
+
+<ol>
+<li id="fn:A">A note. use itself.<sup class="footnote-ref" id="fnref:A"><a rel="footnote" href="#fn:A">1</a></sup>
+</li>
+<li id="fn:C">C note, uses B.<sup class="footnote-ref" id="fnref:B"><a rel="footnote" href="#fn:B">3</a></sup>
+</li>
+<li id="fn:B">B note, uses A to test duplicate.<sup class="footnote-ref" id="fnref:A"><a rel="footnote" href="#fn:A">1</a></sup>
+</li>
+</ol>
+</div>
 `,
 	}
 	doTestsInlineParam(t, tests, Options{Extensions: EXTENSION_FOOTNOTES}, 0,
diff --git a/markdown.go b/markdown.go
index 6d842d3..c82be6b 100644
--- a/markdown.go
+++ b/markdown.go
@@ -241,6 +241,16 @@ func (p *parser) getRef(refid string) (ref *reference, found bool) {
 	return ref, found
 }
 
+func (p *parser) isFootnote(ref *reference) bool {
+	for _, v := range p.notes {
+		if string(ref.link) == string(v.link) {
+			return true
+		}
+	}
+
+	return false
+}
+
 //
 //
 // Public interface

From 8c89af620080cccc7bd1d6d765e566ec2cdd4f19 Mon Sep 17 00:00:00 2001
From: choueric <zhssmail@gmail.com>
Date: Fri, 9 Jun 2017 10:20:58 +0800
Subject: [PATCH 34/42] add 'notesRecord' to check footnote existence fast

It is necessary to use vector for 'notes' instead of map to keep
footnotes ordered. But checking for presence in a vector is not
efficient. So add a map variable 'notesRecord' to tackle this problem.
---
 inline.go   |  2 ++
 markdown.go | 13 +++++--------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/inline.go b/inline.go
index eff1383..ca22af5 100644
--- a/inline.go
+++ b/inline.go
@@ -488,6 +488,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 			}
 
 			p.notes = append(p.notes, ref)
+			p.notesRecord[string(ref.link)] = true
 
 			link = ref.link
 			title = ref.title
@@ -501,6 +502,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 			if t == linkDeferredFootnote && !p.isFootnote(lr) {
 				lr.noteId = len(p.notes) + 1
 				p.notes = append(p.notes, lr)
+				p.notesRecord[string(lr.link)] = true
 			}
 
 			// keep link and title from reference
diff --git a/markdown.go b/markdown.go
index c82be6b..ee03fda 100644
--- a/markdown.go
+++ b/markdown.go
@@ -218,7 +218,8 @@ type parser struct {
 	// Footnotes need to be ordered as well as available to quickly check for
 	// presence. If a ref is also a footnote, it's stored both in refs and here
 	// in notes. Slice is nil if footnotes not enabled.
-	notes []*reference
+	notes       []*reference
+	notesRecord map[string]bool
 }
 
 func (p *parser) getRef(refid string) (ref *reference, found bool) {
@@ -242,13 +243,8 @@ func (p *parser) getRef(refid string) (ref *reference, found bool) {
 }
 
 func (p *parser) isFootnote(ref *reference) bool {
-	for _, v := range p.notes {
-		if string(ref.link) == string(v.link) {
-			return true
-		}
-	}
-
-	return false
+	_, ok := p.notesRecord[string(ref.link)]
+	return ok
 }
 
 //
@@ -386,6 +382,7 @@ func MarkdownOptions(input []byte, renderer Renderer, opts Options) []byte {
 
 	if extensions&EXTENSION_FOOTNOTES != 0 {
 		p.notes = make([]*reference, 0)
+		p.notesRecord = make(map[string]bool)
 	}
 
 	first := firstPass(p, input)

From 5b2fb1b893850a20a483145b676af75eaad51a5d Mon Sep 17 00:00:00 2001
From: choueric <zhssmail@gmail.com>
Date: Sat, 10 Jun 2017 22:24:27 +0800
Subject: [PATCH 35/42] use map[string]struct{} as a set.

Use map[string]struct{} instead of map[string]bool to implement a set
and reduce memory space.
---
 inline.go   | 4 ++--
 markdown.go | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/inline.go b/inline.go
index ca22af5..4483b8f 100644
--- a/inline.go
+++ b/inline.go
@@ -488,7 +488,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 			}
 
 			p.notes = append(p.notes, ref)
-			p.notesRecord[string(ref.link)] = true
+			p.notesRecord[string(ref.link)] = struct{}{}
 
 			link = ref.link
 			title = ref.title
@@ -502,7 +502,7 @@ func link(p *parser, out *bytes.Buffer, data []byte, offset int) int {
 			if t == linkDeferredFootnote && !p.isFootnote(lr) {
 				lr.noteId = len(p.notes) + 1
 				p.notes = append(p.notes, lr)
-				p.notesRecord[string(lr.link)] = true
+				p.notesRecord[string(lr.link)] = struct{}{}
 			}
 
 			// keep link and title from reference
diff --git a/markdown.go b/markdown.go
index ee03fda..1722a73 100644
--- a/markdown.go
+++ b/markdown.go
@@ -219,7 +219,7 @@ type parser struct {
 	// presence. If a ref is also a footnote, it's stored both in refs and here
 	// in notes. Slice is nil if footnotes not enabled.
 	notes       []*reference
-	notesRecord map[string]bool
+	notesRecord map[string]struct{}
 }
 
 func (p *parser) getRef(refid string) (ref *reference, found bool) {
@@ -382,7 +382,7 @@ func MarkdownOptions(input []byte, renderer Renderer, opts Options) []byte {
 
 	if extensions&EXTENSION_FOOTNOTES != 0 {
 		p.notes = make([]*reference, 0)
-		p.notesRecord = make(map[string]bool)
+		p.notesRecord = make(map[string]struct{})
 	}
 
 	first := firstPass(p, input)

From 4ca8c28b21a883c59eb518036a3fe45a3f281463 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?=
 <bjorn.erik.pedersen@gmail.com>
Date: Sat, 22 Jul 2017 11:31:49 +0200
Subject: [PATCH 36/42] Add Smartypants support for French Guillemets
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commits adds flag `HTML_SMARTYPANTS_QUOTES_NBSP` which, when combined with `HTML_USE_SMARTYPANTS` will insert non-breaking spaces between the double quotes and the contained text.

This is mostly relevant for use in French  with `HTML_SMARTYPANTS_ANGLED_QUOTES`.

It should not hurt existing code path in the performance department:

```
name                     old time/op    new time/op    delta
SmartDoubleQuotes-4    2.58µs ± 1%    2.58µs ± 1%   ~             (p=1.000 n=5+5)

name                     old alloc/op   new alloc/op   delta
SmartDoubleQuotes-4    5.27kB ± 0%    5.27kB ± 0%   ~     (all samples are equal)

name                     old allocs/op  new allocs/op  delta
SmartDoubleQuotes-4      13.0 ± 0%      13.0 ± 0%   ~     (all samples are equal)
```

Fixes #378
---
 html.go        |  1 +
 inline_test.go | 30 ++++++++++++++++++++++++++
 smartypants.go | 58 ++++++++++++++++++++++++++++++++++++++------------
 3 files changed, 75 insertions(+), 14 deletions(-)

diff --git a/html.go b/html.go
index 74e67ee..2f0ad3b 100644
--- a/html.go
+++ b/html.go
@@ -42,6 +42,7 @@ const (
 	HTML_SMARTYPANTS_DASHES                    // enable smart dashes (with HTML_USE_SMARTYPANTS)
 	HTML_SMARTYPANTS_LATEX_DASHES              // enable LaTeX-style dashes (with HTML_USE_SMARTYPANTS and HTML_SMARTYPANTS_DASHES)
 	HTML_SMARTYPANTS_ANGLED_QUOTES             // enable angled double quotes (with HTML_USE_SMARTYPANTS) for double quotes rendering
+	HTML_SMARTYPANTS_QUOTES_NBSP               // enable "French guillemets" (with HTML_USE_SMARTYPANTS)
 	HTML_FOOTNOTE_RETURN_LINKS                 // generate a link at the end of a footnote to return to the source
 )
 
diff --git a/inline_test.go b/inline_test.go
index 4061cfb..5bae3e9 100644
--- a/inline_test.go
+++ b/inline_test.go
@@ -1173,6 +1173,18 @@ func TestSmartDoubleQuotes(t *testing.T) {
 	doTestsInlineParam(t, tests, Options{}, HTML_USE_SMARTYPANTS, HtmlRendererParameters{})
 }
 
+func TestSmartDoubleQuotesNbsp(t *testing.T) {
+	var tests = []string{
+		"this should be normal \"quoted\" text.\n",
+		"<p>this should be normal &ldquo;&nbsp;quoted&nbsp;&rdquo; text.</p>\n",
+		"this \" single double\n",
+		"<p>this &ldquo;&nbsp; single double</p>\n",
+		"two pair of \"some\" quoted \"text\".\n",
+		"<p>two pair of &ldquo;&nbsp;some&nbsp;&rdquo; quoted &ldquo;&nbsp;text&nbsp;&rdquo;.</p>\n"}
+
+	doTestsInlineParam(t, tests, Options{}, HTML_USE_SMARTYPANTS|HTML_SMARTYPANTS_QUOTES_NBSP, HtmlRendererParameters{})
+}
+
 func TestSmartAngledDoubleQuotes(t *testing.T) {
 	var tests = []string{
 		"this should be angled \"quoted\" text.\n",
@@ -1185,6 +1197,18 @@ func TestSmartAngledDoubleQuotes(t *testing.T) {
 	doTestsInlineParam(t, tests, Options{}, HTML_USE_SMARTYPANTS|HTML_SMARTYPANTS_ANGLED_QUOTES, HtmlRendererParameters{})
 }
 
+func TestSmartAngledDoubleQuotesNbsp(t *testing.T) {
+	var tests = []string{
+		"this should be angled \"quoted\" text.\n",
+		"<p>this should be angled &laquo;&nbsp;quoted&nbsp;&raquo; text.</p>\n",
+		"this \" single double\n",
+		"<p>this &laquo;&nbsp; single double</p>\n",
+		"two pair of \"some\" quoted \"text\".\n",
+		"<p>two pair of &laquo;&nbsp;some&nbsp;&raquo; quoted &laquo;&nbsp;text&nbsp;&raquo;.</p>\n"}
+
+	doTestsInlineParam(t, tests, Options{}, HTML_USE_SMARTYPANTS|HTML_SMARTYPANTS_ANGLED_QUOTES|HTML_SMARTYPANTS_QUOTES_NBSP, HtmlRendererParameters{})
+}
+
 func TestSmartFractions(t *testing.T) {
 	var tests = []string{
 		"1/2, 1/4 and 3/4; 1/4th and 3/4ths\n",
@@ -1240,3 +1264,9 @@ func TestDisableSmartDashes(t *testing.T) {
 		HTML_USE_SMARTYPANTS|HTML_SMARTYPANTS_LATEX_DASHES,
 		HtmlRendererParameters{})
 }
+
+func BenchmarkSmartDoubleQuotes(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		runMarkdownInline("this should be normal \"quoted\" text.\n", Options{}, HTML_USE_SMARTYPANTS, HtmlRendererParameters{})
+	}
+}
diff --git a/smartypants.go b/smartypants.go
index eeffa5e..f25bd07 100644
--- a/smartypants.go
+++ b/smartypants.go
@@ -39,7 +39,7 @@ func isdigit(c byte) bool {
 	return c >= '0' && c <= '9'
 }
 
-func smartQuoteHelper(out *bytes.Buffer, previousChar byte, nextChar byte, quote byte, isOpen *bool) bool {
+func smartQuoteHelper(out *bytes.Buffer, previousChar byte, nextChar byte, quote byte, isOpen *bool, addNBSP bool) bool {
 	// edge of the buffer is likely to be a tag that we don't get to see,
 	// so we treat it like text sometimes
 
@@ -96,6 +96,12 @@ func smartQuoteHelper(out *bytes.Buffer, previousChar byte, nextChar byte, quote
 		*isOpen = false
 	}
 
+	// Note that with the limited lookahead, this non-breaking
+	// space will also be appended to single double quotes.
+	if addNBSP && !*isOpen {
+		out.WriteString("&nbsp;")
+	}
+
 	out.WriteByte('&')
 	if *isOpen {
 		out.WriteByte('l')
@@ -104,6 +110,11 @@ func smartQuoteHelper(out *bytes.Buffer, previousChar byte, nextChar byte, quote
 	}
 	out.WriteByte(quote)
 	out.WriteString("quo;")
+
+	if addNBSP && *isOpen {
+		out.WriteString("&nbsp;")
+	}
+
 	return true
 }
 
@@ -116,7 +127,7 @@ func smartSingleQuote(out *bytes.Buffer, smrt *smartypantsData, previousChar byt
 			if len(text) >= 3 {
 				nextChar = text[2]
 			}
-			if smartQuoteHelper(out, previousChar, nextChar, 'd', &smrt.inDoubleQuote) {
+			if smartQuoteHelper(out, previousChar, nextChar, 'd', &smrt.inDoubleQuote, false) {
 				return 1
 			}
 		}
@@ -141,7 +152,7 @@ func smartSingleQuote(out *bytes.Buffer, smrt *smartypantsData, previousChar byt
 	if len(text) > 1 {
 		nextChar = text[1]
 	}
-	if smartQuoteHelper(out, previousChar, nextChar, 's', &smrt.inSingleQuote) {
+	if smartQuoteHelper(out, previousChar, nextChar, 's', &smrt.inSingleQuote, false) {
 		return 0
 	}
 
@@ -205,13 +216,13 @@ func smartDashLatex(out *bytes.Buffer, smrt *smartypantsData, previousChar byte,
 	return 0
 }
 
-func smartAmpVariant(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte, quote byte) int {
+func smartAmpVariant(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte, quote byte, addNBSP bool) int {
 	if bytes.HasPrefix(text, []byte("&quot;")) {
 		nextChar := byte(0)
 		if len(text) >= 7 {
 			nextChar = text[6]
 		}
-		if smartQuoteHelper(out, previousChar, nextChar, quote, &smrt.inDoubleQuote) {
+		if smartQuoteHelper(out, previousChar, nextChar, quote, &smrt.inDoubleQuote, addNBSP) {
 			return 5
 		}
 	}
@@ -224,12 +235,15 @@ func smartAmpVariant(out *bytes.Buffer, smrt *smartypantsData, previousChar byte
 	return 0
 }
 
-func smartAmp(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int {
-	return smartAmpVariant(out, smrt, previousChar, text, 'd')
-}
+func smartAmp(angledQuotes, addNBSP bool) func(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int {
+	var quote byte = 'd'
+	if angledQuotes {
+		quote = 'a'
+	}
 
-func smartAmpAngledQuote(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int {
-	return smartAmpVariant(out, smrt, previousChar, text, 'a')
+	return func(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int {
+		return smartAmpVariant(out, smrt, previousChar, text, quote, addNBSP)
+	}
 }
 
 func smartPeriod(out *bytes.Buffer, smrt *smartypantsData, previousChar byte, text []byte) int {
@@ -253,7 +267,7 @@ func smartBacktick(out *bytes.Buffer, smrt *smartypantsData, previousChar byte,
 		if len(text) >= 3 {
 			nextChar = text[2]
 		}
-		if smartQuoteHelper(out, previousChar, nextChar, 'd', &smrt.inDoubleQuote) {
+		if smartQuoteHelper(out, previousChar, nextChar, 'd', &smrt.inDoubleQuote, false) {
 			return 1
 		}
 	}
@@ -337,7 +351,7 @@ func smartDoubleQuoteVariant(out *bytes.Buffer, smrt *smartypantsData, previousC
 	if len(text) > 1 {
 		nextChar = text[1]
 	}
-	if !smartQuoteHelper(out, previousChar, nextChar, quote, &smrt.inDoubleQuote) {
+	if !smartQuoteHelper(out, previousChar, nextChar, quote, &smrt.inDoubleQuote, false) {
 		out.WriteString("&quot;")
 	}
 
@@ -367,14 +381,30 @@ type smartCallback func(out *bytes.Buffer, smrt *smartypantsData, previousChar b
 
 type smartypantsRenderer [256]smartCallback
 
+var (
+	smartAmpAngled      = smartAmp(true, false)
+	smartAmpAngledNBSP  = smartAmp(true, true)
+	smartAmpRegular     = smartAmp(false, false)
+	smartAmpRegularNBSP = smartAmp(false, true)
+)
+
 func smartypants(flags int) *smartypantsRenderer {
 	r := new(smartypantsRenderer)
+	addNBSP := flags&HTML_SMARTYPANTS_QUOTES_NBSP != 0
 	if flags&HTML_SMARTYPANTS_ANGLED_QUOTES == 0 {
 		r['"'] = smartDoubleQuote
-		r['&'] = smartAmp
+		if !addNBSP {
+			r['&'] = smartAmpRegular
+		} else {
+			r['&'] = smartAmpRegularNBSP
+		}
 	} else {
 		r['"'] = smartAngledDoubleQuote
-		r['&'] = smartAmpAngledQuote
+		if !addNBSP {
+			r['&'] = smartAmpAngled
+		} else {
+			r['&'] = smartAmpAngledNBSP
+		}
 	}
 	r['\''] = smartSingleQuote
 	r['('] = smartParens

From 74be510331a2f1a189bb441a61e86cd6c9868e97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Sat, 6 May 2017 19:52:33 +0300
Subject: [PATCH 37/42] Document V2 in master README

---
 README.md | 81 +++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 52 insertions(+), 29 deletions(-)

diff --git a/README.md b/README.md
index 0d1ac9a..9913b3b 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,6 @@
-Blackfriday [![Build Status](https://travis-ci.org/russross/blackfriday.svg?branch=master)](https://travis-ci.org/russross/blackfriday) [![GoDoc](https://godoc.org/github.com/russross/blackfriday?status.svg)](https://godoc.org/github.com/russross/blackfriday)
+Blackfriday [![Build Status][BuildSVG]][BuildURL]
+[![Godoc][GodocV1SVG]][GodocV1URL] (V1)
+[![Godoc][GodocV2SVG]][GodocV2URL] (V2)
 ===========
 
 Blackfriday is a [Markdown][1] processor implemented in [Go][2]. It
@@ -8,7 +10,7 @@ punctuation substitutions, etc.), and it is safe for all utf-8
 (unicode) input.
 
 HTML output is currently supported, along with Smartypants
-extensions. An experimental LaTeX output engine is also included.
+extensions.
 
 It started as a translation from C of [Sundown][3].
 
@@ -16,23 +18,44 @@ It started as a translation from C of [Sundown][3].
 Installation
 ------------
 
-Blackfriday is compatible with Go 1. If you are using an older
-release of Go, consider using v1.1 of blackfriday, which was based
-on the last stable release of Go prior to Go 1. You can find it as a
-tagged commit on github.
+Blackfriday is compatible with any modern Go release. With Go 1.7 and git
+installed:
 
-With Go 1 and git installed:
-
-    go get github.com/russross/blackfriday
+    go get gopkg.in/russross/blackfriday.v2
 
 will download, compile, and install the package into your `$GOPATH`
 directory hierarchy. Alternatively, you can achieve the same if you
 import it into a project:
 
-    import "github.com/russross/blackfriday"
+    import "gopkg.in/russross/blackfriday.v2"
 
 and `go get` without parameters.
 
+
+Versions
+--------
+
+Currently maintained and recommended version of Blackfriday is `v2`. It's being
+developed on its own branch: https://github.com/russross/blackfriday/v2. You
+should install and import it via [gopkg.in][6] at
+`gopkg.in/russross/blackfriday.v2`.
+
+Version 2 offers a number of improvements over v1:
+
+* Cleaned up API
+* A separate call to [`Parse`][4], which produces an abstract syntax tree for
+  the document
+* Latest bug fixes
+* Flexibility to easily add your own rendering extensions
+
+Potential drawbacks:
+
+* Our benchmarks show v2 to be slightly slower than v1. Currently in the
+  ballpark of around 15%.
+* API breakage. If you can't afford modifying your code to adhere to the new API
+  and don't care too much about the new features, v2 is probably not for you.
+
+
 Usage
 -----
 
@@ -49,11 +72,10 @@ feature set, use this instead:
 ### Sanitize untrusted content
 
 Blackfriday itself does nothing to protect against malicious content. If you are
-dealing with user-supplied markdown, we recommend running blackfriday's output
-through HTML sanitizer such as
-[Bluemonday](https://github.com/microcosm-cc/bluemonday).
+dealing with user-supplied markdown, we recommend running Blackfriday's output
+through HTML sanitizer such as [Bluemonday][5].
 
-Here's an example of simple usage of blackfriday together with bluemonday:
+Here's an example of simple usage of Blackfriday together with Bluemonday:
 
 ``` go
 import (
@@ -69,7 +91,7 @@ html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
 ### Custom options
 
 If you want to customize the set of options, first get a renderer
-(currently either the HTML or LaTeX output engines), then use it to
+(currently only the HTML output engine), then use it to
 call the more general `Markdown` function. For examples, see the
 implementations of `MarkdownBasic` and `MarkdownCommon` in
 `markdown.go`.
@@ -249,7 +271,7 @@ are a few of note:
 
 *   [github_flavored_markdown](https://godoc.org/github.com/shurcooL/github_flavored_markdown):
     provides a GitHub Flavored Markdown renderer with fenced code block
-    highlighting, clickable header anchor links.
+    highlighting, clickable heading anchor links.
 
     It's not customizable, and its goal is to produce HTML output
     equivalent to the [GitHub Markdown API endpoint](https://developer.github.com/v3/markdown/#render-a-markdown-document-in-raw-mode),
@@ -258,17 +280,8 @@ are a few of note:
 *   [markdownfmt](https://github.com/shurcooL/markdownfmt): like gofmt,
     but for markdown.
 
-*   LaTeX output: renders output as LaTeX. This is currently part of the
-    main Blackfriday repository, but may be split into its own project
-    in the future. If you are interested in owning and maintaining the
-    LaTeX output component, please be in touch.
-
-    It renders some basic documents, but is only experimental at this
-    point. In particular, it does not do any inline escaping, so input
-    that happens to look like LaTeX code will be passed through without
-    modification.
-    
-*   [Md2Vim](https://github.com/FooSoft/md2vim): transforms markdown files into vimdoc format.
+*   [LaTeX output](https://bitbucket.org/ambrevar/blackfriday-latex):
+    renders output as LaTeX.
 
 
 Todo
@@ -287,6 +300,16 @@ License
 [Blackfriday is distributed under the Simplified BSD License](LICENSE.txt)
 
 
-   [1]: http://daringfireball.net/projects/markdown/ "Markdown"
-   [2]: http://golang.org/ "Go Language"
+   [1]: https://daringfireball.net/projects/markdown/ "Markdown"
+   [2]: https://golang.org/ "Go Language"
    [3]: https://github.com/vmg/sundown "Sundown"
+   [4]: https://godoc.org/gopkg.in/russross/blackfriday.v2#Parse "Parse func"
+   [5]: https://github.com/microcosm-cc/bluemonday "Bluemonday"
+   [6]: https://labix.org/gopkg.in "gopkg.in"
+
+   [BuildSVG]: https://travis-ci.org/russross/blackfriday.svg?branch=master
+   [BuildURL]: https://travis-ci.org/russross/blackfriday
+   [GodocV1SVG]: https://godoc.org/github.com/russross/blackfriday?status.svg
+   [GodocV1URL]: https://godoc.org/github.com/russross/blackfriday
+   [GodocV2SVG]: https://godoc.org/gopkg.in/russross/blackfriday.v2?status.svg
+   [GodocV2URL]: https://godoc.org/gopkg.in/russross/blackfriday.v2

From ad4c953876ab2a00a455904e9f17c0144a93771e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Sat, 6 May 2017 19:59:25 +0300
Subject: [PATCH 38/42] Put row of badges above the top level heading

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9913b3b..d9afcdf 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
-Blackfriday [![Build Status][BuildSVG]][BuildURL]
+[![Build Status][BuildSVG]][BuildURL]
 [![Godoc][GodocV1SVG]][GodocV1URL] (V1)
 [![Godoc][GodocV2SVG]][GodocV2URL] (V2)
+
+Blackfriday
 ===========
 
 Blackfriday is a [Markdown][1] processor implemented in [Go][2]. It

From 52dd06192f86c8944e3e6b0b075580bd4a40b74e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Sat, 6 May 2017 20:18:15 +0300
Subject: [PATCH 39/42] Badges back to heading, reference v1 docs explicitly

---
 README.md | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index d9afcdf..075466e 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,6 @@
-[![Build Status][BuildSVG]][BuildURL]
-[![Godoc][GodocV1SVG]][GodocV1URL] (V1)
-[![Godoc][GodocV2SVG]][GodocV2URL] (V2)
-
 Blackfriday
+[![Build Status][BuildSVG]][BuildURL]
+[![Godoc][GodocV2SVG]][GodocV2URL]
 ===========
 
 Blackfriday is a [Markdown][1] processor implemented in [Go][2]. It
@@ -57,6 +55,9 @@ Potential drawbacks:
 * API breakage. If you can't afford modifying your code to adhere to the new API
   and don't care too much about the new features, v2 is probably not for you.
 
+Documentation for the legacy v1 can be found here:
+https://godoc.org/github.com/russross/blackfriday
+
 
 Usage
 -----
@@ -311,7 +312,5 @@ License
 
    [BuildSVG]: https://travis-ci.org/russross/blackfriday.svg?branch=master
    [BuildURL]: https://travis-ci.org/russross/blackfriday
-   [GodocV1SVG]: https://godoc.org/github.com/russross/blackfriday?status.svg
-   [GodocV1URL]: https://godoc.org/github.com/russross/blackfriday
    [GodocV2SVG]: https://godoc.org/gopkg.in/russross/blackfriday.v2?status.svg
    [GodocV2URL]: https://godoc.org/gopkg.in/russross/blackfriday.v2

From a03b7ee643bdbb6346f21949a06a276bc9dd7e09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Sat, 6 May 2017 20:35:48 +0300
Subject: [PATCH 40/42] Add v1 import path, fix Bluemonday code snippet

---
 README.md | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 075466e..f947a61 100644
--- a/README.md
+++ b/README.md
@@ -21,15 +21,10 @@ Installation
 Blackfriday is compatible with any modern Go release. With Go 1.7 and git
 installed:
 
-    go get gopkg.in/russross/blackfriday.v2
+    go get -u gopkg.in/russross/blackfriday.v2
 
 will download, compile, and install the package into your `$GOPATH`
-directory hierarchy. Alternatively, you can achieve the same if you
-import it into a project:
-
-    import "gopkg.in/russross/blackfriday.v2"
-
-and `go get` without parameters.
+directory hierarchy.
 
 
 Versions
@@ -55,8 +50,9 @@ Potential drawbacks:
 * API breakage. If you can't afford modifying your code to adhere to the new API
   and don't care too much about the new features, v2 is probably not for you.
 
-Documentation for the legacy v1 can be found here:
-https://godoc.org/github.com/russross/blackfriday
+If you are still interested in the legacy `v1`, you can import it from
+`github.com/russross/blackfriday`. Documentation for the legacy v1 can be found
+here: https://godoc.org/github.com/russross/blackfriday
 
 
 Usage
@@ -83,11 +79,11 @@ Here's an example of simple usage of Blackfriday together with Bluemonday:
 ``` go
 import (
     "github.com/microcosm-cc/bluemonday"
-    "github.com/russross/blackfriday"
+    "gopkg.in/russross/blackfriday.v2"
 )
 
 // ...
-unsafe := blackfriday.MarkdownCommon(input)
+unsafe := blackfriday.Markdown(input)
 html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
 ```
 

From 8ad7e40db09b1885469035bbf0345f11e7173883 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Fri, 11 Aug 2017 22:47:52 +0300
Subject: [PATCH 41/42] Update README with the latest changes from v2

---
 README.md | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index f947a61..b73d35b 100644
--- a/README.md
+++ b/README.md
@@ -49,6 +49,9 @@ Potential drawbacks:
   ballpark of around 15%.
 * API breakage. If you can't afford modifying your code to adhere to the new API
   and don't care too much about the new features, v2 is probably not for you.
+* Several bug fixes are trailing behind and still need to be forward-ported to
+  v2. See issue [#348](https://github.com/russross/blackfriday/issues/348) for
+  tracking.
 
 If you are still interested in the legacy `v1`, you can import it from
 `github.com/russross/blackfriday`. Documentation for the legacy v1 can be found
@@ -58,6 +61,8 @@ here: https://godoc.org/github.com/russross/blackfriday
 Usage
 -----
 
+### v1
+
 For basic usage, it is as simple as getting your input into a byte
 slice and calling:
 
@@ -68,6 +73,23 @@ feature set, use this instead:
 
     output := blackfriday.MarkdownCommon(input)
 
+### v2
+
+For the most sensible markdown processing, it is as simple as getting your input
+into a byte slice and calling:
+
+```go
+output := blackfriday.Run(input)
+```
+
+Your input will be parsed and the output rendered with a set of most popular
+extensions enabled. If you want the most basic feature set, corresponding with
+the bare Markdown specification, use:
+
+```go
+output := blackfriday.Run(input, blackfriday.WithNoExtensions())
+```
+
 ### Sanitize untrusted content
 
 Blackfriday itself does nothing to protect against malicious content. If you are
@@ -76,18 +98,18 @@ through HTML sanitizer such as [Bluemonday][5].
 
 Here's an example of simple usage of Blackfriday together with Bluemonday:
 
-``` go
+```go
 import (
     "github.com/microcosm-cc/bluemonday"
     "gopkg.in/russross/blackfriday.v2"
 )
 
 // ...
-unsafe := blackfriday.Markdown(input)
+unsafe := blackfriday.Run(input)
 html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
 ```
 
-### Custom options
+### Custom options, v1
 
 If you want to customize the set of options, first get a renderer
 (currently only the HTML output engine), then use it to
@@ -95,6 +117,13 @@ call the more general `Markdown` function. For examples, see the
 implementations of `MarkdownBasic` and `MarkdownCommon` in
 `markdown.go`.
 
+### Custom options, v2
+
+If you want to customize the set of options, use `blackfriday.WithExtensions`,
+`blackfriday.WithRenderer` and `blackfriday.WithRefOverride`.
+
+### `blackfriday-tool`
+
 You can also check out `blackfriday-tool` for a more complete example
 of how to use it. Download and install it using:
 

From 8249792ba80f9c15f6e21eb22b7d0acd884b930e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vytautas=20=C5=A0altenis?= <vytas@rtfb.lt>
Date: Sat, 7 Oct 2017 12:41:58 +0300
Subject: [PATCH 42/42] Update on import paths and dep transitive import FAQ

Update usage and import paths with and without package management.
Document the dep transitive dependency issue and how to get around it.
Couple more minor edits.
---
 README.md | 42 ++++++++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index b73d35b..ad9e5ee 100644
--- a/README.md
+++ b/README.md
@@ -18,22 +18,27 @@ It started as a translation from C of [Sundown][3].
 Installation
 ------------
 
-Blackfriday is compatible with any modern Go release. With Go 1.7 and git
-installed:
+Blackfriday is compatible with any modern Go release. With Go and git installed:
 
     go get -u gopkg.in/russross/blackfriday.v2
 
-will download, compile, and install the package into your `$GOPATH`
-directory hierarchy.
+will download, compile, and install the package into your `$GOPATH` directory
+hierarchy.
 
 
 Versions
 --------
 
 Currently maintained and recommended version of Blackfriday is `v2`. It's being
-developed on its own branch: https://github.com/russross/blackfriday/v2. You
-should install and import it via [gopkg.in][6] at
-`gopkg.in/russross/blackfriday.v2`.
+developed on its own branch: https://github.com/russross/blackfriday/v2 and the
+documentation is available at
+https://godoc.org/gopkg.in/russross/blackfriday.v2.
+
+It is `go get`-able via via [gopkg.in][6] at `gopkg.in/russross/blackfriday.v2`,
+but we highly recommend using package management tool like [dep][7] or
+[Glide][8] and make use of semantic versioning. With package management you
+should import `github.com/russross/blackfriday` and specify that you're using
+version 2.0.0.
 
 Version 2 offers a number of improvements over v1:
 
@@ -57,6 +62,21 @@ If you are still interested in the legacy `v1`, you can import it from
 `github.com/russross/blackfriday`. Documentation for the legacy v1 can be found
 here: https://godoc.org/github.com/russross/blackfriday
 
+### Known issue with `dep`
+
+There is a known problem with using Blackfriday v1 _transitively_ and `dep`.
+Currently `dep` prioritizes semver versions over anything else, and picks the
+latest one, plus it does not apply a `[[constraint]]` specifier to transitively
+pulled in packages. So if you're using something that uses Blackfriday v1, but
+that something does not use `dep` yet, you will get Blackfriday v2 pulled in and
+your first dependency will fail to build.
+
+There are couple of fixes for it, documented here:
+https://github.com/golang/dep/blob/master/docs/FAQ.md#how-do-i-constrain-a-transitive-dependencys-version
+
+Meanwhile, `dep` team is working on a more general solution to the constraints
+on transitive dependencies problem: https://github.com/golang/dep/issues/1124.
+
 
 Usage
 -----
@@ -312,14 +332,14 @@ are a few of note:
     renders output as LaTeX.
 
 
-Todo
+TODO
 ----
 
 *   More unit testing
-*   Improve unicode support. It does not understand all unicode
+*   Improve Unicode support. It does not understand all Unicode
     rules (about what constitutes a letter, a punctuation symbol,
     etc.), so it may fail to detect word boundaries correctly in
-    some instances. It is safe on all utf-8 input.
+    some instances. It is safe on all UTF-8 input.
 
 
 License
@@ -334,6 +354,8 @@ License
    [4]: https://godoc.org/gopkg.in/russross/blackfriday.v2#Parse "Parse func"
    [5]: https://github.com/microcosm-cc/bluemonday "Bluemonday"
    [6]: https://labix.org/gopkg.in "gopkg.in"
+   [7]: https://github.com/golang/dep/ "dep"
+   [8]: https://github.com/Masterminds/glide "Glide"
 
    [BuildSVG]: https://travis-ci.org/russross/blackfriday.svg?branch=master
    [BuildURL]: https://travis-ci.org/russross/blackfriday