status-go/protocol/messenger_linkpreview_test.go

590 lines
20 KiB
Go

package protocol
import (
"bytes"
"fmt"
"io/ioutil"
"math"
"net/http"
"net/url"
"regexp"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/status-im/status-go/eth-node/crypto"
"github.com/status-im/status-go/images"
"github.com/status-im/status-go/multiaccounts/settings"
"github.com/status-im/status-go/protocol/common"
"github.com/status-im/status-go/protocol/protobuf"
"github.com/status-im/status-go/protocol/requests"
)
func TestMessengerLinkPreviews(t *testing.T) {
suite.Run(t, new(MessengerLinkPreviewsTestSuite))
}
type MessengerLinkPreviewsTestSuite struct {
MessengerBaseTestSuite
}
// StubMatcher should either return an http.Response or nil in case the request
// doesn't match.
type StubMatcher func(req *http.Request) *http.Response
type StubTransport struct {
// fallbackToDefaultTransport when true will make the transport use
// http.DefaultTransport in case no matcher is found.
fallbackToDefaultTransport bool
// disabledStubs when true, will skip all matchers and use
// http.DefaultTransport.
//
// Useful while testing to toggle between the original and stubbed responses.
disabledStubs bool
// matchers are http.RoundTripper functions.
matchers []StubMatcher
}
// RoundTrip returns a stubbed response if any matcher returns a non-nil
// http.Response. If no matcher is found and fallbackToDefaultTransport is true,
// then it executes the HTTP request using the default http transport.
//
// If StubTransport#disabledStubs is true, the default http transport is used.
func (t *StubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if t.disabledStubs {
return http.DefaultTransport.RoundTrip(req)
}
for _, matcher := range t.matchers {
res := matcher(req)
if res != nil {
return res, nil
}
}
if t.fallbackToDefaultTransport {
return http.DefaultTransport.RoundTrip(req)
}
return nil, fmt.Errorf("no HTTP matcher found")
}
func (t *StubTransport) AddURLMatcherRoundTrip(urlRegexp string, roundTrip func(r *http.Request) *http.Response) {
matcher := func(req *http.Request) *http.Response {
rx, err := regexp.Compile(regexp.QuoteMeta(urlRegexp))
if err != nil {
return nil
}
if !rx.MatchString(req.URL.String()) {
return nil
}
return roundTrip(req)
}
t.matchers = append(t.matchers, matcher)
}
// Add a matcher based on a URL regexp. If a given request URL matches the
// regexp, then responseBody will be returned with a hardcoded 200 status code.
// If headers is non-nil, use it as the value of http.Response.Header.
func (t *StubTransport) AddURLMatcher(urlRegexp string, responseBody []byte, headers map[string]string) {
matcher := func(req *http.Request) *http.Response {
res := &http.Response{
StatusCode: http.StatusOK,
Body: ioutil.NopCloser(bytes.NewBuffer(responseBody)),
}
if headers != nil {
res.Header = http.Header{}
for k, v := range headers {
res.Header.Set(k, v)
}
}
return res
}
t.AddURLMatcherRoundTrip(urlRegexp, matcher)
}
// assertContainsLongString verifies if actual contains a slice of expected and
// correctly prints the cause of the failure. The default behavior of
// require.Contains with long strings is to not print the formatted message
// (varargs to require.Contains).
func (s *MessengerLinkPreviewsTestSuite) assertContainsLongString(expected string, actual string, maxLength int) {
var safeIdx float64
var actualShort string
var expectedShort string
if len(actual) > 0 {
safeIdx = math.Min(float64(maxLength), float64(len(actual)-1))
actualShort = actual[:int(safeIdx)]
}
if len(expected) > 0 {
safeIdx = math.Min(float64(maxLength), float64(len(expected)-1))
expectedShort = expected[:int(safeIdx)]
}
s.Require().Contains(
actual, expected,
"'%s' should contain '%s'",
actualShort,
expectedShort,
)
}
func (s *MessengerLinkPreviewsTestSuite) Test_GetLinks() {
examples := []struct {
args string
expected []string
}{
// Invalid URLs are not taken in consideration.
{args: "", expected: []string{}},
{args: " ", expected: []string{}},
{args: "https", expected: []string{}},
{args: "https://", expected: []string{}},
{args: "https://status", expected: []string{}},
{args: "https://status.", expected: []string{}},
// URLs must include the sheme.
{args: "status.com", expected: []string{}},
{args: "https://status.im", expected: []string{"https://status.im"}},
// Only the host should be lowercased.
{args: "HTTPS://STATUS.IM/path/to?Q=AbCdE", expected: []string{"https://status.im/path/to?Q=AbCdE"}},
// Remove trailing forward slash.
{args: "https://github.com/", expected: []string{"https://github.com"}},
{args: "https://www.youtube.com/watch?v=mzOyYtfXkb0/", expected: []string{"https://www.youtube.com/watch?v=mzOyYtfXkb0"}},
// Valid URL.
{args: "https://status.c", expected: []string{"https://status.c"}},
{args: "https://status.im/test", expected: []string{"https://status.im/test"}},
{args: "https://192.168.0.100:9999/xyz", expected: []string{"https://192.168.0.100:9999/xyz"}},
// There is a bug in the code that builds the AST from markdown text,
// because it removes the closing parenthesis, which means it won't be
// possible to unfurl this URL.
{args: "https://en.wikipedia.org/wiki/Status_message_(instant_messaging)", expected: []string{"https://en.wikipedia.org/wiki/Status_message_(instant_messaging"}},
// Multiple URLs.
{
args: "https://status.im/test https://www.youtube.com/watch?v=mzOyYtfXkb0",
expected: []string{"https://status.im/test", "https://www.youtube.com/watch?v=mzOyYtfXkb0"},
},
{
args: "status.im https://www.youtube.com/watch?v=mzOyYtfXkb0",
expected: []string{"https://www.youtube.com/watch?v=mzOyYtfXkb0"},
},
}
for _, ex := range examples {
links := GetURLs(ex.args)
s.Require().Equal(ex.expected, links, "Failed for args: '%s'", ex.args)
}
}
func (s *MessengerLinkPreviewsTestSuite) readAsset(filename string) []byte {
b, err := ioutil.ReadFile("../_assets/tests/" + filename)
s.Require().NoError(err)
return b
}
func (s *MessengerLinkPreviewsTestSuite) Test_UnfurlURLs_YouTube() {
u := "https://www.youtube.com/watch?v=lE4UXdJSJM4"
thumbnailURL := "https://i.ytimg.com/vi/lE4UXdJSJM4/maxresdefault.jpg"
expected := common.LinkPreview{
Type: protobuf.UnfurledLink_LINK,
URL: u,
Hostname: "www.youtube.com",
Title: "Interview with a GNU/Linux user - Partition 1",
Description: "GNU/Linux Operating SystemInterview with a GNU/Linux user with Richie Guix - aired on © The GNU Linux.Programmer humorLinux humorProgramming jokesProgramming...",
Thumbnail: common.LinkPreviewThumbnail{
Width: 1,
Height: 1,
DataURI: "data:image/webp;base64,UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAAQAaJaQAA3AA/vpMgAA",
},
}
transport := StubTransport{}
transport.AddURLMatcher(
u,
[]byte(fmt.Sprintf(`
<html>
<head>
<meta property="og:title" content="%s">
<meta property="og:description" content="%s">
<meta property="og:image" content="%s">
</head>
</html>
`, expected.Title, expected.Description, thumbnailURL)),
nil,
)
transport.AddURLMatcher(thumbnailURL, s.readAsset("1.jpg"), nil)
stubbedClient := http.Client{Transport: &transport}
response, err := s.m.UnfurlURLs(&stubbedClient, []string{u})
s.Require().NoError(err)
s.Require().Len(response.StatusLinkPreviews, 0)
s.Require().Len(response.LinkPreviews, 1)
preview := response.LinkPreviews[0]
s.Require().Equal(expected.Type, preview.Type)
s.Require().Equal(expected.URL, preview.URL)
s.Require().Equal(expected.Hostname, preview.Hostname)
s.Require().Equal(expected.Title, preview.Title)
s.Require().Equal(expected.Description, preview.Description)
s.Require().Equal(expected.Thumbnail.Width, preview.Thumbnail.Width)
s.Require().Equal(expected.Thumbnail.Height, preview.Thumbnail.Height)
s.Require().Equal(expected.Thumbnail.URL, preview.Thumbnail.URL)
s.assertContainsLongString(expected.Thumbnail.DataURI, preview.Thumbnail.DataURI, 100)
}
func (s *MessengerLinkPreviewsTestSuite) Test_UnfurlURLs_Reddit() {
u := "https://www.reddit.com/r/Bitcoin/comments/13j0tzr/the_best_bitcoin_explanation_of_all_times/?utm_source=share"
expected := common.LinkPreview{
Type: protobuf.UnfurledLink_LINK,
URL: u,
Hostname: "www.reddit.com",
Title: "The best bitcoin explanation of all times.",
Description: "",
Thumbnail: common.LinkPreviewThumbnail{},
}
transport := StubTransport{}
transport.AddURLMatcher(
"https://www.reddit.com/oembed",
[]byte(`
{
"provider_url": "https://www.reddit.com/",
"version": "1.0",
"title": "The best bitcoin explanation of all times.",
"provider_name": "reddit",
"type": "rich",
"author_name": "DTheDev"
}
`),
nil,
)
stubbedClient := http.Client{Transport: &transport}
response, err := s.m.UnfurlURLs(&stubbedClient, []string{u})
s.Require().NoError(err)
s.Require().Len(response.StatusLinkPreviews, 0)
s.Require().Len(response.LinkPreviews, 1)
preview := response.LinkPreviews[0]
s.Require().Equal(expected.Type, preview.Type)
s.Require().Equal(expected.URL, preview.URL)
s.Require().Equal(expected.Hostname, preview.Hostname)
s.Require().Equal(expected.Title, preview.Title)
s.Require().Equal(expected.Description, preview.Description)
s.Require().Equal(expected.Thumbnail, preview.Thumbnail)
}
func (s *MessengerLinkPreviewsTestSuite) Test_UnfurlURLs_Timeout() {
httpClient := http.Client{Timeout: time.Nanosecond}
response, err := s.m.UnfurlURLs(&httpClient, []string{"https://status.im"})
s.Require().NoError(err)
s.Require().Len(response.StatusLinkPreviews, 0)
s.Require().Empty(response.LinkPreviews)
}
func (s *MessengerLinkPreviewsTestSuite) Test_UnfurlURLs_CommonFailures() {
httpClient := http.Client{}
// Test URL that doesn't return any OpenGraph title.
transport := StubTransport{}
transport.AddURLMatcher(
"https://wikipedia.org",
[]byte("<html><head></head></html>"),
nil,
)
stubbedClient := http.Client{Transport: &transport}
response, err := s.m.UnfurlURLs(&stubbedClient, []string{"https://wikipedia.org"})
s.Require().NoError(err)
s.Require().Len(response.StatusLinkPreviews, 0)
s.Require().Empty(response.LinkPreviews)
// Test 404.
response, err = s.m.UnfurlURLs(&httpClient, []string{"https://github.com/status-im/i_do_not_exist"})
s.Require().NoError(err)
s.Require().Len(response.StatusLinkPreviews, 0)
s.Require().Empty(response.LinkPreviews)
// Test no response when trying to get OpenGraph metadata.
response, err = s.m.UnfurlURLs(&httpClient, []string{"https://wikipedia.o"})
s.Require().NoError(err)
s.Require().Len(response.StatusLinkPreviews, 0)
s.Require().Empty(response.LinkPreviews)
}
func (s *MessengerLinkPreviewsTestSuite) Test_isSupportedImageURL() {
examples := []struct {
url string
expected bool
}{
{url: "https://placehold.co/600x400@2x.png", expected: true},
{url: "https://placehold.co/600x400@2x.PNG", expected: true},
{url: "https://placehold.co/600x400@2x.jpg", expected: true},
{url: "https://placehold.co/600x400@2x.JPG", expected: true},
{url: "https://placehold.co/600x400@2x.jpeg", expected: true},
{url: "https://placehold.co/600x400@2x.Jpeg", expected: true},
{url: "https://placehold.co/600x400@2x.webp", expected: true},
{url: "https://placehold.co/600x400@2x.WebP", expected: true},
{url: "https://placehold.co/600x400@2x.PnGs", expected: false},
{url: "https://placehold.co/600x400@2x.tiff", expected: false},
}
for _, e := range examples {
parsedURL, err := url.Parse(e.url)
s.Require().NoError(err, e)
s.Require().Equal(e.expected, IsSupportedImageURL(parsedURL), e.url)
}
}
func (s *MessengerLinkPreviewsTestSuite) Test_UnfurlURLs_Image() {
u := "https://placehold.co/600x400@3x.png"
expected := common.LinkPreview{
Type: protobuf.UnfurledLink_IMAGE,
URL: u,
Hostname: "placehold.co",
Title: "",
Description: "",
Thumbnail: common.LinkPreviewThumbnail{
Width: 1293,
Height: 1900,
DataURI: "data:image/jpeg;base64,/9j/2wCEABALDA4MChAODQ4SERATGCgaGBYWGDEjJR0oOjM9PDkzODdASFxOQERXRTc4UG1RV19iZ",
},
}
transport := StubTransport{}
// Use a larger image to verify Thumbnail.DataURI is compressed.
transport.AddURLMatcher(u, s.readAsset("IMG_1205.HEIC.jpg"), nil)
stubbedClient := http.Client{Transport: &transport}
response, err := s.m.UnfurlURLs(&stubbedClient, []string{u})
s.Require().NoError(err)
s.Require().Len(response.StatusLinkPreviews, 0)
s.Require().Len(response.LinkPreviews, 1)
preview := response.LinkPreviews[0]
s.Require().Equal(expected.Type, preview.Type)
s.Require().Equal(expected.URL, preview.URL)
s.Require().Equal(expected.Hostname, preview.Hostname)
s.Require().Equal(expected.Title, preview.Title)
s.Require().Equal(expected.Description, preview.Description)
s.Require().Equal(expected.Thumbnail.Width, preview.Thumbnail.Width)
s.Require().Equal(expected.Thumbnail.Height, preview.Thumbnail.Height)
s.Require().Equal(expected.Thumbnail.URL, preview.Thumbnail.URL)
s.assertContainsLongString(expected.Thumbnail.DataURI, preview.Thumbnail.DataURI, 100)
}
func (s *MessengerLinkPreviewsTestSuite) Test_UnfurlURLs_StatusContactAdded() {
identity, err := crypto.GenerateKey()
s.Require().NoError(err)
c, err := BuildContactFromPublicKey(&identity.PublicKey)
s.Require().NoError(err)
s.Require().NotNil(c)
pubkey, err := c.PublicKey()
s.Require().NoError(err)
shortKey, err := s.m.SerializePublicKey(crypto.CompressPubkey(pubkey))
s.Require().NoError(err)
payload, err := images.GetPayloadFromURI("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADIAAAAyCAYAAAAeP4ixA" +
"AAAiklEQVR4nOzWwQmFQAwG4ffEXmzLIizDImzLarQBhSwSGH7mO+9hh0DI9AthCI0hNIbQGEJjCI0hNIbQxITM1YfHfl69X3m2bsu/8i5mI" +
"obQGEJjCI0hNIbQlG+tUW83UtfNFjMRQ2gMofm8tUa3U9c2i5mIITSGqEnMRAyhMYTGEBpDaO4AAAD//5POEGncqtj1AAAAAElFTkSuQmCC")
s.Require().NoError(err)
icon := images.IdentityImage{
Width: 50,
Height: 50,
Payload: payload,
}
c.Bio = "TestBio_1"
c.DisplayName = "TestDisplayName_2"
c.Images = map[string]images.IdentityImage{}
c.Images[images.SmallDimName] = icon
s.m.allContacts.Store(c.ID, c)
// Generate a shared URL
u, err := s.m.ShareUserURLWithData(c.ID)
s.Require().NoError(err)
// Update contact info locally after creating the shared URL
// This is required to test that URL-decoded data is not used in the preview.
c.Bio = "TestBio_2"
c.DisplayName = "TestDisplayName_2"
s.m.allContacts.Store(c.ID, c)
r, err := s.m.UnfurlURLs(nil, []string{u})
s.Require().NoError(err)
s.Require().Len(r.StatusLinkPreviews, 1)
s.Require().Len(r.LinkPreviews, 0)
preview := r.StatusLinkPreviews[0]
s.Require().Equal(u, preview.URL)
s.Require().Nil(preview.Community)
s.Require().Nil(preview.Channel)
s.Require().NotNil(preview.Contact)
s.Require().Equal(shortKey, preview.Contact.PublicKey)
s.Require().Equal(c.DisplayName, preview.Contact.DisplayName)
s.Require().Equal(c.Bio, preview.Contact.Description)
s.Require().Equal(icon.Width, preview.Contact.Icon.Width)
s.Require().Equal(icon.Height, preview.Contact.Icon.Height)
s.Require().Equal("", preview.Contact.Icon.URL)
expectedDataURI, err := images.GetPayloadDataURI(icon.Payload)
s.Require().NoError(err)
s.Require().Equal(expectedDataURI, preview.Contact.Icon.DataURI)
}
func (s *MessengerLinkPreviewsTestSuite) Test_UnfurlURLs_StatusCommunityJoined() {
description := &requests.CreateCommunity{
Membership: protobuf.CommunityPermissions_NO_MEMBERSHIP,
Name: "status",
Description: "status community description",
Color: "#123456",
Image: "../_assets/tests/status.png", // 256*256 px
ImageAx: 0,
ImageAy: 0,
ImageBx: 256,
ImageBy: 256,
Banner: images.CroppedImage{
ImagePath: "../_assets/tests/IMG_1205.HEIC.jpg", // 2282*3352 px
X: 0,
Y: 0,
Width: 160,
Height: 90,
},
}
response, err := s.m.CreateCommunity(description, false)
s.Require().NoError(err)
s.Require().NotNil(response)
community := response.Communities()[0]
communityImages := community.Images()
s.Require().Len(communityImages, 3)
// Get icon data
icon, ok := communityImages[images.SmallDimName]
s.Require().True(ok)
iconWidth, iconHeight, err := images.GetImageDimensions(icon.Payload)
s.Require().NoError(err)
iconDataURI, err := images.GetPayloadDataURI(icon.Payload)
s.Require().NoError(err)
// Get banner data
banner, ok := communityImages[images.BannerIdentityName]
s.Require().True(ok)
bannerWidth, bannerHeight, err := images.GetImageDimensions(banner.Payload)
s.Require().NoError(err)
bannerDataURI, err := images.GetPayloadDataURI(banner.Payload)
s.Require().NoError(err)
// Create shared URL
u, err := s.m.ShareCommunityURLWithData(community.ID())
s.Require().NoError(err)
// Unfurl community shared URL
r, err := s.m.UnfurlURLs(nil, []string{u})
s.Require().NoError(err)
s.Require().Len(r.StatusLinkPreviews, 1)
s.Require().Len(r.LinkPreviews, 0)
preview := r.StatusLinkPreviews[0]
s.Require().Equal(u, preview.URL)
s.Require().NotNil(preview.Community)
s.Require().Nil(preview.Channel)
s.Require().Nil(preview.Contact)
s.Require().Equal(community.IDString(), preview.Community.CommunityID)
s.Require().Equal(community.Name(), preview.Community.DisplayName)
s.Require().Equal(community.Identity().Description, preview.Community.Description)
s.Require().Equal(iconWidth, preview.Community.Icon.Width)
s.Require().Equal(iconHeight, preview.Community.Icon.Height)
s.Require().Equal(iconDataURI, preview.Community.Icon.DataURI)
s.Require().Equal(bannerWidth, preview.Community.Banner.Width)
s.Require().Equal(bannerHeight, preview.Community.Banner.Height)
s.Require().Equal(bannerDataURI, preview.Community.Banner.DataURI)
}
func (s *MessengerLinkPreviewsTestSuite) Test_UnfurlURLs_Settings() {
// Create website stub
ogLink := "https://github.com"
requestsCount := 0
transport := StubTransport{}
transport.AddURLMatcherRoundTrip(
ogLink,
func(req *http.Request) *http.Response {
requestsCount++
responseBody := []byte(`<html><head><meta property="og:title" content="TestTitle"></head></html>`)
return &http.Response{
StatusCode: http.StatusOK,
Body: ioutil.NopCloser(bytes.NewBuffer(responseBody)),
}
},
)
stubbedClient := http.Client{Transport: &transport}
// Add contact
identity, err := crypto.GenerateKey()
s.Require().NoError(err)
c, err := BuildContactFromPublicKey(&identity.PublicKey)
s.Require().NoError(err)
s.Require().NotNil(c)
c.Bio = "TestBio_1"
c.DisplayName = "TestDisplayName_2"
s.m.allContacts.Store(c.ID, c)
statusUserLink, err := s.m.ShareUserURLWithData(c.ID)
s.Require().NoError(err)
linksToUnfurl := []string{ogLink, statusUserLink}
// Test `AlwaysAsk`
// NOTE: on status-go side `AlwaysAsk` == `EnableAll`, "asking" should be processed by the app
requestsCount = 0
err = s.m.settings.SaveSettingField(settings.URLUnfurlingMode, settings.URLUnfurlingAlwaysAsk)
s.Require().NoError(err)
linkPreviews, err := s.m.UnfurlURLs(&stubbedClient, linksToUnfurl)
s.Require().NoError(err)
s.Require().Len(linkPreviews.LinkPreviews, 1)
s.Require().Len(linkPreviews.StatusLinkPreviews, 1)
s.Require().Equal(requestsCount, 1)
// Test `EnableAll`
requestsCount = 0
err = s.m.settings.SaveSettingField(settings.URLUnfurlingMode, settings.URLUnfurlingEnableAll)
s.Require().NoError(err)
linkPreviews, err = s.m.UnfurlURLs(&stubbedClient, linksToUnfurl)
s.Require().NoError(err)
s.Require().Len(linkPreviews.LinkPreviews, 1)
s.Require().Len(linkPreviews.StatusLinkPreviews, 1)
s.Require().Equal(requestsCount, 1)
// Test `DisableAll`
requestsCount = 0
err = s.m.settings.SaveSettingField(settings.URLUnfurlingMode, settings.URLUnfurlingDisableAll)
s.Require().NoError(err)
linkPreviews, err = s.m.UnfurlURLs(&stubbedClient, linksToUnfurl)
s.Require().NoError(err)
s.Require().Len(linkPreviews.LinkPreviews, 0)
s.Require().Len(linkPreviews.StatusLinkPreviews, 1) // Status links are always unfurled
s.Require().Equal(requestsCount, 0)
}