status-go/protocol/urls/urls.go

283 lines
6.9 KiB
Go
Raw Normal View History

2020-10-27 17:35:28 +00:00
package urls
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
"github.com/keighl/metabolize"
2020-10-27 17:35:28 +00:00
)
2021-01-19 15:53:27 +00:00
type YoutubeOembedData struct {
2020-10-27 17:35:28 +00:00
ProviderName string `json:"provider_name"`
Title string `json:"title"`
ThumbnailURL string `json:"thumbnail_url"`
}
2021-01-19 15:53:27 +00:00
type GiphyOembedData struct {
ProviderName string `json:"provider_name"`
2021-01-20 11:24:41 +00:00
Title string `json:"title"`
URL string `json:"url"`
2021-02-05 12:58:09 +00:00
Height int `json:"height"`
Width int `json:"width"`
2021-01-19 15:53:27 +00:00
}
type TenorOembedData struct {
ProviderName string `json:"provider_name"`
ThumbnailURL string `json:"thumbnail_url"`
AuthorName string `json:"author_name"`
2021-02-05 12:58:09 +00:00
Height int `json:"height"`
Width int `json:"width"`
}
2020-10-27 17:35:28 +00:00
type LinkPreviewData struct {
Site string `json:"site" meta:"og:site_name"`
Title string `json:"title" meta:"og:title"`
ThumbnailURL string `json:"thumbnailUrl" meta:"og:image"`
ContentType string `json:"contentType"`
2021-02-05 12:58:09 +00:00
Height int `json:"height"`
Width int `json:"width"`
2020-10-27 17:35:28 +00:00
}
type Site struct {
Title string `json:"title"`
Address string `json:"address"`
ImageSite bool `json:"imageSite"`
2020-10-27 17:35:28 +00:00
}
2021-01-19 15:53:27 +00:00
const YoutubeOembedLink = "https://www.youtube.com/oembed?format=json&url=%s"
const GiphyOembedLink = "https://giphy.com/services/oembed?url=%s"
const TenorOembedLink = "https://tenor.com/oembed?url=%s"
var httpClient = http.Client{
Timeout: 30 * time.Second,
}
2020-10-27 17:35:28 +00:00
func LinkPreviewWhitelist() []Site {
return []Site{
Site{
Title: "Status",
Address: "our.status.im",
ImageSite: false,
},
2020-10-27 17:35:28 +00:00
Site{
Title: "YouTube",
Address: "youtube.com",
ImageSite: false,
2020-10-27 17:35:28 +00:00
},
Site{
Title: "YouTube shortener",
Address: "youtu.be",
ImageSite: false,
},
// Site{
// Title: "Tenor GIFs",
// Address: "tenor.com",
// ImageSite: true,
// },
Site{
Title: "GIPHY GIFs shortener",
Address: "gph.is",
ImageSite: true,
},
Site{
Title: "GIPHY GIFs",
Address: "giphy.com",
ImageSite: true,
2020-10-27 17:35:28 +00:00
},
2021-02-03 11:27:04 +00:00
Site{
Title: "GIPHY GIFs subdomain",
Address: "media.giphy.com",
ImageSite: true,
},
Site{
Title: "GitHub",
Address: "github.com",
ImageSite: false,
},
2021-04-12 09:19:34 +00:00
// Medium unfurling is failing - https://github.com/status-im/status-go/issues/2192
//
// Site{
// Title: "Medium",
// Address: "medium.com",
// ImageSite: false,
// },
2020-10-27 17:35:28 +00:00
}
}
func GetURLContent(url string) (data []byte, err error) {
// nolint: gosec
response, err := httpClient.Get(url)
2020-10-27 17:35:28 +00:00
if err != nil {
2021-01-28 09:27:35 +00:00
return data, fmt.Errorf("can't get content from link %s", url)
2020-10-27 17:35:28 +00:00
}
defer response.Body.Close()
return ioutil.ReadAll(response.Body)
}
2021-01-19 15:53:27 +00:00
func GetYoutubeOembed(url string) (data YoutubeOembedData, err error) {
oembedLink := fmt.Sprintf(YoutubeOembedLink, url)
2020-10-27 17:35:28 +00:00
jsonBytes, err := GetURLContent(oembedLink)
if err != nil {
2021-01-28 09:27:35 +00:00
return data, fmt.Errorf("can't get bytes from youtube oembed response on %s link", oembedLink)
2020-10-27 17:35:28 +00:00
}
err = json.Unmarshal(jsonBytes, &data)
if err != nil {
2021-01-28 09:27:35 +00:00
return data, fmt.Errorf("can't unmarshall json")
2020-10-27 17:35:28 +00:00
}
return data, nil
}
func GetYoutubePreviewData(link string) (previewData LinkPreviewData, err error) {
oembedData, err := GetYoutubeOembed(link)
if err != nil {
return previewData, err
}
previewData.Title = oembedData.Title
previewData.Site = oembedData.ProviderName
previewData.ThumbnailURL = oembedData.ThumbnailURL
return previewData, nil
}
func GetGenericLinkPreviewData(link string) (previewData LinkPreviewData, err error) {
// nolint: gosec
res, err := httpClient.Get(link)
if err != nil {
2021-01-28 09:27:35 +00:00
return previewData, fmt.Errorf("can't get content from link %s", link)
}
err = metabolize.Metabolize(res.Body, &previewData)
if err != nil {
2021-01-28 09:27:35 +00:00
return previewData, fmt.Errorf("can't get meta info from link %s", link)
}
return previewData, nil
}
2021-01-19 15:53:27 +00:00
func GetGiphyOembed(url string) (data GiphyOembedData, err error) {
oembedLink := fmt.Sprintf(GiphyOembedLink, url)
jsonBytes, err := GetURLContent(oembedLink)
if err != nil {
2021-01-28 09:27:35 +00:00
return data, fmt.Errorf("can't get bytes from Giphy oembed response at %s", oembedLink)
2021-01-19 15:53:27 +00:00
}
err = json.Unmarshal(jsonBytes, &data)
if err != nil {
2021-01-28 09:27:35 +00:00
return data, fmt.Errorf("can't unmarshall json")
2021-01-19 15:53:27 +00:00
}
return data, nil
}
func GetGiphyPreviewData(link string) (previewData LinkPreviewData, err error) {
oembedData, err := GetGiphyOembed(link)
if err != nil {
return previewData, err
}
previewData.Title = oembedData.Title
previewData.Site = oembedData.ProviderName
previewData.ThumbnailURL = oembedData.URL
previewData.Height = oembedData.Height
previewData.Width = oembedData.Width
2021-01-19 15:53:27 +00:00
return previewData, nil
}
// Giphy has a shortener service called gph.is, the oembed service doesn't work with shortened urls,
// so we need to fetch the long url first
func GetGiphyLongURL(shortURL string) (longURL string, err error) {
2021-02-03 13:25:42 +00:00
// nolint: gosec
res, err := http.Get(shortURL)
if err != nil {
return longURL, fmt.Errorf("can't get bytes from Giphy's short url at %s", shortURL)
}
canonicalURL := res.Request.URL.String()
2021-02-03 13:25:42 +00:00
if canonicalURL == shortURL {
// no redirect, ie. not a valid url
return longURL, fmt.Errorf("unable to process Giphy's short url at %s", shortURL)
}
2021-02-03 13:25:42 +00:00
return canonicalURL, err
}
2021-02-03 13:25:42 +00:00
func GetGiphyShortURLPreviewData(shortURL string) (data LinkPreviewData, err error) {
longURL, err := GetGiphyLongURL(shortURL)
if err != nil {
return data, err
}
2021-02-03 13:25:42 +00:00
return GetGiphyPreviewData(longURL)
}
func GetTenorOembed(url string) (data TenorOembedData, err error) {
oembedLink := fmt.Sprintf(TenorOembedLink, url)
jsonBytes, err := GetURLContent(oembedLink)
if err != nil {
2021-01-28 09:27:35 +00:00
return data, fmt.Errorf("can't get bytes from Tenor oembed response at %s", oembedLink)
}
err = json.Unmarshal(jsonBytes, &data)
if err != nil {
2021-01-28 09:27:35 +00:00
return data, fmt.Errorf("can't unmarshall json")
}
2020-10-27 17:35:28 +00:00
return data, nil
}
func GetTenorPreviewData(link string) (previewData LinkPreviewData, err error) {
oembedData, err := GetTenorOembed(link)
if err != nil {
return previewData, err
}
previewData.Title = oembedData.AuthorName // Tenor Oembed service doesn't return title of the Gif
previewData.Site = oembedData.ProviderName
previewData.ThumbnailURL = oembedData.ThumbnailURL
previewData.Height = oembedData.Height
previewData.Width = oembedData.Width
return previewData, nil
}
func GetLinkPreviewData(link string) (previewData LinkPreviewData, err error) {
2020-10-27 17:35:28 +00:00
url, err := url.Parse(link)
if err != nil {
2021-01-28 09:27:35 +00:00
return previewData, fmt.Errorf("cant't parse link %s", link)
2020-10-27 17:35:28 +00:00
}
hostname := strings.ToLower(url.Hostname())
2021-01-20 12:02:26 +00:00
switch hostname {
case "youtube.com", "youtu.be", "www.youtube.com":
return GetYoutubePreviewData(link)
2021-04-12 09:19:34 +00:00
case "github.com", "our.status.im":
return GetGenericLinkPreviewData(link)
case "giphy.com", "media.giphy.com":
return GetGiphyPreviewData(link)
case "gph.is":
return GetGiphyShortURLPreviewData(link)
2021-01-20 12:02:26 +00:00
case "tenor.com":
return GetTenorPreviewData(link)
2021-01-20 12:02:26 +00:00
default:
2021-01-28 09:27:35 +00:00
return previewData, fmt.Errorf("link %s isn't whitelisted. Hostname - %s", link, url.Hostname())
}
2020-10-27 17:35:28 +00:00
}