Add config option MediaDownloadPath (#443)

* Add config option MediaUploadPath

MediaDownloadPath can be used instead of MediaServerUpload, for when your
webserver is on the same system as matterbridge and matterbridge has
write access to the serve dir.

* Limit length of hash in MediaServer urls to 8chars

Full SHA256 is unnecessary for uniqueness.
Also; if a file has the same first 8 charachters of the SHA256 hash,
it's still not a problem, as long as the filename is not the same.
This commit is contained in:
Remi Reuvekamp 2018-06-08 22:30:35 +02:00 committed by Wim
parent 7e54474111
commit 33bd60528b
4 changed files with 87 additions and 38 deletions

View File

@ -2,14 +2,15 @@ package config
import ( import (
"bytes" "bytes"
"github.com/fsnotify/fsnotify"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
prefixed "github.com/x-cray/logrus-prefixed-formatter"
"os" "os"
"strings" "strings"
"sync" "sync"
"time" "time"
"github.com/fsnotify/fsnotify"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
prefixed "github.com/x-cray/logrus-prefixed-formatter"
) )
const ( const (
@ -76,6 +77,7 @@ type Protocol struct {
MediaDownloadSize int // all protocols MediaDownloadSize int // all protocols
MediaServerDownload string MediaServerDownload string
MediaServerUpload string MediaServerUpload string
MediaDownloadPath string // Basically MediaServerUpload, but instead of uploading it, just write it to a file on the same server.
MessageDelay int // IRC, time in millisecond to wait between messages MessageDelay int // IRC, time in millisecond to wait between messages
MessageFormat string // telegram MessageFormat string // telegram
MessageLength int // IRC, max length of a message allowed MessageLength int // IRC, max length of a message allowed

View File

@ -3,6 +3,10 @@ package gateway
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"io/ioutil"
"net/http"
"os"
"github.com/42wim/matterbridge/bridge" "github.com/42wim/matterbridge/bridge"
"github.com/42wim/matterbridge/bridge/api" "github.com/42wim/matterbridge/bridge/api"
"github.com/42wim/matterbridge/bridge/config" "github.com/42wim/matterbridge/bridge/config"
@ -18,16 +22,16 @@ import (
"github.com/42wim/matterbridge/bridge/telegram" "github.com/42wim/matterbridge/bridge/telegram"
"github.com/42wim/matterbridge/bridge/xmpp" "github.com/42wim/matterbridge/bridge/xmpp"
"github.com/42wim/matterbridge/bridge/zulip" "github.com/42wim/matterbridge/bridge/zulip"
"github.com/hashicorp/golang-lru"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
// "github.com/davecgh/go-spew/spew" // "github.com/davecgh/go-spew/spew"
"crypto/sha1" "crypto/sha1"
"github.com/hashicorp/golang-lru"
"github.com/peterhellberg/emojilib"
"net/http"
"path/filepath" "path/filepath"
"regexp" "regexp"
"strings" "strings"
"time" "time"
"github.com/peterhellberg/emojilib"
) )
type Gateway struct { type Gateway struct {
@ -411,46 +415,83 @@ func (gw *Gateway) modifyMessage(msg *config.Message) {
} }
} }
// handleFiles uploads or places all files on the given msg to the MediaServer and
// adds the new URL of the file on the MediaServer onto the given msg.
func (gw *Gateway) handleFiles(msg *config.Message) { func (gw *Gateway) handleFiles(msg *config.Message) {
reg := regexp.MustCompile("[^a-zA-Z0-9]+") reg := regexp.MustCompile("[^a-zA-Z0-9]+")
// if we don't have a attachfield or we don't have a mediaserver configured return
if msg.Extra == nil || gw.Config.General.MediaServerUpload == "" { // If we don't have a attachfield or we don't have a mediaserver configured return
if msg.Extra == nil || (gw.Config.General.MediaServerUpload == "" && gw.Config.General.MediaDownloadPath == "") {
return return
} }
// if we actually have files, start uploading them to the mediaserver // If we don't have files, nothing to upload.
if len(msg.Extra["file"]) > 0 { if len(msg.Extra["file"]) == 0 {
client := &http.Client{ return
Timeout: time.Second * 5, }
}
for i, f := range msg.Extra["file"] { client := &http.Client{
fi := f.(config.FileInfo) Timeout: time.Second * 5,
ext := filepath.Ext(fi.Name) }
fi.Name = fi.Name[0 : len(fi.Name)-len(ext)]
fi.Name = reg.ReplaceAllString(fi.Name, "_") for i, f := range msg.Extra["file"] {
fi.Name = fi.Name + ext fi := f.(config.FileInfo)
sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data)) ext := filepath.Ext(fi.Name)
reader := bytes.NewReader(*fi.Data) fi.Name = fi.Name[0 : len(fi.Name)-len(ext)]
fi.Name = reg.ReplaceAllString(fi.Name, "_")
fi.Name = fi.Name + ext
sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8]
if gw.Config.General.MediaServerUpload != "" {
// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
url := gw.Config.General.MediaServerUpload + "/" + sha1sum + "/" + fi.Name url := gw.Config.General.MediaServerUpload + "/" + sha1sum + "/" + fi.Name
durl := gw.Config.General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name
extra := msg.Extra["file"][i].(config.FileInfo) req, err := http.NewRequest("PUT", url, bytes.NewReader(*fi.Data))
extra.URL = durl
req, err := http.NewRequest("PUT", url, reader)
if err != nil { if err != nil {
flog.Errorf("mediaserver upload failed: %#v", err) flog.Errorf("mediaserver upload failed, could not create request: %#v", err)
continue continue
} }
flog.Debugf("mediaserver upload url: %s", url)
req.Header.Set("Content-Type", "binary/octet-stream") req.Header.Set("Content-Type", "binary/octet-stream")
_, err = client.Do(req) _, err = client.Do(req)
if err != nil { if err != nil {
flog.Errorf("mediaserver upload failed: %#v", err) flog.Errorf("mediaserver upload failed, could not Do request: %#v", err)
continue
}
} else {
// Use MediaServerPath. Place the file on the current filesystem.
dir := gw.Config.General.MediaDownloadPath + "/" + sha1sum
err := os.Mkdir(dir, os.ModePerm)
if err != nil && !os.IsExist(err) {
flog.Errorf("mediaserver path failed, could not mkdir: %s %#v", err, err)
continue
}
path := dir + "/" + fi.Name
flog.Debugf("mediaserver path placing file: %s", path)
err = ioutil.WriteFile(path, *fi.Data, os.ModePerm)
if err != nil {
flog.Errorf("mediaserver path failed, could not writefile: %s %#v", err, err)
continue continue
} }
flog.Debugf("mediaserver download URL = %s", durl)
// we uploaded the file successfully. Add the SHA
extra.SHA = sha1sum
msg.Extra["file"][i] = extra
} }
// Download URL.
durl := gw.Config.General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name
flog.Debugf("mediaserver download URL = %s", durl)
// We uploaded/placed the file successfully. Add the SHA and URL.
extra := msg.Extra["file"][i].(config.FileInfo)
extra.URL = durl
extra.SHA = sha1sum
msg.Extra["file"][i] = extra
} }
} }

View File

@ -3,13 +3,14 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"os"
"strings"
"github.com/42wim/matterbridge/bridge/config" "github.com/42wim/matterbridge/bridge/config"
"github.com/42wim/matterbridge/gateway" "github.com/42wim/matterbridge/gateway"
"github.com/google/gops/agent" "github.com/google/gops/agent"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
prefixed "github.com/x-cray/logrus-prefixed-formatter" prefixed "github.com/x-cray/logrus-prefixed-formatter"
"os"
"strings"
) )
var ( var (

View File

@ -1306,10 +1306,13 @@ RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
StripNick=false StripNick=false
#MediaServerUpload and MediaServerDownload are used for uploading images/files/video to #MediaServerUpload (or MediaDownloadPath) and MediaServerDownload are used for uploading
#a remote "mediaserver" (a webserver like caddy for example). #images/files/video to a remote "mediaserver" (a webserver like caddy for example).
#When configured images/files uploaded on bridges like mattermost,slack, telegram will be downloaded #When configured images/files uploaded on bridges like mattermost, slack, telegram will be
#and uploaded again to MediaServerUpload URL #downloaded and uploaded again to MediaServerUpload URL
#MediaDownloadPath is the filesystem path where the media file will be placed, instead of uploaded,
#for if Matterbridge has write access to the directory your webserver is serving.
#It is an alternative to MediaServerUpload.
#The MediaServerDownload will be used so that bridges without native uploading support: #The MediaServerDownload will be used so that bridges without native uploading support:
#gitter, irc and xmpp will be shown links to the files on MediaServerDownload #gitter, irc and xmpp will be shown links to the files on MediaServerDownload
# #
@ -1317,6 +1320,8 @@ StripNick=false
#OPTIONAL (default empty) #OPTIONAL (default empty)
MediaServerUpload="https://user:pass@yourserver.com/upload" MediaServerUpload="https://user:pass@yourserver.com/upload"
#OPTIONAL (default empty) #OPTIONAL (default empty)
MediaDownloadPath="/srv/http/yourserver.com/public/download"
#OPTIONAL (default empty)
MediaServerDownload="https://youserver.com/download" MediaServerDownload="https://youserver.com/download"
#MediaDownloadSize is the maximum size of attachments, videos, images #MediaDownloadSize is the maximum size of attachments, videos, images