status-im
/
status-go
mirror of https://github.com/status-im/status-go.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: add IPFS rate limiter for downloading stickers and use http server for retrieving stickers (#2611)

This commit is contained in:
Richard Ramos 2022-05-09 09:07:57 -04:00 committed by GitHub
parent 2485e84bf5
commit 0048aaebcc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 550 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION
View File

@ -1 +1 @@
0.98.5
0.98.6

4
api/geth_backend.go
View File

@ -690,11 +690,11 @@ func (b *GethStatusBackend) loadNodeConfig(inputNodeCfg *params.NodeConfig) erro
// Start WakuV1 if WakuV2 is not enabled
conf.WakuConfig.Enabled = !conf.WakuV2Config.Enabled
// NodeConfig.Version should be taken from params.Version
// which is set at the compile time.
// What's cached is usually outdated so we overwrite it here.
conf.Version = params.Version
conf.RootDataDir = b.rootDataDir
conf.DataDir = filepath.Join(b.rootDataDir, conf.DataDir)
conf.ShhextConfig.BackupDisabledDataDir = filepath.Join(b.rootDataDir, conf.ShhextConfig.BackupDisabledDataDir)
if len(conf.LogDir) == 0 {
@ -1193,7 +1193,7 @@ func (b *GethStatusBackend) injectAccountsIntoWakuService(w types.WakuKeyManager
}
if st != nil {
if err := st.InitProtocol(b.statusNode.GethNode().Config().Name, identity, b.appDB, b.multiaccountsDB, acc, logutils.ZapLogger()); err != nil {
if err := st.InitProtocol(b.statusNode.GethNode().Config().Name, identity, b.appDB, b.statusNode.HTTPServer(), b.multiaccountsDB, acc, logutils.ZapLogger()); err != nil {
return err
}
// Set initial connection state

6
appdatabase/migrations/bindata.go
View File

@ -360,7 +360,7 @@ func _1649164719_add_community_archives_info_tableUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "1649164719_add_community_archives_info_table.up.sql", size: 208, mode: os.FileMode(0664), modTime: time.Unix(1649593899, 0)}
info := bindataFileInfo{name: "1649164719_add_community_archives_info_table.up.sql", size: 208, mode: os.FileMode(0664), modTime: time.Unix(1652098406, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd1, 0x4f, 0x80, 0x45, 0xb9, 0xd9, 0x15, 0xe2, 0x78, 0xd0, 0xcb, 0x71, 0xc1, 0x1b, 0xb7, 0x1b, 0x1b, 0x97, 0xfe, 0x47, 0x53, 0x3c, 0x62, 0xbc, 0xdd, 0x3a, 0x94, 0x1a, 0xc, 0x48, 0x76, 0xe}}
return a, nil
}
@ -380,7 +380,7 @@ func _1649174829_add_visitble_tokenUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "1649174829_add_visitble_token.up.sql", size: 84, mode: os.FileMode(0664), modTime: time.Unix(1649882240, 0)}
info := bindataFileInfo{name: "1649174829_add_visitble_token.up.sql", size: 84, mode: os.FileMode(0664), modTime: time.Unix(1652098406, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xa3, 0x22, 0xc0, 0x2b, 0x3f, 0x4f, 0x3d, 0x5e, 0x4c, 0x68, 0x7c, 0xd0, 0x15, 0x36, 0x9f, 0xec, 0xa1, 0x2a, 0x7b, 0xb4, 0xe3, 0xc6, 0xc9, 0xb4, 0x81, 0x50, 0x4a, 0x11, 0x3b, 0x35, 0x7, 0xcf}}
return a, nil
}
@ -400,7 +400,7 @@ func _1649882262_add_derived_from_accountsUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "1649882262_add_derived_from_accounts.up.sql", size: 110, mode: os.FileMode(0664), modTime: time.Unix(1649882324, 0)}
info := bindataFileInfo{name: "1649882262_add_derived_from_accounts.up.sql", size: 110, mode: os.FileMode(0664), modTime: time.Unix(1652098406, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x11, 0xb9, 0x44, 0x4d, 0x85, 0x8d, 0x7f, 0xb4, 0xae, 0x4f, 0x5c, 0x66, 0x64, 0xb6, 0xe2, 0xe, 0x3d, 0xad, 0x9d, 0x8, 0x4f, 0xab, 0x6e, 0xa8, 0x7d, 0x76, 0x3, 0xad, 0x96, 0x1, 0xee, 0x5c}}
return a, nil
}

236
ipfs/ipfs.go Normal file
View File

@ -0,0 +1,236 @@
package ipfs
import (
"errors"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"sync"
"time"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multibase"
"github.com/wealdtech/go-multicodec"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/log"
)
const infuraAPIURL = "https://ipfs.infura.io:5001/api/v0/cat?arg="
const maxRequestsPerSecond = 3
type taskResponse struct {
err error
response []byte
}
type taskRequest struct {
cid string
download bool
doneChan chan taskResponse
}
type Downloader struct {
ipfsDir string
wg sync.WaitGroup
rateLimiterChan chan taskRequest
inputTaskChan chan taskRequest
client *http.Client
quit chan struct{}
}
func NewDownloader(rootDir string) *Downloader {
ipfsDir := filepath.Clean(filepath.Join(rootDir, "./ipfs"))
if err := os.MkdirAll(ipfsDir, 0700); err != nil {
panic("could not create IPFSDir")
}
d := &Downloader{
ipfsDir: ipfsDir,
rateLimiterChan: make(chan taskRequest, maxRequestsPerSecond),
inputTaskChan: make(chan taskRequest, 1000),
wg: sync.WaitGroup{},
client: &http.Client{
Timeout: time.Second * 5,
},
quit: make(chan struct{}, 1),
}
go d.taskDispatcher()
go d.worker()
return d
}
func (d *Downloader) Stop() {
close(d.quit)
d.wg.Wait()
close(d.inputTaskChan)
close(d.rateLimiterChan)
}
func (d *Downloader) worker() {
for {
select {
case <-d.quit:
return
case request := <-d.rateLimiterChan:
resp, err := d.download(request.cid, request.download)
request.doneChan <- taskResponse{
err: err,
response: resp,
}
}
}
}
func (d *Downloader) taskDispatcher() {
ticker := time.NewTicker(time.Second / maxRequestsPerSecond)
defer ticker.Stop()
for {
select {
case <-d.quit:
return
case <-ticker.C:
request, ok := <-d.inputTaskChan
if !ok {
return
}
d.rateLimiterChan <- request
}
}
}
func hashToCid(hash []byte) (string, error) {
// contract response includes a contenthash, which needs to be decoded to reveal
// an IPFS identifier. Once decoded, download the content from IPFS. This content
// is in EDN format, ie https://ipfs.infura.io/ipfs/QmWVVLwVKCwkVNjYJrRzQWREVvEk917PhbHYAUhA1gECTM
// and it also needs to be decoded in to a nim type
data, codec, err := multicodec.RemoveCodec(hash)
if err != nil {
return "", err
}
codecName, err := multicodec.Name(codec)
if err != nil {
return "", err
}
if codecName != "ipfs-ns" {
return "", errors.New("codecName is not ipfs-ns")
}
thisCID, err := cid.Parse(data)
if err != nil {
return "", err
}
return thisCID.StringOfBase(multibase.Base32)
}
func decodeStringHash(input string) (string, error) {
hash, err := hexutil.Decode("0x" + input)
if err != nil {
return "", err
}
cid, err := hashToCid(hash)
if err != nil {
return "", err
}
return cid, nil
}
// Get checks if an IPFS image exists and returns it from cache
// otherwise downloads it from INFURA's ipfs gateway
func (d *Downloader) Get(hash string, download bool) ([]byte, error) {
cid, err := decodeStringHash(hash)
if err != nil {
return nil, err
}
exists, content, err := d.exists(cid)
if err != nil {
return nil, err
}
if exists {
return content, nil
}
doneChan := make(chan taskResponse, 1)
d.wg.Add(1)
d.inputTaskChan <- taskRequest{
cid: cid,
download: download,
doneChan: doneChan,
}
done := <-doneChan
close(doneChan)
d.wg.Done()
return done.response, done.err
}
func (d *Downloader) exists(cid string) (bool, []byte, error) {
path := filepath.Join(d.ipfsDir, cid)
_, err := os.Stat(path)
if err == nil {
fileContent, err := os.ReadFile(path)
return true, fileContent, err
}
return false, nil, nil
}
func (d *Downloader) download(cid string, download bool) ([]byte, error) {
path := filepath.Join(d.ipfsDir, cid)
req, err := http.NewRequest(http.MethodPost, infuraAPIURL+cid, nil)
if err != nil {
return nil, err
}
resp, err := d.client.Do(req)
if err != nil {
return nil, err
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Error("failed to close the stickerpack request body", "err", err)
}
}()
if resp.StatusCode < 200 || resp.StatusCode > 299 {
log.Error("could not load data for", "cid", cid, "code", resp.StatusCode)
return nil, errors.New("could not load ipfs data")
}
fileContent, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if download {
// #nosec G306
err = os.WriteFile(path, fileContent, 0700)
if err != nil {
return nil, err
}
}
return fileContent, nil
}

34
node/get_status_node.go
View File

@ -25,10 +25,12 @@ import (
"github.com/status-im/status-go/connection"
"github.com/status-im/status-go/db"
"github.com/status-im/status-go/discovery"
"github.com/status-im/status-go/ipfs"
"github.com/status-im/status-go/multiaccounts"
"github.com/status-im/status-go/params"
"github.com/status-im/status-go/peers"
"github.com/status-im/status-go/rpc"
"github.com/status-im/status-go/server"
accountssvc "github.com/status-im/status-go/services/accounts"
appmetricsservice "github.com/status-im/status-go/services/appmetrics"
"github.com/status-im/status-go/services/browsers"
@ -78,6 +80,9 @@ type StatusNode struct {
gethNode *node.Node // reference to Geth P2P stack/node
rpcClient *rpc.Client // reference to an RPC client
downloader *ipfs.Downloader
httpServer *server.Server
discovery discovery.Discovery
register *peers.Register
peerPool *peers.PeerPool
@ -145,6 +150,13 @@ func (n *StatusNode) GethNode() *node.Node {
return n.gethNode
}
func (n *StatusNode) HTTPServer() *server.Server {
n.mu.RLock()
defer n.mu.RUnlock()
return n.httpServer
}
// Server retrieves the currently running P2P network layer.
func (n *StatusNode) Server() *p2p.Server {
n.mu.RLock()
@ -222,6 +234,19 @@ func (n *StatusNode) startWithDB(config *params.NodeConfig, accs *accounts.Manag
return err
}
n.downloader = ipfs.NewDownloader(config.RootDataDir)
httpServer, err := server.NewServer(n.appDB, n.downloader)
if err != nil {
return err
}
if err := httpServer.Start(); err != nil {
return err
}
n.httpServer = httpServer
if err := n.initServices(config); err != nil {
return err
}
@ -399,6 +424,15 @@ func (n *StatusNode) stop() error {
n.gethNode = nil
n.config = nil
err := n.httpServer.Stop()
if err != nil {
return err
}
n.httpServer = nil
n.downloader.Stop()
n.downloader = nil
if n.db != nil {
err := n.db.Close()

4
node/status_node_services.go
View File

@ -392,7 +392,7 @@ func (b *StatusNode) ensService() *ens.Service {
func (b *StatusNode) stickersService(accountDB *accounts.Database) *stickers.Service {
if b.stickersSrvc == nil {
b.stickersSrvc = stickers.NewService(accountDB, b.rpcClient, b.gethAccountManager, b.rpcFiltersSrvc, b.config)
b.stickersSrvc = stickers.NewService(accountDB, b.rpcClient, b.gethAccountManager, b.rpcFiltersSrvc, b.config, b.downloader, b.httpServer)
}
return b.stickersSrvc
}
@ -566,8 +566,8 @@ func (b *StatusNode) Cleanup() error {
}
}
}
return nil
return nil
}
type RPCCall struct {

33
nodecfg/node_config.go
View File

@ -325,6 +325,8 @@ func insertClusterConfigNodes(tx *sql.Tx, c *params.NodeConfig) error {
return nil
}
// List of inserts to be executed when upgrading a node
// These INSERT queries should not be modified
func nodeConfigUpgradeInserts() []insertFn {
return []insertFn{
insertNodeConfig,
@ -346,6 +348,32 @@ func nodeConfigUpgradeInserts() []insertFn {
}
}
func nodeConfigNormalInserts() []insertFn {
// WARNING: if you are modifying one of the node config tables
// you need to edit `nodeConfigUpgradeInserts` to guarantee that
// the selects being used there are not affected.
return []insertFn{
insertNodeConfig,
insertHTTPConfig,
insertIPCConfig,
insertLogConfig,
insertUpstreamConfig,
insertNetworkConfig,
insertClusterConfig,
insertClusterConfigNodes,
insertLightETHConfig,
insertLightETHConfigTrustedNodes,
insertRegisterTopics,
insertRequireTopics,
insertPushNotificationsServerConfig,
insertShhExtConfig,
insertWakuConfig,
insertWakuV2Config,
insertTorrentConfig,
}
}
func execInsertFns(inFn []insertFn, tx *sql.Tx, c *params.NodeConfig) error {
for _, fn := range inFn {
err := fn(tx, c)
@ -362,10 +390,7 @@ func insertNodeConfigUpgrade(tx *sql.Tx, c *params.NodeConfig) error {
}
func SaveConfigWithTx(tx *sql.Tx, c *params.NodeConfig) error {
insertFNs := append(nodeConfigUpgradeInserts(),
insertTorrentConfig,
)
insertFNs := nodeConfigNormalInserts()
return execInsertFns(insertFNs, tx, c)
}

2
params/config.go
View File

@ -313,6 +313,8 @@ type NodeConfig struct {
// NetworkID sets network to use for selecting peers to connect to
NetworkID uint64 `json:"NetworkId" validate:"required"`
RootDataDir string `json:"-"`
// DataDir is the file system folder the node should use for any data storage needs.
DataDir string `validate:"required"`

7
protocol/common/message.go
View File

@ -137,6 +137,8 @@ type Message struct {
ImageLocalURL string `json:"imageLocalUrl,omitempty"`
// AudioLocalURL is the local url of the audio
AudioLocalURL string `json:"audioLocalUrl,omitempty"`
// StickerLocalURL is the local url of the sticker
StickerLocalURL string `json:"stickerLocalUrl,omitempty"`
// CommunityID is the id of the community to advertise
CommunityID string `json:"communityId,omitempty"`
@ -176,12 +178,16 @@ func (m *Message) PrepareServerURLs(port int) {
if m.ContentType == protobuf.ChatMessage_AUDIO {
m.AudioLocalURL = fmt.Sprintf("https://localhost:%d/messages/audio?messageId=%s", port, m.ID)
}
if m.ContentType == protobuf.ChatMessage_STICKER {
m.StickerLocalURL = fmt.Sprintf("https://localhost:%d/ipfs?hash=%s", port, m.GetSticker().Hash)
}
}
func (m *Message) MarshalJSON() ([]byte, error) {
type StickerAlias struct {
Hash string `json:"hash"`
Pack int32 `json:"pack"`
URL string `json:"url"`
}
item := struct {
ID string `json:"id"`
@ -258,6 +264,7 @@ func (m *Message) MarshalJSON() ([]byte, error) {
item.Sticker = &StickerAlias{
Pack: sticker.Pack,
Hash: sticker.Hash,
URL: m.StickerLocalURL,
}
}

15
protocol/messenger.go
View File

@ -391,11 +391,6 @@ func NewMessenger(
}
mailservers := mailserversDB.NewDB(database)
httpServer, err := server.NewServer(database, logger)
if err != nil {
return nil, err
}
messenger = &Messenger{
config: &c,
@ -434,14 +429,13 @@ func NewMessenger(
quit: make(chan struct{}),
requestedCommunities: make(map[string]*transport.Filter),
browserDatabase: c.browserDatabase,
httpServer: httpServer,
httpServer: c.httpServer,
shutdownTasks: []func() error{
ensVerifier.Stop,
pushNotificationClient.Stop,
communitiesManager.Stop,
encryptionProtocol.Stop,
transp.ResetFilters,
httpServer.Stop,
transp.Stop,
func() error { sender.Stop(); return nil },
// Currently this often fails, seems like it's safe to ignore them
@ -671,10 +665,12 @@ func (m *Messenger) Start() (*MessengerResponse, error) {
}
}
if m.httpServer != nil {
err = m.httpServer.Start()
if err != nil {
return nil, err
}
}
return response, nil
}
@ -4051,18 +4047,21 @@ func (m *Messenger) MessageByChatID(chatID, cursor string, limit int) ([]*common
}
}
if m.httpServer != nil {
for idx := range msgs {
msgs[idx].PrepareServerURLs(m.httpServer.Port)
}
}
return msgs, nextCursor, nil
}
func (m *Messenger) prepareMessages(messages map[string]*common.Message) {
if m.httpServer != nil {
for idx := range messages {
messages[idx].PrepareServerURLs(m.httpServer.Port)
}
}
}
func (m *Messenger) AllMessageByChatIDWhichMatchTerm(chatID string, searchTerm string, caseSensitive bool) ([]*common.Message, error) {

9
protocol/messenger_config.go
View File

@ -4,6 +4,7 @@ import (
"database/sql"
"encoding/json"
"github.com/status-im/status-go/server"
"github.com/status-im/status-go/services/browsers"
"go.uber.org/zap"
@ -68,6 +69,7 @@ type config struct {
clusterConfig params.ClusterConfig
browserDatabase *browsers.Database
torrentConfig *params.TorrentConfig
httpServer *server.Server
verifyTransactionClient EthClient
verifyENSURL string
@ -275,3 +277,10 @@ func WithTorrentConfig(tc *params.TorrentConfig) Option {
return nil
}
}
func WithHTTPServer(s *server.Server) Option {
return func(c *config) error {
c.httpServer = s
return nil
}
}

38
server/server.go
View File

@ -14,6 +14,8 @@ import (
"go.uber.org/zap"
"github.com/status-im/status-go/ipfs"
"github.com/status-im/status-go/logutils"
"github.com/status-im/status-go/protocol/identity/identicon"
"github.com/status-im/status-go/protocol/images"
)
@ -78,6 +80,11 @@ type identiconHandler struct {
logger *zap.Logger
}
type ipfsHandler struct {
logger *zap.Logger
downloader *ipfs.Downloader
}
func (s *identiconHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
pks, ok := r.URL.Query()["publicKey"]
if !ok || len(pks) == 0 {
@ -160,6 +167,30 @@ func (s *audioHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
func (s *ipfsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
hashes, ok := r.URL.Query()["hash"]
if !ok || len(hashes) == 0 {
s.logger.Error("no hash")
return
}
_, download := r.URL.Query()["download"]
content, err := s.downloader.Get(hashes[0], download)
if err != nil {
s.logger.Error("could not download hash", zap.Error(err))
return
}
w.Header().Set("Cache-Control", "max-age:290304000, public")
w.Header().Set("Expires", time.Now().AddDate(60, 0, 0).Format(http.TimeFormat))
_, err = w.Write(content)
if err != nil {
s.logger.Error("failed to write ipfs resource", zap.Error(err))
}
}
type Server struct {
Port int
run bool
@ -167,16 +198,17 @@ type Server struct {
logger *zap.Logger
db *sql.DB
cert *tls.Certificate
downloader *ipfs.Downloader
}
func NewServer(db *sql.DB, logger *zap.Logger) (*Server, error) {
func NewServer(db *sql.DB, downloader *ipfs.Downloader) (*Server, error) {
err := generateTLSCert()
if err != nil {
return nil, err
}
return &Server{db: db, logger: logger, cert: globalCertificate, Port: 0}, nil
return &Server{db: db, logger: logutils.ZapLogger(), cert: globalCertificate, Port: 0, downloader: downloader}, nil
}
func (s *Server) listenAndServe() {
@ -216,6 +248,8 @@ func (s *Server) Start() error {
handler.Handle("/messages/images", &imageHandler{db: s.db, logger: s.logger})
handler.Handle("/messages/audio", &audioHandler{db: s.db, logger: s.logger})
handler.Handle("/messages/identicons", &identiconHandler{logger: s.logger})
handler.Handle("/ipfs", &ipfsHandler{logger: s.logger, downloader: s.downloader})
s.server = &http.Server{Handler: handler}
go s.listenAndServe()

7
services/ext/service.go
View File

@ -11,6 +11,7 @@ import (
"path/filepath"
"time"
"github.com/status-im/status-go/server"
"github.com/status-im/status-go/services/browsers"
"github.com/syndtr/goleveldb/leveldb"
@ -104,7 +105,7 @@ func (s *Service) GetPeer(rawURL string) (*enode.Node, error) {
return enode.ParseV4(rawURL)
}
func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, db *sql.DB, multiAccountDb *multiaccounts.Database, acc *multiaccounts.Account, logger *zap.Logger) error {
func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, db *sql.DB, httpServer *server.Server, multiAccountDb *multiaccounts.Database, acc *multiaccounts.Account, logger *zap.Logger) error {
var err error
if !s.config.ShhextConfig.PFSEnabled {
return nil
@ -143,7 +144,7 @@ func (s *Service) InitProtocol(nodeName string, identity *ecdsa.PrivateKey, db *
s.multiAccountsDB = multiAccountDb
s.account = acc
options, err := buildMessengerOptions(s.config, identity, db, s.multiAccountsDB, acc, envelopesMonitorConfig, s.accountsDB, logger, &MessengerSignalsHandler{})
options, err := buildMessengerOptions(s.config, identity, db, httpServer, s.multiAccountsDB, acc, envelopesMonitorConfig, s.accountsDB, logger, &MessengerSignalsHandler{})
if err != nil {
return err
}
@ -389,6 +390,7 @@ func buildMessengerOptions(
config params.NodeConfig,
identity *ecdsa.PrivateKey,
db *sql.DB,
httpServer *server.Server,
multiAccounts *multiaccounts.Database,
account *multiaccounts.Account,
envelopesMonitorConfig *transport.EnvelopesMonitorConfig,
@ -409,6 +411,7 @@ func buildMessengerOptions(
protocol.WithENSVerificationConfig(publishMessengerResponse, config.ShhextConfig.VerifyENSURL, config.ShhextConfig.VerifyENSContractAddress),
protocol.WithClusterConfig(config.ClusterConfig),
protocol.WithTorrentConfig(&config.TorrentConfig),
protocol.WithHTTPServer(httpServer),
}
if config.ShhextConfig.DataSyncEnabled {

140
services/stickers/api.go
View File

@ -2,15 +2,9 @@ package stickers
import (
"context"
"errors"
"io/ioutil"
"fmt"
"math/big"
"net/http"
"time"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multibase"
"github.com/wealdtech/go-multicodec"
"github.com/zenthangplus/goccm"
"olympos.io/encoding/edn"
@ -22,14 +16,14 @@ import (
"github.com/status-im/status-go/contracts"
"github.com/status-im/status-go/contracts/stickers"
"github.com/status-im/status-go/eth-node/types"
"github.com/status-im/status-go/ipfs"
"github.com/status-im/status-go/multiaccounts/accounts"
"github.com/status-im/status-go/params"
"github.com/status-im/status-go/rpc"
"github.com/status-im/status-go/server"
"github.com/status-im/status-go/services/rpcfilters"
"github.com/status-im/status-go/services/wallet/bigint"
)
const ipfsGateway = ".ipfs.infura-ipfs.io/"
const maxConcurrentRequests = 3
// ConnectionType constants
@ -47,9 +41,12 @@ type API struct {
accountsManager *account.GethManager
accountsDB *accounts.Database
rpcFiltersSrvc *rpcfilters.Service
config *params.NodeConfig
keyStoreDir string
downloader *ipfs.Downloader
httpServer *server.Server
ctx context.Context
client *http.Client
}
type Sticker struct {
@ -68,7 +65,7 @@ type StickerPack struct {
Thumbnail string `json:"thumbnail"`
Stickers []Sticker `json:"stickers"`
Status stickerStatus `json:"status,omitempty"`
Status stickerStatus `json:"status"`
}
type StickerPackCollection map[uint]StickerPack
@ -88,20 +85,21 @@ type ednStickerPackInfo struct {
Meta ednStickerPack
}
func NewAPI(ctx context.Context, acc *accounts.Database, rpcClient *rpc.Client, accountsManager *account.GethManager, rpcFiltersSrvc *rpcfilters.Service, config *params.NodeConfig) *API {
return &API{
func NewAPI(ctx context.Context, acc *accounts.Database, rpcClient *rpc.Client, accountsManager *account.GethManager, rpcFiltersSrvc *rpcfilters.Service, keyStoreDir string, downloader *ipfs.Downloader, httpServer *server.Server) *API {
result := &API{
contractMaker: &contracts.ContractMaker{
RPCClient: rpcClient,
},
accountsManager: accountsManager,
accountsDB: acc,
rpcFiltersSrvc: rpcFiltersSrvc,
config: config,
keyStoreDir: keyStoreDir,
downloader: downloader,
ctx: ctx,
client: &http.Client{
Timeout: time.Second * 5,
},
httpServer: httpServer,
}
return result
}
func (api *API) Market(chainID uint64) ([]StickerPack, error) {
@ -232,39 +230,6 @@ func (api *API) getPurchasedPackIDs(chainID uint64, account types.Address) ([]*b
return api.getTokenPackIDs(chainID, tokenIDs)
}
func hashToURL(hash []byte) (string, error) {
// contract response includes a contenthash, which needs to be decoded to reveal
// an IPFS identifier. Once decoded, download the content from IPFS. This content
// is in EDN format, ie https://ipfs.infura.io/ipfs/QmWVVLwVKCwkVNjYJrRzQWREVvEk917PhbHYAUhA1gECTM
// and it also needs to be decoded in to a nim type
data, codec, err := multicodec.RemoveCodec(hash)
if err != nil {
return "", err
}
codecName, err := multicodec.Name(codec)
if err != nil {
return "", err
}
if codecName != "ipfs-ns" {
return "", errors.New("codecName is not ipfs-ns")
}
thisCID, err := cid.Parse(data)
if err != nil {
return "", err
}
str, err := thisCID.StringOfBase(multibase.Base32)
if err != nil {
return "", err
}
return "https://" + str + ipfsGateway, nil
}
func (api *API) fetchStickerPacks(chainID uint64, resultChan chan<- *StickerPack, errChan chan<- error, doneChan chan<- struct{}) {
defer close(doneChan)
defer close(errChan)
@ -339,18 +304,13 @@ func (api *API) fetchPackData(stickerType *stickers.StickerType, packID *big.Int
return nil, err
}
packDetailsURL, err := hashToURL(packData.Contenthash)
if err != nil {
return nil, err
}
stickerPack := &StickerPack{
ID: &bigint.BigInt{Int: packID},
Owner: packData.Owner,
Price: &bigint.BigInt{Int: packData.Price},
}
err = api.downloadIPFSData(stickerPack, packDetailsURL, translateHashes)
err = api.downloadPackData(stickerPack, packData.Contenthash, translateHashes)
if err != nil {
return nil, err
}
@ -358,34 +318,19 @@ func (api *API) fetchPackData(stickerType *stickers.StickerType, packID *big.Int
return stickerPack, nil
}
func (api *API) downloadIPFSData(stickerPack *StickerPack, packDetailsURL string, translateHashes bool) error {
// This can be improved by adding a cache using packDetailsURL as key
req, err := http.NewRequest(http.MethodGet, packDetailsURL, nil)
func (api *API) downloadPackData(stickerPack *StickerPack, contentHash []byte, translateHashes bool) error {
fileContent, err := api.downloader.Get(hexutil.Encode(contentHash)[2:], true)
if err != nil {
return err
}
resp, err := api.client.Do(req)
if err != nil {
return err
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Error("failed to close the stickerpack request body", "err", err)
}
}()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return populateStickerPackAttributes(stickerPack, body, translateHashes)
return api.populateStickerPackAttributes(stickerPack, fileContent, translateHashes)
}
func populateStickerPackAttributes(stickerPack *StickerPack, ednSource []byte, translateHashes bool) error {
func (api *API) hashToURL(hash string) string {
return fmt.Sprintf("https://localhost:%d/ipfs?hash=%s", api.httpServer.Port, hash)
}
func (api *API) populateStickerPackAttributes(stickerPack *StickerPack, ednSource []byte, translateHashes bool) error {
var stickerpackIPFSInfo ednStickerPackInfo
err := edn.Unmarshal(ednSource, &stickerpackIPFSInfo)
if err != nil {
@ -396,15 +341,8 @@ func populateStickerPackAttributes(stickerPack *StickerPack, ednSource []byte, t
stickerPack.Name = stickerpackIPFSInfo.Meta.Name
if translateHashes {
stickerPack.Preview, err = decodeStringHash(stickerpackIPFSInfo.Meta.Preview)
if err != nil {
return err
}
stickerPack.Thumbnail, err = decodeStringHash(stickerpackIPFSInfo.Meta.Thumbnail)
if err != nil {
return err
}
stickerPack.Preview = api.hashToURL(stickerpackIPFSInfo.Meta.Preview)
stickerPack.Thumbnail = api.hashToURL(stickerpackIPFSInfo.Meta.Thumbnail)
} else {
stickerPack.Preview = stickerpackIPFSInfo.Meta.Preview
stickerPack.Thumbnail = stickerpackIPFSInfo.Meta.Thumbnail
@ -413,15 +351,7 @@ func populateStickerPackAttributes(stickerPack *StickerPack, ednSource []byte, t
for _, s := range stickerpackIPFSInfo.Meta.Stickers {
url := ""
if translateHashes {
hash, err := hexutil.Decode("0x" + s.Hash)
if err != nil {
return err
}
url, err = hashToURL(hash)
if err != nil {
return err
}
url = api.hashToURL(s.Hash)
}
stickerPack.Stickers = append(stickerPack.Stickers, Sticker{
@ -434,20 +364,6 @@ func populateStickerPackAttributes(stickerPack *StickerPack, ednSource []byte, t
return nil
}
func decodeStringHash(input string) (string, error) {
hash, err := hexutil.Decode("0x" + input)
if err != nil {
return "", err
}
url, err := hashToURL(hash)
if err != nil {
return "", err
}
return url, nil
}
func (api *API) getContractPacks(chainID uint64) ([]StickerPack, error) {
stickerPackChan := make(chan *StickerPack)
errChan := make(chan error)

16
services/stickers/install.go
View File

@ -68,25 +68,15 @@ func (api *API) Installed() (StickerPackCollection, error) {
for packID, stickerPack := range stickerPacks {
stickerPack.Status = statusInstalled
stickerPack.Preview, err = decodeStringHash(stickerPack.Preview)
if err != nil {
return nil, err
}
stickerPack.Thumbnail, err = decodeStringHash(stickerPack.Thumbnail)
if err != nil {
return nil, err
}
stickerPack.Preview = api.hashToURL(stickerPack.Preview)
stickerPack.Thumbnail = api.hashToURL(stickerPack.Thumbnail)
for i, sticker := range stickerPack.Stickers {
sticker.URL, err = decodeStringHash(sticker.Hash)
sticker.URL = api.hashToURL(sticker.Hash)
if err != nil {
return nil, err
}
stickerPack.Stickers[i] = sticker
}
stickerPacks[packID] = stickerPack
}

63
services/stickers/pending.go
View File

@ -3,6 +3,7 @@ package stickers
import (
"encoding/json"
"errors"
"math/big"
"github.com/status-im/status-go/multiaccounts/settings"
"github.com/status-im/status-go/services/wallet/bigint"
@ -61,31 +62,61 @@ func (api *API) Pending() (StickerPackCollection, error) {
for packID, stickerPack := range stickerPacks {
stickerPack.Status = statusPending
stickerPack.Preview, err = decodeStringHash(stickerPack.Preview)
if err != nil {
return nil, err
}
stickerPack.Thumbnail, err = decodeStringHash(stickerPack.Thumbnail)
if err != nil {
return nil, err
}
stickerPack.Preview = api.hashToURL(stickerPack.Preview)
stickerPack.Thumbnail = api.hashToURL(stickerPack.Thumbnail)
for i, sticker := range stickerPack.Stickers {
sticker.URL, err = decodeStringHash(sticker.Hash)
if err != nil {
return nil, err
}
sticker.URL = api.hashToURL(sticker.Hash)
stickerPack.Stickers[i] = sticker
}
stickerPacks[packID] = stickerPack
}
return stickerPacks, nil
}
func (api *API) ProcessPending(chainID uint64) (pendingChanged StickerPackCollection, err error) {
pendingStickerPacks, err := api.pendingStickerPacks()
if err != nil {
return nil, err
}
accs, err := api.accountsDB.GetAccounts()
if err != nil {
return nil, err
}
purchasedPacks := make(map[uint]struct{})
purchasedPackChan := make(chan *big.Int)
errChan := make(chan error)
doneChan := make(chan struct{}, 1)
go api.getAccountsPurchasedPack(chainID, accs, purchasedPackChan, errChan, doneChan)
for {
select {
case err := <-errChan:
if err != nil {
return nil, err
}
case packID := <-purchasedPackChan:
if packID != nil {
purchasedPacks[uint(packID.Uint64())] = struct{}{}
}
case <-doneChan:
result := make(StickerPackCollection)
for _, stickerPack := range pendingStickerPacks {
packID := uint(stickerPack.ID.Uint64())
if _, exists := purchasedPacks[packID]; !exists {
continue
}
delete(pendingStickerPacks, packID)
stickerPack.Status = statusPurchased
result[packID] = stickerPack
}
err = api.accountsDB.SaveSettingField(settings.StickersPacksPending, pendingStickerPacks)
return result, err
}
}
}
func (api *API) RemovePending(packID *bigint.BigInt) error {
pendingPacks, err := api.pendingStickerPacks()
if err != nil {

13
services/stickers/recent.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"github.com/status-im/status-go/multiaccounts/settings"
"github.com/status-im/status-go/services/wallet/bigint"
)
const maxNumberRecentStickers = 24
@ -40,17 +41,19 @@ func (api *API) Recent() ([]Sticker, error) {
}
for i, sticker := range recentStickersList {
sticker.URL, err = decodeStringHash(sticker.Hash)
if err != nil {
return nil, err
}
sticker.URL = api.hashToURL(sticker.Hash)
recentStickersList[i] = sticker
}
return recentStickersList, nil
}
func (api *API) AddRecent(sticker Sticker) error {
func (api *API) AddRecent(packID *bigint.BigInt, hash string) error {
sticker := Sticker{
PackID: packID,
Hash: hash,
}
recentStickersList, err := api.recentStickers()
if err != nil {
return err

16
services/stickers/service.go
View File

@ -6,14 +6,16 @@ import (
"github.com/ethereum/go-ethereum/p2p"
ethRpc "github.com/ethereum/go-ethereum/rpc"
"github.com/status-im/status-go/account"
"github.com/status-im/status-go/ipfs"
"github.com/status-im/status-go/multiaccounts/accounts"
"github.com/status-im/status-go/params"
"github.com/status-im/status-go/rpc"
"github.com/status-im/status-go/server"
"github.com/status-im/status-go/services/rpcfilters"
)
// NewService initializes service instance.
func NewService(acc *accounts.Database, rpcClient *rpc.Client, accountsManager *account.GethManager, rpcFiltersSrvc *rpcfilters.Service, config *params.NodeConfig) *Service {
func NewService(acc *accounts.Database, rpcClient *rpc.Client, accountsManager *account.GethManager, rpcFiltersSrvc *rpcfilters.Service, config *params.NodeConfig, downloader *ipfs.Downloader, httpServer *server.Server) *Service {
ctx, cancel := context.WithCancel(context.Background())
return &Service{
@ -21,8 +23,9 @@ func NewService(acc *accounts.Database, rpcClient *rpc.Client, accountsManager *
rpcClient: rpcClient,
accountsManager: accountsManager,
rpcFiltersSrvc: rpcFiltersSrvc,
config: config,
keyStoreDir: config.KeyStoreDir,
downloader: downloader,
httpServer: httpServer,
ctx: ctx,
cancel: cancel,
}
@ -34,8 +37,9 @@ type Service struct {
rpcClient *rpc.Client
accountsManager *account.GethManager
rpcFiltersSrvc *rpcfilters.Service
config *params.NodeConfig
downloader *ipfs.Downloader
keyStoreDir string
httpServer *server.Server
ctx context.Context
cancel context.CancelFunc
}
@ -57,7 +61,7 @@ func (s *Service) APIs() []ethRpc.API {
{
Namespace: "stickers",
Version: "0.1.0",
Service: NewAPI(s.ctx, s.accountsDB, s.rpcClient, s.accountsManager, s.rpcFiltersSrvc, s.config),
Service: NewAPI(s.ctx, s.accountsDB, s.rpcClient, s.accountsManager, s.rpcFiltersSrvc, s.keyStoreDir, s.downloader, s.httpServer),
},
}
}

80
services/stickers/transactions.go
View File

@ -5,6 +5,8 @@ import (
"math/big"
"strings"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
@ -19,7 +21,7 @@ import (
func (api *API) getSigner(chainID uint64, from types.Address, password string) bind.SignerFn {
return func(addr common.Address, tx *ethTypes.Transaction) (*ethTypes.Transaction, error) {
selectedAccount, err := api.accountsManager.VerifyAccountPassword(api.config.KeyStoreDir, from.Hex(), password)
selectedAccount, err := api.accountsManager.VerifyAccountPassword(api.keyStoreDir, from.Hex(), password)
if err != nil {
return nil, err
}
@ -84,63 +86,99 @@ func (api *API) Buy(ctx context.Context, chainID uint64, txArgs transactions.Sen
return tx.Hash().String(), nil
}
func (api *API) BuyEstimate(ctx context.Context, chainID uint64, from types.Address, packID *bigint.BigInt) (uint64, error) {
func (api *API) BuyPrepareTxCallMsg(chainID uint64, from types.Address, packID *bigint.BigInt) (ethereum.CallMsg, error) {
callOpts := &bind.CallOpts{Context: api.ctx, Pending: false}
stickerType, err := api.contractMaker.NewStickerType(chainID)
if err != nil {
return 0, err
return ethereum.CallMsg{}, err
}
packInfo, err := stickerType.GetPackData(callOpts, packID.Int)
if err != nil {
return 0, err
return ethereum.CallMsg{}, err
}
stickerMarketABI, err := abi.JSON(strings.NewReader(stickers.StickerMarketABI))
if err != nil {
return 0, err
return ethereum.CallMsg{}, err
}
extraData, err := stickerMarketABI.Pack("buyToken", packID.Int, from, packInfo.Price)
if err != nil {
return 0, err
return ethereum.CallMsg{}, err
}
sntABI, err := abi.JSON(strings.NewReader(snt.SNTABI))
if err != nil {
return 0, err
return ethereum.CallMsg{}, err
}
stickerMarketAddress, err := stickers.StickerMarketContractAddress(chainID)
if err != nil {
return 0, err
return ethereum.CallMsg{}, err
}
data, err := sntABI.Pack("approveAndCall", stickerMarketAddress, packInfo.Price, extraData)
if err != nil {
return 0, err
return ethereum.CallMsg{}, err
}
sntAddress, err := snt.ContractAddress(chainID)
if err != nil {
return ethereum.CallMsg{}, err
}
return ethereum.CallMsg{
From: common.Address(from),
To: &sntAddress,
Value: big.NewInt(0),
Data: data,
}, nil
}
func (api *API) BuyPrepareTx(ctx context.Context, chainID uint64, from types.Address, packID *bigint.BigInt) (interface{}, error) {
callMsg, err := api.BuyPrepareTxCallMsg(chainID, from, packID)
if err != nil {
return nil, err
}
return toCallArg(callMsg), nil
}
func (api *API) BuyEstimate(ctx context.Context, chainID uint64, from types.Address, packID *bigint.BigInt) (uint64, error) {
callMsg, err := api.BuyPrepareTxCallMsg(chainID, from, packID)
if err != nil {
return 0, err
}
ethClient, err := api.contractMaker.RPCClient.EthClient(chainID)
if err != nil {
return 0, err
}
sntAddress, err := snt.ContractAddress(chainID)
if err != nil {
return 0, err
}
return ethClient.EstimateGas(ctx, ethereum.CallMsg{
From: common.Address(from),
To: &sntAddress,
Value: big.NewInt(0),
Data: data,
})
return ethClient.EstimateGas(ctx, callMsg)
}
func (api *API) StickerMarketAddress(ctx context.Context, chainID uint64) (common.Address, error) {
return stickers.StickerMarketContractAddress(chainID)
}
func toCallArg(msg ethereum.CallMsg) interface{} {
arg := map[string]interface{}{
"from": msg.From,
"to": msg.To,
}
if len(msg.Data) > 0 {
arg["data"] = hexutil.Bytes(msg.Data)
}
if msg.Value != nil {
arg["value"] = (*hexutil.Big)(msg.Value)
}
if msg.Gas != 0 {
arg["gas"] = hexutil.Uint64(msg.Gas)
}
if msg.GasPrice != nil {
arg["gasPrice"] = (*hexutil.Big)(msg.GasPrice)
}
return arg
}

4
services/wakuext/api_test.go
View File

@ -136,7 +136,7 @@ func TestInitProtocol(t *testing.T) {
acc := &multiaccounts.Account{KeyUID: "0xdeadbeef"}
err = service.InitProtocol("Test", privateKey, sqlDB, multiAccounts, acc, zap.NewNop())
err = service.InitProtocol("Test", privateKey, sqlDB, nil, multiAccounts, acc, zap.NewNop())
require.NoError(t, err)
}
@ -201,7 +201,7 @@ func (s *ShhExtSuite) createAndAddNode() {
acc := &multiaccounts.Account{KeyUID: "0xdeadbeef"}
err = service.InitProtocol("Test", privateKey, sqlDB, multiAccounts, acc, zap.NewNop())
err = service.InitProtocol("Test", privateKey, sqlDB, nil, multiAccounts, acc, zap.NewNop())
s.NoError(err)
stack.RegisterLifecycle(service)

2
services/wallet/price.go
View File

@ -12,7 +12,7 @@ import (
func fetchCryptoComparePrices(symbols []string, currency string) (map[string]float64, error) {
httpClient := http.Client{Timeout: time.Minute}
url := fmt.Sprintf("https://min-api.cryptocompare.com/data/pricemulti?fsyms=%s&tsyms=%s", strings.Join(symbols, ","), currency)
url := fmt.Sprintf("https://min-api.cryptocompare.com/data/pricemulti?fsyms=%s&tsyms=%s&extraParams=Status.im", strings.Join(symbols, ","), currency)
resp, err := httpClient.Get(url)
if err != nil {
return nil, err