Add support for request messages by topics (#1805)

This commit is contained in:
Adam Babik 2020-01-21 08:11:24 +01:00 committed by GitHub
parent 3b81bd2878
commit bc2d018483
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 270 additions and 109 deletions

View File

@ -1,6 +1,9 @@
package gethbridge package gethbridge
import ( import (
"io"
"github.com/ethereum/go-ethereum/rlp"
"github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/eth-node/types"
"github.com/status-im/status-go/waku" "github.com/status-im/status-go/waku"
"github.com/status-im/status-go/whisper/v6" "github.com/status-im/status-go/whisper/v6"
@ -15,15 +18,8 @@ func NewWhisperEnvelope(e *whisper.Envelope) types.Envelope {
return &whisperEnvelope{env: e} return &whisperEnvelope{env: e}
} }
func UnwrapWhisperEnvelope(e types.Envelope) (*whisper.Envelope, bool) { func (w *whisperEnvelope) Unwrap() interface{} {
if env, ok := e.(*whisperEnvelope); ok { return w.env
return env.env, true
}
return nil, false
}
func MustUnwrapWhisperEnvelope(e types.Envelope) *whisper.Envelope {
return e.(*whisperEnvelope).env
} }
func (w *whisperEnvelope) Hash() types.Hash { func (w *whisperEnvelope) Hash() types.Hash {
@ -54,6 +50,14 @@ func (w *whisperEnvelope) Size() int {
return len(w.env.Data) return len(w.env.Data)
} }
func (w *whisperEnvelope) DecodeRLP(s *rlp.Stream) error {
return w.env.DecodeRLP(s)
}
func (w *whisperEnvelope) EncodeRLP(writer io.Writer) error {
return rlp.Encode(writer, w.env)
}
type wakuEnvelope struct { type wakuEnvelope struct {
env *waku.Envelope env *waku.Envelope
} }
@ -63,15 +67,8 @@ func NewWakuEnvelope(e *waku.Envelope) types.Envelope {
return &wakuEnvelope{env: e} return &wakuEnvelope{env: e}
} }
func UnwrapWakuEnvelope(e types.Envelope) (*waku.Envelope, bool) { func (w *wakuEnvelope) Unwrap() interface{} {
if env, ok := e.(*wakuEnvelope); ok { return w.env
return env.env, true
}
return nil, false
}
func MustUnwrapWakuEnvelope(e types.Envelope) *waku.Envelope {
return e.(*wakuEnvelope).env
} }
func (w *wakuEnvelope) Hash() types.Hash { func (w *wakuEnvelope) Hash() types.Hash {
@ -101,3 +98,11 @@ func (w *wakuEnvelope) Topic() types.TopicType {
func (w *wakuEnvelope) Size() int { func (w *wakuEnvelope) Size() int {
return len(w.env.Data) return len(w.env.Data)
} }
func (w *wakuEnvelope) DecodeRLP(s *rlp.Stream) error {
return w.env.DecodeRLP(s)
}
func (w *wakuEnvelope) EncodeRLP(writer io.Writer) error {
return rlp.Encode(writer, w.env)
}

View File

@ -166,7 +166,7 @@ func (w *gethWakuWrapper) SendMessagesRequest(peerID []byte, r types.MessagesReq
// which are not supposed to be forwarded any further. // which are not supposed to be forwarded any further.
// The whisper protocol is agnostic of the format and contents of envelope. // The whisper protocol is agnostic of the format and contents of envelope.
func (w *gethWakuWrapper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope types.Envelope, timeout time.Duration) error { func (w *gethWakuWrapper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope types.Envelope, timeout time.Duration) error {
return w.waku.RequestHistoricMessagesWithTimeout(peerID, MustUnwrapWakuEnvelope(envelope), timeout) return w.waku.RequestHistoricMessagesWithTimeout(peerID, envelope.Unwrap().(*waku.Envelope), timeout)
} }
type wakuFilterWrapper struct { type wakuFilterWrapper struct {

View File

@ -171,7 +171,7 @@ func (w *gethWhisperWrapper) SendMessagesRequest(peerID []byte, r types.Messages
// which are not supposed to be forwarded any further. // which are not supposed to be forwarded any further.
// The whisper protocol is agnostic of the format and contents of envelope. // The whisper protocol is agnostic of the format and contents of envelope.
func (w *gethWhisperWrapper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope types.Envelope, timeout time.Duration) error { func (w *gethWhisperWrapper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope types.Envelope, timeout time.Duration) error {
return w.whisper.RequestHistoricMessagesWithTimeout(peerID, MustUnwrapWhisperEnvelope(envelope), timeout) return w.whisper.RequestHistoricMessagesWithTimeout(peerID, envelope.Unwrap().(*whisper.Envelope), timeout)
} }
// SyncMessages can be sent between two Mail Servers and syncs envelopes between them. // SyncMessages can be sent between two Mail Servers and syncs envelopes between them.

View File

@ -3,7 +3,9 @@ package types
// Envelope represents a clear-text data packet to transmit through the Whisper // Envelope represents a clear-text data packet to transmit through the Whisper
// network. Its contents may or may not be encrypted and signed. // network. Its contents may or may not be encrypted and signed.
type Envelope interface { type Envelope interface {
Hash() Hash // Cached hash of the envelope to avoid rehashing every time. Wrapped
Hash() Hash // cached hash of the envelope to avoid rehashing every time
Bloom() []byte Bloom() []byte
PoW() float64 PoW() float64
Expiry() uint32 Expiry() uint32

View File

@ -0,0 +1,7 @@
package types
// Wrapped tells that a given object has an underlying representation
// and this representation can be accessed using `Unwrap` method.
type Wrapped interface {
Unwrap() interface{}
}

View File

@ -136,7 +136,7 @@ func countMessages(t *testing.T, db DB) int {
} }
i, _ := db.BuildIterator(query) i, _ := db.BuildIterator(query)
defer i.Release() defer func() { _ = i.Release() }()
for i.Next() { for i.Next() {
var env whisper.Envelope var env whisper.Envelope

View File

@ -379,6 +379,7 @@ func (s *WakuMailServer) Deliver(peerID []byte, req waku.MessagesRequest) {
Lower: req.From, Lower: req.From,
Upper: req.To, Upper: req.To,
Bloom: req.Bloom, Bloom: req.Bloom,
Topics: req.Topics,
Limit: req.Limit, Limit: req.Limit,
Cursor: req.Cursor, Cursor: req.Cursor,
Batch: true, Batch: true,
@ -512,7 +513,7 @@ func (whisperAdapter) CreateRequestCompletedPayload(reqID, lastEnvelopeHash type
func (whisperAdapter) CreateSyncResponse(envelopes []types.Envelope, cursor []byte, final bool, err string) interface{} { func (whisperAdapter) CreateSyncResponse(envelopes []types.Envelope, cursor []byte, final bool, err string) interface{} {
whisperEnvelopes := make([]*whisper.Envelope, len(envelopes)) whisperEnvelopes := make([]*whisper.Envelope, len(envelopes))
for i, env := range envelopes { for i, env := range envelopes {
whisperEnvelopes[i] = gethbridge.MustUnwrapWhisperEnvelope(env) whisperEnvelopes[i] = env.Unwrap().(*whisper.Envelope)
} }
return whisper.SyncResponse{ return whisper.SyncResponse{
Envelopes: whisperEnvelopes, Envelopes: whisperEnvelopes,
@ -698,6 +699,19 @@ func (s *mailServer) DeliverMail(peerID, reqID types.Hash, req MessagesRequestPa
req.SetDefaults() req.SetDefaults()
log.Info(
"[mailserver:DeliverMail] processing request",
"peerID", peerID.String(),
"requestID", reqID.String(),
"lower", req.Lower,
"upper", req.Upper,
"bloom", req.Bloom,
"topics", req.Topics,
"limit", req.Limit,
"cursor", req.Cursor,
"batch", req.Batch,
)
if err := req.Validate(); err != nil { if err := req.Validate(); err != nil {
syncFailuresCounter.WithLabelValues("req_invalid").Inc() syncFailuresCounter.WithLabelValues("req_invalid").Inc()
log.Error( log.Error(
@ -721,17 +735,6 @@ func (s *mailServer) DeliverMail(peerID, reqID types.Hash, req MessagesRequestPa
return return
} }
log.Info(
"[mailserver:DeliverMail] processing request",
"peerID", peerID.String(),
"requestID", reqID.String(),
"lower", req.Lower,
"upper", req.Upper,
"bloom", req.Bloom,
"limit", req.Limit,
"cursor", req.Cursor,
"batch", req.Batch,
)
if req.Batch { if req.Batch {
requestsBatchedCounter.Inc() requestsBatchedCounter.Inc()
} }
@ -745,7 +748,7 @@ func (s *mailServer) DeliverMail(peerID, reqID types.Hash, req MessagesRequestPa
) )
return return
} }
defer iter.Release() defer func() { _ = iter.Release() }()
bundles := make(chan []rlp.RawValue, 5) bundles := make(chan []rlp.RawValue, 5)
errCh := make(chan error) errCh := make(chan error)
@ -773,6 +776,7 @@ func (s *mailServer) DeliverMail(peerID, reqID types.Hash, req MessagesRequestPa
nextPageCursor, lastEnvelopeHash := s.processRequestInBundles( nextPageCursor, lastEnvelopeHash := s.processRequestInBundles(
iter, iter,
req.Bloom, req.Bloom,
req.Topics,
int(req.Limit), int(req.Limit),
processRequestTimeout, processRequestTimeout,
reqID.String(), reqID.String(),
@ -843,7 +847,7 @@ func (s *mailServer) SyncMail(peerID types.Hash, req MessagesRequestPayload) err
syncFailuresCounter.WithLabelValues("iterator").Inc() syncFailuresCounter.WithLabelValues("iterator").Inc()
return err return err
} }
defer iter.Release() defer func() { _ = iter.Release() }()
bundles := make(chan []rlp.RawValue, 5) bundles := make(chan []rlp.RawValue, 5)
errCh := make(chan error) errCh := make(chan error)
@ -864,6 +868,7 @@ func (s *mailServer) SyncMail(peerID types.Hash, req MessagesRequestPayload) err
nextCursor, _ := s.processRequestInBundles( nextCursor, _ := s.processRequestInBundles(
iter, iter,
req.Bloom, req.Bloom,
req.Topics,
int(req.Limit), int(req.Limit),
processRequestTimeout, processRequestTimeout,
requestID, requestID,
@ -960,6 +965,7 @@ func (s *mailServer) createIterator(req MessagesRequestPayload) (Iterator, error
func (s *mailServer) processRequestInBundles( func (s *mailServer) processRequestInBundles(
iter Iterator, iter Iterator,
bloom []byte, bloom []byte,
topics [][]byte,
limit int, limit int,
timeout time.Duration, timeout time.Duration,
requestID string, requestID string,

View File

@ -23,7 +23,7 @@ type DB interface {
type Iterator interface { type Iterator interface {
Next() bool Next() bool
DBKey() (*DBKey, error) DBKey() (*DBKey, error)
Release() Release() error
Error() error Error() error
GetEnvelope(bloom []byte) ([]byte, error) GetEnvelope(bloom []byte) ([]byte, error)
} }
@ -34,4 +34,5 @@ type CursorQuery struct {
cursor []byte cursor []byte
limit uint32 limit uint32
bloom []byte bloom []byte
topics [][]byte
} }

View File

@ -12,7 +12,6 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
gethbridge "github.com/status-im/status-go/eth-node/bridge/geth"
"github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/eth-node/types"
"github.com/status-im/status-go/whisper/v6" "github.com/status-im/status-go/whisper/v6"
) )
@ -55,7 +54,11 @@ func (i *LevelDBIterator) GetEnvelope(bloom []byte) ([]byte, error) {
return nil, nil return nil, nil
} }
return rawValue, nil return rawValue, nil
}
func (i *LevelDBIterator) Release() error {
i.Iterator.Release()
return nil
} }
func NewLevelDB(dataDir string) (*LevelDB, error) { func NewLevelDB(dataDir string) (*LevelDB, error) {
@ -103,7 +106,7 @@ func (db *LevelDB) Prune(t time.Time, batchSize int) (int, error) {
if err != nil { if err != nil {
return 0, err return 0, err
} }
defer i.Release() defer func() { _ = i.Release() }()
batch := leveldb.Batch{} batch := leveldb.Batch{}
removed := 0 removed := 0
@ -142,18 +145,7 @@ func (db *LevelDB) SaveEnvelope(env types.Envelope) error {
defer recoverLevelDBPanics("SaveEnvelope") defer recoverLevelDBPanics("SaveEnvelope")
key := NewDBKey(env.Expiry()-env.TTL(), env.Topic(), env.Hash()) key := NewDBKey(env.Expiry()-env.TTL(), env.Topic(), env.Hash())
rawEnvelope, err := rlp.EncodeToBytes(env.Unwrap())
var (
rawEnvelope []byte
err error
)
if whisperEnv, ok := gethbridge.UnwrapWhisperEnvelope(env); ok {
rawEnvelope, err = rlp.EncodeToBytes(whisperEnv)
} else if wakuEnv, ok := gethbridge.UnwrapWakuEnvelope(env); ok {
rawEnvelope, err = rlp.EncodeToBytes(wakuEnv)
} else {
return errors.New("unsupported underlying types.Envelope type")
}
if err != nil { if err != nil {
log.Error(fmt.Sprintf("rlp.EncodeToBytes failed: %s", err)) log.Error(fmt.Sprintf("rlp.EncodeToBytes failed: %s", err))
archivedErrorsCounter.Inc() archivedErrorsCounter.Inc()

View File

@ -2,9 +2,12 @@ package mailserver
import ( import (
"database/sql" "database/sql"
"errors"
"fmt" "fmt"
"time" "time"
"github.com/lib/pq"
// Import postgres driver // Import postgres driver
_ "github.com/lib/pq" _ "github.com/lib/pq"
"github.com/status-im/migrate/v4" "github.com/status-im/migrate/v4"
@ -20,6 +23,10 @@ import (
"github.com/status-im/status-go/whisper/v6" "github.com/status-im/status-go/whisper/v6"
) )
type PostgresDB struct {
db *sql.DB
}
func NewPostgresDB(uri string) (*PostgresDB, error) { func NewPostgresDB(uri string) (*PostgresDB, error) {
db, err := sql.Open("postgres", uri) db, err := sql.Open("postgres", uri)
if err != nil { if err != nil {
@ -34,10 +41,6 @@ func NewPostgresDB(uri string) (*PostgresDB, error) {
return instance, nil return instance, nil
} }
type PostgresDB struct {
db *sql.DB
}
type postgresIterator struct { type postgresIterator struct {
*sql.Rows *sql.Rows
} }
@ -52,11 +55,11 @@ func (i *postgresIterator) DBKey() (*DBKey, error) {
} }
func (i *postgresIterator) Error() error { func (i *postgresIterator) Error() error {
return nil return i.Err()
} }
func (i *postgresIterator) Release() { func (i *postgresIterator) Release() error {
i.Close() return i.Close()
} }
func (i *postgresIterator) GetEnvelope(bloom []byte) ([]byte, error) { func (i *postgresIterator) GetEnvelope(bloom []byte) ([]byte, error) {
@ -70,35 +73,40 @@ func (i *postgresIterator) GetEnvelope(bloom []byte) ([]byte, error) {
} }
func (i *PostgresDB) BuildIterator(query CursorQuery) (Iterator, error) { func (i *PostgresDB) BuildIterator(query CursorQuery) (Iterator, error) {
var upperLimit []byte var args []interface{}
var stmtString string
if len(query.cursor) > 0 {
// If we have a cursor, we don't want to include that envelope in the result set
upperLimit = query.cursor
// We disable security checks as we need to use string interpolation stmtString := "SELECT id, data FROM envelopes"
// for this, but it's converted to 0s and 1s so no injection should be possible
/* #nosec */ if len(query.cursor) > 0 {
stmtString = fmt.Sprintf("SELECT id, data FROM envelopes where id >= $1 AND id < $2 AND bloom & b'%s'::bit(512) = bloom ORDER BY ID DESC LIMIT $3", toBitString(query.bloom)) args = append(args, query.start, query.cursor)
// If we have a cursor, we don't want to include that envelope in the result set
stmtString += " " + "WHERE id >= $1 AND id < $2"
} else { } else {
upperLimit = query.end args = append(args, query.start, query.end)
// We disable security checks as we need to use string interpolation stmtString += " " + "WHERE id >= $1 AND id <= $2"
// for this, but it's converted to 0s and 1s so no injection should be possible
/* #nosec */
stmtString = fmt.Sprintf("SELECT id, data FROM envelopes where id >= $1 AND id <= $2 AND bloom & b'%s'::bit(512) = bloom ORDER BY ID DESC LIMIT $3", toBitString(query.bloom))
} }
if len(query.topics) > 0 {
args = append(args, pq.Array(query.topics))
stmtString += " " + "AND topic = any($3)"
} else {
stmtString += " " + fmt.Sprintf("AND bloom & b'%s'::bit(512) = bloom", toBitString(query.bloom))
}
// Positional argument depends on the fact whether the query uses topics or bloom filter.
// If topic is used, the list of topics is passed as an argument to the query.
// If bloom filter is used, it is included into the query statement.
args = append(args, query.limit)
stmtString += " " + fmt.Sprintf("ORDER BY ID DESC LIMIT $%d", len(args))
stmt, err := i.db.Prepare(stmtString) stmt, err := i.db.Prepare(stmtString)
if err != nil { if err != nil {
return nil, err return nil, err
} }
rows, err := stmt.Query(args...)
rows, err := stmt.Query(query.start, upperLimit, query.limit)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &postgresIterator{rows}, nil return &postgresIterator{rows}, nil
} }
@ -185,12 +193,16 @@ func (i *PostgresDB) Prune(t time.Time, batch int) (int, error) {
func (i *PostgresDB) SaveEnvelope(env types.Envelope) error { func (i *PostgresDB) SaveEnvelope(env types.Envelope) error {
topic := env.Topic() topic := env.Topic()
key := NewDBKey(env.Expiry()-env.TTL(), topic, env.Hash()) key := NewDBKey(env.Expiry()-env.TTL(), topic, env.Hash())
rawEnvelope, err := rlp.EncodeToBytes(env) rawEnvelope, err := rlp.EncodeToBytes(env.Unwrap())
if err != nil { if err != nil {
log.Error(fmt.Sprintf("rlp.EncodeToBytes failed: %s", err)) log.Error(fmt.Sprintf("rlp.EncodeToBytes failed: %s", err))
archivedErrorsCounter.Inc() archivedErrorsCounter.Inc()
return err return err
} }
if rawEnvelope == nil {
archivedErrorsCounter.Inc()
return errors.New("failed to encode envelope to bytes")
}
statement := "INSERT INTO envelopes (id, data, topic, bloom) VALUES ($1, $2, $3, B'" statement := "INSERT INTO envelopes (id, data, topic, bloom) VALUES ($1, $2, $3, B'"
statement += toBitString(env.Bloom()) statement += toBitString(env.Bloom())

View File

@ -0,0 +1,112 @@
// +build postgres
// In order to run these tests, you must run a PostgreSQL database.
//
// Using Docker:
// docker run --name mailserver-db -e POSTGRES_USER=whisper -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_DB=whisper -d -p 5432:5432 postgres:9.6-alpine
//
package mailserver
import (
"testing"
"time"
"github.com/ethereum/go-ethereum/rlp"
gethbridge "github.com/status-im/status-go/eth-node/bridge/geth"
"github.com/status-im/status-go/eth-node/crypto"
"github.com/status-im/status-go/eth-node/types"
"github.com/status-im/status-go/whisper/v6"
"github.com/stretchr/testify/require"
)
func TestPostgresDB_BuildIteratorWithBloomFilter(t *testing.T) {
topic := []byte{0xaa, 0xbb, 0xcc, 0xdd}
db, err := NewPostgresDB("postgres://whisper:mysecretpassword@127.0.0.1:5432/whisper?sslmode=disable")
require.NoError(t, err)
envelope, err := newTestEnvelope(topic)
require.NoError(t, err)
err = db.SaveEnvelope(envelope)
require.NoError(t, err)
iter, err := db.BuildIterator(CursorQuery{
start: NewDBKey(uint32(time.Now().Add(-time.Hour).Unix()), types.BytesToTopic(topic), types.Hash{}).Bytes(),
end: NewDBKey(uint32(time.Now().Add(time.Second).Unix()), types.BytesToTopic(topic), types.Hash{}).Bytes(),
bloom: types.TopicToBloom(types.BytesToTopic(topic)),
limit: 10,
})
require.NoError(t, err)
hasNext := iter.Next()
require.True(t, hasNext)
rawValue, err := iter.GetEnvelope(nil)
require.NoError(t, err)
require.NotEmpty(t, rawValue)
var receivedEnvelope whisper.Envelope
err = rlp.DecodeBytes(rawValue, &receivedEnvelope)
require.NoError(t, err)
require.EqualValues(t, whisper.BytesToTopic(topic), receivedEnvelope.Topic)
err = iter.Release()
require.NoError(t, err)
require.NoError(t, iter.Error())
}
func TestPostgresDB_BuildIteratorWithTopic(t *testing.T) {
topic := []byte{0x01, 0x02, 0x03, 0x04}
db, err := NewPostgresDB("postgres://whisper:mysecretpassword@127.0.0.1:5432/whisper?sslmode=disable")
require.NoError(t, err)
envelope, err := newTestEnvelope(topic)
require.NoError(t, err)
err = db.SaveEnvelope(envelope)
require.NoError(t, err)
iter, err := db.BuildIterator(CursorQuery{
start: NewDBKey(uint32(time.Now().Add(-time.Hour).Unix()), types.BytesToTopic(topic), types.Hash{}).Bytes(),
end: NewDBKey(uint32(time.Now().Add(time.Second).Unix()), types.BytesToTopic(topic), types.Hash{}).Bytes(),
topics: [][]byte{topic},
limit: 10,
})
require.NoError(t, err)
hasNext := iter.Next()
require.True(t, hasNext)
rawValue, err := iter.GetEnvelope(nil)
require.NoError(t, err)
require.NotEmpty(t, rawValue)
var receivedEnvelope whisper.Envelope
err = rlp.DecodeBytes(rawValue, &receivedEnvelope)
require.NoError(t, err)
require.EqualValues(t, whisper.BytesToTopic(topic), receivedEnvelope.Topic)
err = iter.Release()
require.NoError(t, err)
require.NoError(t, iter.Error())
}
func newTestEnvelope(topic []byte) (types.Envelope, error) {
privateKey, err := crypto.GenerateKey()
if err != nil {
return nil, err
}
params := whisper.MessageParams{
TTL: 10,
PoW: 2.0,
Payload: []byte("hello world"),
WorkTime: 1,
Topic: whisper.BytesToTopic(topic),
Dst: &privateKey.PublicKey,
}
message, err := whisper.NewSentMessage(&params)
if err != nil {
return nil, err
}
now := time.Now()
envelope, err := message.Wrap(&params, now)
if err != nil {
return nil, err
}
return gethbridge.NewWhisperEnvelope(envelope), nil
}

View File

@ -440,6 +440,7 @@ func (s *MailserverSuite) TestDecodeRequest() {
Lower: 50, Lower: 50,
Upper: 100, Upper: 100,
Bloom: []byte{0x01}, Bloom: []byte{0x01},
Topics: [][]byte{},
Limit: 10, Limit: 10,
Cursor: []byte{}, Cursor: []byte{},
Batch: true, Batch: true,
@ -530,7 +531,7 @@ func (s *MailserverSuite) TestProcessRequestDeadlockHandling() {
processFinished := make(chan struct{}) processFinished := make(chan struct{})
go func() { go func() {
s.server.ms.processRequestInBundles(iter, payload.Bloom, int(payload.Limit), timeout, "req-01", bundles, done) s.server.ms.processRequestInBundles(iter, payload.Bloom, payload.Topics, int(payload.Limit), timeout, "req-01", bundles, done)
close(processFinished) close(processFinished)
}() }()
go close(done) go close(done)
@ -554,7 +555,7 @@ func (s *MailserverSuite) TestProcessRequestDeadlockHandling() {
processFinished := make(chan struct{}) processFinished := make(chan struct{})
go func() { go func() {
s.server.ms.processRequestInBundles(iter, payload.Bloom, int(payload.Limit), time.Second, "req-01", bundles, done) s.server.ms.processRequestInBundles(iter, payload.Bloom, payload.Topics, int(payload.Limit), time.Second, "req-01", bundles, done)
close(processFinished) close(processFinished)
}() }()
@ -572,7 +573,7 @@ func (s *MailserverSuite) TestProcessRequestDeadlockHandling() {
iter, err := s.server.ms.createIterator(payload) iter, err := s.server.ms.createIterator(payload)
s.Require().NoError(err) s.Require().NoError(err)
defer iter.Release() defer func() { _ = iter.Release() }()
// Nothing reads from this unbuffered channel which simulates a situation // Nothing reads from this unbuffered channel which simulates a situation
// when a connection between a peer and mail server was dropped. // when a connection between a peer and mail server was dropped.
@ -772,7 +773,7 @@ func generateEnvelope(sentTime time.Time) (*whisper.Envelope, error) {
func processRequestAndCollectHashes(server *WhisperMailServer, payload MessagesRequestPayload) ([]common.Hash, []byte, types.Hash) { func processRequestAndCollectHashes(server *WhisperMailServer, payload MessagesRequestPayload) ([]common.Hash, []byte, types.Hash) {
iter, _ := server.ms.createIterator(payload) iter, _ := server.ms.createIterator(payload)
defer iter.Release() defer func() { _ = iter.Release() }()
bundles := make(chan []rlp.RawValue, 10) bundles := make(chan []rlp.RawValue, 10)
done := make(chan struct{}) done := make(chan struct{})
@ -790,7 +791,7 @@ func processRequestAndCollectHashes(server *WhisperMailServer, payload MessagesR
close(done) close(done)
}() }()
cursor, lastHash := server.ms.processRequestInBundles(iter, payload.Bloom, int(payload.Limit), time.Minute, "req-01", bundles, done) cursor, lastHash := server.ms.processRequestInBundles(iter, payload.Bloom, payload.Topics, int(payload.Limit), time.Minute, "req-01", bundles, done)
<-done <-done

View File

@ -17,6 +17,8 @@ type MessagesRequestPayload struct {
Upper uint32 Upper uint32
// Bloom is a bloom filter to filter envelopes. // Bloom is a bloom filter to filter envelopes.
Bloom []byte Bloom []byte
// Topics is a list of topics to filter envelopes.
Topics [][]byte
// Limit is the max number of envelopes to return. // Limit is the max number of envelopes to return.
Limit uint32 Limit uint32
// Cursor is used for pagination of the results. // Cursor is used for pagination of the results.

View File

@ -75,8 +75,7 @@ func testMailserverPeer(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// register mail service as well // register mail service as well
err = n.Register(func(ctx *node.ServiceContext) (node.Service, error) { err = n.Register(func(ctx *node.ServiceContext) (node.Service, error) {
mailService := shhext.New(config, gethbridge.NewNodeBridge(n), ctx, nil, nil) return shhext.New(config, gethbridge.NewNodeBridge(n), ctx, nil, nil), nil
return mailService, nil
}) })
require.NoError(t, err) require.NoError(t, err)
var mailService *shhext.Service var mailService *shhext.Service

View File

@ -1,6 +1,9 @@
package gethbridge package gethbridge
import ( import (
"io"
"github.com/ethereum/go-ethereum/rlp"
"github.com/status-im/status-go/eth-node/types" "github.com/status-im/status-go/eth-node/types"
"github.com/status-im/status-go/waku" "github.com/status-im/status-go/waku"
"github.com/status-im/status-go/whisper/v6" "github.com/status-im/status-go/whisper/v6"
@ -15,15 +18,8 @@ func NewWhisperEnvelope(e *whisper.Envelope) types.Envelope {
return &whisperEnvelope{env: e} return &whisperEnvelope{env: e}
} }
func UnwrapWhisperEnvelope(e types.Envelope) (*whisper.Envelope, bool) { func (w *whisperEnvelope) Unwrap() interface{} {
if env, ok := e.(*whisperEnvelope); ok { return w.env
return env.env, true
}
return nil, false
}
func MustUnwrapWhisperEnvelope(e types.Envelope) *whisper.Envelope {
return e.(*whisperEnvelope).env
} }
func (w *whisperEnvelope) Hash() types.Hash { func (w *whisperEnvelope) Hash() types.Hash {
@ -54,6 +50,14 @@ func (w *whisperEnvelope) Size() int {
return len(w.env.Data) return len(w.env.Data)
} }
func (w *whisperEnvelope) DecodeRLP(s *rlp.Stream) error {
return w.env.DecodeRLP(s)
}
func (w *whisperEnvelope) EncodeRLP(writer io.Writer) error {
return rlp.Encode(writer, w.env)
}
type wakuEnvelope struct { type wakuEnvelope struct {
env *waku.Envelope env *waku.Envelope
} }
@ -63,15 +67,8 @@ func NewWakuEnvelope(e *waku.Envelope) types.Envelope {
return &wakuEnvelope{env: e} return &wakuEnvelope{env: e}
} }
func UnwrapWakuEnvelope(e types.Envelope) (*waku.Envelope, bool) { func (w *wakuEnvelope) Unwrap() interface{} {
if env, ok := e.(*wakuEnvelope); ok { return w.env
return env.env, true
}
return nil, false
}
func MustUnwrapWakuEnvelope(e types.Envelope) *waku.Envelope {
return e.(*wakuEnvelope).env
} }
func (w *wakuEnvelope) Hash() types.Hash { func (w *wakuEnvelope) Hash() types.Hash {
@ -101,3 +98,11 @@ func (w *wakuEnvelope) Topic() types.TopicType {
func (w *wakuEnvelope) Size() int { func (w *wakuEnvelope) Size() int {
return len(w.env.Data) return len(w.env.Data)
} }
func (w *wakuEnvelope) DecodeRLP(s *rlp.Stream) error {
return w.env.DecodeRLP(s)
}
func (w *wakuEnvelope) EncodeRLP(writer io.Writer) error {
return rlp.Encode(writer, w.env)
}

View File

@ -166,7 +166,7 @@ func (w *gethWakuWrapper) SendMessagesRequest(peerID []byte, r types.MessagesReq
// which are not supposed to be forwarded any further. // which are not supposed to be forwarded any further.
// The whisper protocol is agnostic of the format and contents of envelope. // The whisper protocol is agnostic of the format and contents of envelope.
func (w *gethWakuWrapper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope types.Envelope, timeout time.Duration) error { func (w *gethWakuWrapper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope types.Envelope, timeout time.Duration) error {
return w.waku.RequestHistoricMessagesWithTimeout(peerID, MustUnwrapWakuEnvelope(envelope), timeout) return w.waku.RequestHistoricMessagesWithTimeout(peerID, envelope.Unwrap().(*waku.Envelope), timeout)
} }
type wakuFilterWrapper struct { type wakuFilterWrapper struct {

View File

@ -171,7 +171,7 @@ func (w *gethWhisperWrapper) SendMessagesRequest(peerID []byte, r types.Messages
// which are not supposed to be forwarded any further. // which are not supposed to be forwarded any further.
// The whisper protocol is agnostic of the format and contents of envelope. // The whisper protocol is agnostic of the format and contents of envelope.
func (w *gethWhisperWrapper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope types.Envelope, timeout time.Duration) error { func (w *gethWhisperWrapper) RequestHistoricMessagesWithTimeout(peerID []byte, envelope types.Envelope, timeout time.Duration) error {
return w.whisper.RequestHistoricMessagesWithTimeout(peerID, MustUnwrapWhisperEnvelope(envelope), timeout) return w.whisper.RequestHistoricMessagesWithTimeout(peerID, envelope.Unwrap().(*whisper.Envelope), timeout)
} }
// SyncMessages can be sent between two Mail Servers and syncs envelopes between them. // SyncMessages can be sent between two Mail Servers and syncs envelopes between them.

View File

@ -3,7 +3,9 @@ package types
// Envelope represents a clear-text data packet to transmit through the Whisper // Envelope represents a clear-text data packet to transmit through the Whisper
// network. Its contents may or may not be encrypted and signed. // network. Its contents may or may not be encrypted and signed.
type Envelope interface { type Envelope interface {
Hash() Hash // Cached hash of the envelope to avoid rehashing every time. Wrapped
Hash() Hash // cached hash of the envelope to avoid rehashing every time
Bloom() []byte Bloom() []byte
PoW() float64 PoW() float64
Expiry() uint32 Expiry() uint32

View File

@ -0,0 +1,7 @@
package types
// Wrapped tells that a given object has an underlying representation
// and this representation can be accessed using `Unwrap` method.
type Wrapped interface {
Unwrap() interface{}
}

View File

@ -114,6 +114,10 @@ type MessagesRequest struct {
// Bloom is a filter to match requested messages. // Bloom is a filter to match requested messages.
Bloom []byte `json:"bloom"` Bloom []byte `json:"bloom"`
// Topics is a list of topics. A returned message should
// belong to one of the topics from the list.
Topics [][]byte `json:"topics"`
} }
func (r MessagesRequest) Validate() error { func (r MessagesRequest) Validate() error {
@ -129,8 +133,8 @@ func (r MessagesRequest) Validate() error {
return fmt.Errorf("invalid 'Limit' value, expected value lower than %d", MaxLimitInMessagesRequest) return fmt.Errorf("invalid 'Limit' value, expected value lower than %d", MaxLimitInMessagesRequest)
} }
if len(r.Bloom) == 0 { if len(r.Bloom) == 0 && len(r.Topics) == 0 {
return errors.New("invalid 'Bloom' provided") return errors.New("invalid 'Bloom' or 'Topics', one must be non-empty")
} }
return nil return nil

View File

@ -114,6 +114,10 @@ type MessagesRequest struct {
// Bloom is a filter to match requested messages. // Bloom is a filter to match requested messages.
Bloom []byte `json:"bloom"` Bloom []byte `json:"bloom"`
// Topics is a list of topics. A returned message should
// belong to one of the topics from the list.
Topics [][]byte `json:"topics"`
} }
func (r MessagesRequest) Validate() error { func (r MessagesRequest) Validate() error {
@ -129,8 +133,8 @@ func (r MessagesRequest) Validate() error {
return fmt.Errorf("invalid 'Limit' value, expected value lower than %d", MaxLimitInMessagesRequest) return fmt.Errorf("invalid 'Limit' value, expected value lower than %d", MaxLimitInMessagesRequest)
} }
if len(r.Bloom) == 0 { if len(r.Bloom) == 0 && len(r.Topics) == 0 {
return errors.New("invalid 'Bloom' provided") return errors.New("invalid 'Bloom' or 'Topics', one must be non-empty")
} }
return nil return nil