2018-04-20 11:26:54 +00:00
|
|
|
package dedup
|
|
|
|
|
|
|
|
import (
|
|
|
|
"github.com/ethereum/go-ethereum/log"
|
2019-08-20 11:20:25 +00:00
|
|
|
|
2019-10-09 14:22:53 +00:00
|
|
|
whispertypes "github.com/status-im/status-protocol-go/transport/whisper/types"
|
|
|
|
statusproto "github.com/status-im/status-protocol-go/types"
|
2018-04-20 11:26:54 +00:00
|
|
|
"github.com/syndtr/goleveldb/leveldb"
|
|
|
|
)
|
|
|
|
|
|
|
|
type keyPairProvider interface {
|
|
|
|
SelectedKeyPairID() string
|
|
|
|
}
|
|
|
|
|
|
|
|
// Deduplicator filters out already received messages for a current filter.
|
|
|
|
// It keeps a limited cache of the messages.
|
|
|
|
type Deduplicator struct {
|
|
|
|
keyPairProvider keyPairProvider
|
|
|
|
cache *cache
|
|
|
|
log log.Logger
|
|
|
|
}
|
|
|
|
|
2019-09-26 07:01:17 +00:00
|
|
|
type Author struct {
|
2019-10-09 14:22:53 +00:00
|
|
|
PublicKey statusproto.HexBytes `json:"publicKey"`
|
|
|
|
Alias string `json:"alias"`
|
|
|
|
Identicon string `json:"identicon"`
|
2019-09-26 07:01:17 +00:00
|
|
|
}
|
|
|
|
|
2019-08-20 11:20:25 +00:00
|
|
|
type Metadata struct {
|
2019-10-09 14:22:53 +00:00
|
|
|
DedupID []byte `json:"dedupId"`
|
|
|
|
EncryptionID statusproto.HexBytes `json:"encryptionId"`
|
|
|
|
MessageID statusproto.HexBytes `json:"messageId"`
|
|
|
|
Author Author `json:"author"`
|
2019-08-20 11:20:25 +00:00
|
|
|
}
|
|
|
|
|
2019-02-19 12:58:42 +00:00
|
|
|
type DeduplicateMessage struct {
|
2019-10-09 14:22:53 +00:00
|
|
|
Message *whispertypes.Message `json:"message"`
|
|
|
|
Metadata Metadata `json:"metadata"`
|
2019-02-19 12:58:42 +00:00
|
|
|
}
|
|
|
|
|
2018-04-20 11:26:54 +00:00
|
|
|
// NewDeduplicator creates a new deduplicator
|
2018-05-02 12:14:08 +00:00
|
|
|
func NewDeduplicator(keyPairProvider keyPairProvider, db *leveldb.DB) *Deduplicator {
|
2018-04-20 11:26:54 +00:00
|
|
|
return &Deduplicator{
|
|
|
|
log: log.New("package", "status-go/services/sshext.deduplicator"),
|
|
|
|
keyPairProvider: keyPairProvider,
|
2018-05-02 12:14:08 +00:00
|
|
|
cache: newCache(db),
|
2018-04-20 11:26:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Deduplicate receives a list of whisper messages and
|
|
|
|
// returns the list of the messages that weren't filtered previously for the
|
|
|
|
// specified filter.
|
2019-08-20 11:20:25 +00:00
|
|
|
func (d *Deduplicator) Deduplicate(messages []*DeduplicateMessage) []*DeduplicateMessage {
|
|
|
|
result := make([]*DeduplicateMessage, 0)
|
2019-02-19 12:58:42 +00:00
|
|
|
selectedKeyPairID := d.keyPairProvider.SelectedKeyPairID()
|
2018-04-20 11:26:54 +00:00
|
|
|
|
|
|
|
for _, message := range messages {
|
2019-08-20 11:20:25 +00:00
|
|
|
if has, err := d.cache.Has(selectedKeyPairID, message.Message); !has {
|
2018-04-20 11:26:54 +00:00
|
|
|
if err != nil {
|
|
|
|
d.log.Error("error while deduplicating messages: search cache failed", "err", err)
|
|
|
|
}
|
2019-08-20 11:20:25 +00:00
|
|
|
message.Metadata.DedupID = d.cache.KeyToday(selectedKeyPairID, message.Message)
|
|
|
|
result = append(result, message)
|
2018-04-20 11:26:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return result
|
|
|
|
}
|
2018-05-02 12:14:08 +00:00
|
|
|
|
|
|
|
// AddMessages adds a message to the deduplicator DB, so it will be filtered
|
|
|
|
// out.
|
2019-08-20 11:20:25 +00:00
|
|
|
func (d *Deduplicator) AddMessagesByID(messageIDs [][]byte) error {
|
|
|
|
return d.cache.PutIDs(messageIDs)
|
2018-05-02 12:14:08 +00:00
|
|
|
}
|
2019-02-19 12:58:42 +00:00
|
|
|
|
|
|
|
// AddMessageByID adds a message to the deduplicator DB, so it will be filtered
|
|
|
|
// out.
|
|
|
|
func (d *Deduplicator) AddMessageByID(messageIDs [][]byte) error {
|
|
|
|
return d.cache.PutIDs(messageIDs)
|
|
|
|
}
|