2019-05-10 10:26:57 +00:00
|
|
|
package mailserver
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"github.com/ethereum/go-ethereum/common"
|
|
|
|
"github.com/ethereum/go-ethereum/log"
|
|
|
|
"github.com/ethereum/go-ethereum/rlp"
|
|
|
|
"github.com/status-im/status-go/params"
|
|
|
|
whisper "github.com/status-im/whisper/whisperv6"
|
|
|
|
"github.com/syndtr/goleveldb/leveldb"
|
|
|
|
"github.com/syndtr/goleveldb/leveldb/errors"
|
|
|
|
"github.com/syndtr/goleveldb/leveldb/iterator"
|
|
|
|
"github.com/syndtr/goleveldb/leveldb/util"
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
2019-05-13 09:25:46 +00:00
|
|
|
type LevelDB struct {
|
2019-05-10 10:26:57 +00:00
|
|
|
// We can't embed as there are some state problems with go-routines
|
|
|
|
ldb *leveldb.DB
|
|
|
|
}
|
|
|
|
|
|
|
|
type LevelDBIterator struct {
|
|
|
|
iterator.Iterator
|
|
|
|
}
|
|
|
|
|
2019-05-13 09:25:46 +00:00
|
|
|
func (i *LevelDBIterator) DBKey() (*DBKey, error) {
|
2019-05-10 10:26:57 +00:00
|
|
|
return &DBKey{
|
|
|
|
raw: i.Key(),
|
2019-05-13 09:25:46 +00:00
|
|
|
}, nil
|
2019-05-10 10:26:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (i *LevelDBIterator) GetEnvelope(bloom []byte) ([]byte, error) {
|
|
|
|
var envelopeBloom []byte
|
|
|
|
rawValue := make([]byte, len(i.Value()))
|
|
|
|
copy(rawValue, i.Value())
|
|
|
|
|
2019-05-13 09:25:46 +00:00
|
|
|
key, err := i.DBKey()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2019-05-10 10:26:57 +00:00
|
|
|
if len(key.Bytes()) != DBKeyLength {
|
|
|
|
var err error
|
|
|
|
envelopeBloom, err = extractBloomFromEncodedEnvelope(rawValue)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
envelopeBloom = whisper.TopicToBloom(key.Topic())
|
|
|
|
}
|
|
|
|
if !whisper.BloomFilterMatch(bloom, envelopeBloom) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
return rawValue, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2019-05-13 09:25:46 +00:00
|
|
|
func NewLevelDB(config *params.WhisperConfig) (*LevelDB, error) {
|
2019-05-10 10:26:57 +00:00
|
|
|
// Open opens an existing leveldb database
|
|
|
|
db, err := leveldb.OpenFile(config.DataDir, nil)
|
|
|
|
if _, iscorrupted := err.(*errors.ErrCorrupted); iscorrupted {
|
|
|
|
log.Info("database is corrupted trying to recover", "path", config.DataDir)
|
|
|
|
db, err = leveldb.RecoverFile(config.DataDir, nil)
|
|
|
|
}
|
2019-05-13 09:25:46 +00:00
|
|
|
return &LevelDB{ldb: db}, err
|
2019-05-10 10:26:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Build iterator returns an iterator given a start/end and a cursor
|
2019-05-13 09:25:46 +00:00
|
|
|
func (db *LevelDB) BuildIterator(query CursorQuery) (Iterator, error) {
|
2019-05-10 10:26:57 +00:00
|
|
|
defer recoverLevelDBPanics("BuildIterator")
|
|
|
|
|
|
|
|
i := db.ldb.NewIterator(&util.Range{Start: query.start, Limit: query.end}, nil)
|
|
|
|
// seek to the end as we want to return envelopes in a descending order
|
|
|
|
if len(query.cursor) == CursorLength {
|
|
|
|
i.Seek(query.cursor)
|
|
|
|
}
|
2019-05-13 09:25:46 +00:00
|
|
|
return &LevelDBIterator{i}, nil
|
2019-05-10 10:26:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// GetEnvelope get an envelope by its key
|
2019-05-13 09:25:46 +00:00
|
|
|
func (db *LevelDB) GetEnvelope(key *DBKey) ([]byte, error) {
|
2019-05-10 10:26:57 +00:00
|
|
|
defer recoverLevelDBPanics("GetEnvelope")
|
|
|
|
|
|
|
|
return db.ldb.Get(key.Bytes(), nil)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Prune removes envelopes older than time
|
2019-05-13 09:25:46 +00:00
|
|
|
func (db *LevelDB) Prune(t time.Time, batchSize int) (int, error) {
|
2019-05-10 10:26:57 +00:00
|
|
|
defer recoverLevelDBPanics("Prune")
|
|
|
|
|
|
|
|
var zero common.Hash
|
|
|
|
var emptyTopic whisper.TopicType
|
|
|
|
kl := NewDBKey(0, emptyTopic, zero)
|
|
|
|
ku := NewDBKey(uint32(t.Unix()), emptyTopic, zero)
|
|
|
|
query := CursorQuery{
|
|
|
|
start: kl.Bytes(),
|
|
|
|
end: ku.Bytes(),
|
|
|
|
}
|
2019-05-13 09:25:46 +00:00
|
|
|
i, err := db.BuildIterator(query)
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2019-05-10 10:26:57 +00:00
|
|
|
defer i.Release()
|
|
|
|
|
|
|
|
batch := leveldb.Batch{}
|
|
|
|
removed := 0
|
|
|
|
|
|
|
|
for i.Next() {
|
2019-05-13 09:25:46 +00:00
|
|
|
dbKey, err := i.DBKey()
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
|
|
|
|
batch.Delete(dbKey.Bytes())
|
2019-05-10 10:26:57 +00:00
|
|
|
|
|
|
|
if batch.Len() == batchSize {
|
|
|
|
if err := db.ldb.Write(&batch, nil); err != nil {
|
|
|
|
return removed, err
|
|
|
|
}
|
|
|
|
|
|
|
|
removed = removed + batch.Len()
|
|
|
|
batch.Reset()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if batch.Len() > 0 {
|
|
|
|
if err := db.ldb.Write(&batch, nil); err != nil {
|
|
|
|
return removed, err
|
|
|
|
}
|
|
|
|
|
|
|
|
removed = removed + batch.Len()
|
|
|
|
}
|
|
|
|
|
|
|
|
return removed, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// SaveEnvelope stores an envelope in leveldb and increments the metrics
|
2019-05-13 09:25:46 +00:00
|
|
|
func (db *LevelDB) SaveEnvelope(env *whisper.Envelope) error {
|
2019-05-10 10:26:57 +00:00
|
|
|
defer recoverLevelDBPanics("SaveEnvelope")
|
|
|
|
|
|
|
|
key := NewDBKey(env.Expiry-env.TTL, env.Topic, env.Hash())
|
|
|
|
rawEnvelope, err := rlp.EncodeToBytes(env)
|
|
|
|
if err != nil {
|
|
|
|
log.Error(fmt.Sprintf("rlp.EncodeToBytes failed: %s", err))
|
2019-10-22 17:32:05 +00:00
|
|
|
archivedErrorsCounter.Inc()
|
2019-05-10 10:26:57 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = db.ldb.Put(key.Bytes(), rawEnvelope, nil); err != nil {
|
|
|
|
log.Error(fmt.Sprintf("Writing to DB failed: %s", err))
|
2019-10-22 17:32:05 +00:00
|
|
|
archivedErrorsCounter.Inc()
|
2019-05-10 10:26:57 +00:00
|
|
|
}
|
2019-10-22 17:32:05 +00:00
|
|
|
archivedEnvelopesCounter.Inc()
|
|
|
|
archivedEnvelopeSizeMeter.Observe(float64(whisper.EnvelopeHeaderLength + len(env.Data)))
|
2019-05-10 10:26:57 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-05-13 09:25:46 +00:00
|
|
|
func (db *LevelDB) Close() error {
|
2019-05-10 10:26:57 +00:00
|
|
|
return db.ldb.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
func recoverLevelDBPanics(calleMethodName string) {
|
|
|
|
// Recover from possible goleveldb panics
|
|
|
|
if r := recover(); r != nil {
|
|
|
|
if errString, ok := r.(string); ok {
|
|
|
|
log.Error(fmt.Sprintf("recovered from panic in %s: %s", calleMethodName, errString))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|