149 lines
3.2 KiB
Go
149 lines
3.2 KiB
Go
package dedup
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/suite"
|
|
"github.com/syndtr/goleveldb/leveldb"
|
|
"github.com/syndtr/goleveldb/leveldb/storage"
|
|
)
|
|
|
|
type dummyKeyPairProvider struct {
|
|
id string
|
|
}
|
|
|
|
func (p dummyKeyPairProvider) SelectedKeyPairID() string {
|
|
return p.id
|
|
}
|
|
|
|
func BenchmarkDeduplicate30000MessagesADay(b *testing.B) {
|
|
// using on-disk db here for real benchmarks
|
|
dir, err := ioutil.TempDir("", "dedup-30000")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
defer func() {
|
|
err := os.RemoveAll(dir)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}()
|
|
|
|
db, err := leveldb.OpenFile(dir, nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
d := NewDeduplicator(dummyKeyPairProvider{}, db)
|
|
|
|
b.Log("generating messages")
|
|
messagesOld := generateDedupMessages(100000)
|
|
b.Log("generation is done")
|
|
|
|
// pre-fill deduplicator
|
|
d.Deduplicate(messagesOld[:1000])
|
|
|
|
b.ResetTimer()
|
|
length := 300
|
|
start := 1000
|
|
for n := 0; n < b.N; n++ {
|
|
if n%100 == 0 {
|
|
d.cache.now = func() time.Time { return time.Now().Add(time.Duration(24*(n/100)) * time.Hour) }
|
|
}
|
|
if (start + length) >= len(messagesOld) {
|
|
start = 0
|
|
fmt.Println("cycle!")
|
|
}
|
|
messages := messagesOld[start:(start + length)]
|
|
start += length
|
|
dedupMessages := d.Deduplicate(messages)
|
|
ids := make([][]byte, len(dedupMessages))
|
|
for i, m := range dedupMessages {
|
|
ids[i] = m.Metadata.DedupID
|
|
}
|
|
assert.NoError(b, d.AddMessagesByID(ids))
|
|
}
|
|
}
|
|
|
|
func TestDeduplicatorTestSuite(t *testing.T) {
|
|
suite.Run(t, new(DeduplicatorTestSuite))
|
|
}
|
|
|
|
type DeduplicatorTestSuite struct {
|
|
suite.Suite
|
|
d *Deduplicator
|
|
db *leveldb.DB
|
|
}
|
|
|
|
func (s *DeduplicatorTestSuite) SetupTest() {
|
|
db, err := leveldb.Open(storage.NewMemStorage(), nil)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
s.db = db
|
|
s.d = NewDeduplicator(dummyKeyPairProvider{}, db)
|
|
}
|
|
|
|
func (s *DeduplicatorTestSuite) TearDownTest() {
|
|
s.NoError(s.db.Close())
|
|
}
|
|
|
|
func (s *DeduplicatorTestSuite) TestDeduplicateSingleFilter() {
|
|
s.d.keyPairProvider = dummyKeyPairProvider{"acc1"}
|
|
messages1 := generateDedupMessages(10)
|
|
messages2 := generateDedupMessages(12)
|
|
|
|
result := s.d.Deduplicate(messages1)
|
|
s.Equal(len(messages1), len(result))
|
|
|
|
ids := make([][]byte, len(result))
|
|
for i, m := range result {
|
|
ids[i] = m.Metadata.DedupID
|
|
}
|
|
s.NoError(s.d.AddMessagesByID(ids))
|
|
|
|
result = s.d.Deduplicate(messages1)
|
|
s.Equal(0, len(result))
|
|
|
|
result = s.d.Deduplicate(messages2)
|
|
s.Equal(len(messages2), len(result))
|
|
|
|
ids = make([][]byte, len(result))
|
|
for i, m := range result {
|
|
ids[i] = m.Metadata.DedupID
|
|
}
|
|
s.NoError(s.d.AddMessagesByID(ids))
|
|
|
|
messages3 := append(messages2, generateDedupMessages(11)...)
|
|
|
|
result = s.d.Deduplicate(messages3)
|
|
s.Equal(11, len(result))
|
|
}
|
|
|
|
func (s *DeduplicatorTestSuite) TestDeduplicateMultipleFilters() {
|
|
messages1 := generateDedupMessages(10)
|
|
|
|
s.d.keyPairProvider = dummyKeyPairProvider{"acc1"}
|
|
result := s.d.Deduplicate(messages1)
|
|
s.Equal(len(messages1), len(result))
|
|
ids := make([][]byte, len(result))
|
|
for i, m := range result {
|
|
ids[i] = m.Metadata.DedupID
|
|
}
|
|
|
|
s.NoError(s.d.AddMessagesByID(ids))
|
|
|
|
result = s.d.Deduplicate(messages1)
|
|
s.Equal(0, len(result))
|
|
|
|
s.d.keyPairProvider = dummyKeyPairProvider{"acc2"}
|
|
result = s.d.Deduplicate(messages1)
|
|
s.Equal(len(messages1), len(result))
|
|
}
|