feat: add garbage collection for soft deleted bookmarks
This introduces a simple garbage collection which checks for all soft deleted bookmarks (`removed = 1`) which have been marked for garbage collection more than 30 days ago from the time of bootstapping the messenger. Closes #2705
This commit is contained in:
Pascal Precht
r4bbit.eth
parent
f3af6aa4dc
commit
c855272340
|
|
@ -24,6 +24,7 @@
|
|||
// 1655385721_drop_networks_config.up.sql (27B)
|
||||
// 1655385724_networks_chainColor_shortName.up.sql (220B)
|
||||
// 1655456688_add_deleted_at_field_to_bookmarks_table.up.sql (69B)
|
||||
// 1655462032_create_bookmarks_deleted_at_index.up.sql (81B)
|
||||
// doc.go (74B)
|
||||
|
||||
package migrations
|
||||
|
|
@ -508,7 +509,7 @@ func _1655375270_add_clock_field_to_communities_settings_tableUpSql() (*asset, e
|
|||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "1655375270_add_clock_field_to_communities_settings_table.up.sql", size: 74, mode: os.FileMode(0664), modTime: time.Unix(1655379619, 0)}
|
||||
info := bindataFileInfo{name: "1655375270_add_clock_field_to_communities_settings_table.up.sql", size: 74, mode: os.FileMode(0664), modTime: time.Unix(1655462002, 0)}
|
||||
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x19, 0xc5, 0xc0, 0xf9, 0x84, 0x53, 0xdf, 0x83, 0xcf, 0xb6, 0x40, 0x6d, 0xf5, 0xdc, 0x77, 0x37, 0xb7, 0xe3, 0xa, 0x75, 0xe7, 0x6, 0x11, 0xca, 0x2b, 0x51, 0x92, 0xdd, 0x7d, 0xdb, 0xc3, 0xf5}}
|
||||
return a, nil
|
||||
}
|
||||
|
|
@ -528,7 +529,7 @@ func _1655385721_drop_networks_configUpSql() (*asset, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "1655385721_drop_networks_config.up.sql", size: 27, mode: os.FileMode(0664), modTime: time.Unix(1655456669, 0)}
|
||||
info := bindataFileInfo{name: "1655385721_drop_networks_config.up.sql", size: 27, mode: os.FileMode(0664), modTime: time.Unix(1655462002, 0)}
|
||||
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xfc, 0xa7, 0x20, 0xbb, 0x67, 0x21, 0xe, 0xc6, 0xc8, 0x21, 0x74, 0xe0, 0xce, 0xc8, 0xe2, 0x2, 0xb4, 0xea, 0xf0, 0xe5, 0xc4, 0x4d, 0xdd, 0xd4, 0x52, 0x31, 0xa9, 0x3d, 0xcd, 0xd8, 0x9b, 0xab}}
|
||||
return a, nil
|
||||
}
|
||||
|
|
@ -548,7 +549,7 @@ func _1655385724_networks_chaincolor_shortnameUpSql() (*asset, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "1655385724_networks_chainColor_shortName.up.sql", size: 220, mode: os.FileMode(0664), modTime: time.Unix(1655456669, 0)}
|
||||
info := bindataFileInfo{name: "1655385724_networks_chainColor_shortName.up.sql", size: 220, mode: os.FileMode(0664), modTime: time.Unix(1655462002, 0)}
|
||||
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd9, 0xe7, 0x84, 0xbb, 0x5f, 0xd2, 0x2c, 0x42, 0x88, 0x62, 0x52, 0xb6, 0x58, 0x31, 0xac, 0xc, 0x96, 0x2b, 0x1b, 0xe5, 0x4e, 0x9a, 0x3a, 0xf6, 0xf6, 0xfc, 0xa9, 0x1a, 0x35, 0x62, 0x28, 0x88}}
|
||||
return a, nil
|
||||
}
|
||||
|
|
@ -568,11 +569,31 @@ func _1655456688_add_deleted_at_field_to_bookmarks_tableUpSql() (*asset, error)
|
|||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "1655456688_add_deleted_at_field_to_bookmarks_table.up.sql", size: 69, mode: os.FileMode(0664), modTime: time.Unix(1655456719, 0)}
|
||||
info := bindataFileInfo{name: "1655456688_add_deleted_at_field_to_bookmarks_table.up.sql", size: 69, mode: os.FileMode(0664), modTime: time.Unix(1655462002, 0)}
|
||||
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe7, 0x9a, 0xbd, 0x9a, 0xc9, 0xf, 0xdf, 0x90, 0x0, 0x5d, 0xea, 0x6e, 0x7d, 0x51, 0x95, 0xcd, 0x90, 0xd3, 0x1a, 0x36, 0x6c, 0xf4, 0xbd, 0xa7, 0x6b, 0xbf, 0xe5, 0xdb, 0xa3, 0x88, 0xe3, 0x50}}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
var __1655462032_create_bookmarks_deleted_at_indexUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x72\x0e\x72\x75\x0c\x71\x55\xf0\xf4\x73\x71\x8d\x50\x48\x49\xcd\x49\x2d\x49\x4d\x89\x4f\x2c\x89\x4f\xca\xcf\xcf\xce\x4d\x2c\xca\x2e\x56\xf0\xf7\x53\x40\x70\x34\x10\x4a\x34\x15\xc2\x3d\x5c\x83\x5c\x51\x84\xac\xb9\xb8\x00\x01\x00\x00\xff\xff\xb9\x08\x04\xda\x51\x00\x00\x00")
|
||||
|
||||
func _1655462032_create_bookmarks_deleted_at_indexUpSqlBytes() ([]byte, error) {
|
||||
return bindataRead(
|
||||
__1655462032_create_bookmarks_deleted_at_indexUpSql,
|
||||
"1655462032_create_bookmarks_deleted_at_index.up.sql",
|
||||
)
|
||||
}
|
||||
|
||||
func _1655462032_create_bookmarks_deleted_at_indexUpSql() (*asset, error) {
|
||||
bytes, err := _1655462032_create_bookmarks_deleted_at_indexUpSqlBytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "1655462032_create_bookmarks_deleted_at_index.up.sql", size: 81, mode: os.FileMode(0664), modTime: time.Unix(1655462091, 0)}
|
||||
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xf, 0x8e, 0x20, 0x6b, 0x14, 0x9e, 0xcd, 0x97, 0xd3, 0xfe, 0x62, 0x3, 0x26, 0x59, 0x1, 0x6c, 0x99, 0xef, 0x6d, 0x21, 0xd4, 0xb5, 0xa3, 0xf4, 0x39, 0x40, 0x54, 0x6, 0xd, 0x60, 0x13, 0x38}}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
var _docGo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x2c\xc9\xb1\x0d\xc4\x20\x0c\x05\xd0\x9e\x29\xfe\x02\xd8\xfd\x6d\xe3\x4b\xac\x2f\x44\x82\x09\x78\x7f\xa5\x49\xfd\xa6\x1d\xdd\xe8\xd8\xcf\x55\x8a\x2a\xe3\x47\x1f\xbe\x2c\x1d\x8c\xfa\x6f\xe3\xb4\x34\xd4\xd9\x89\xbb\x71\x59\xb6\x18\x1b\x35\x20\xa2\x9f\x0a\x03\xa2\xe5\x0d\x00\x00\xff\xff\x60\xcd\x06\xbe\x4a\x00\x00\x00")
|
||||
|
||||
func docGoBytes() ([]byte, error) {
|
||||
|
|
@ -732,6 +753,8 @@ var _bindata = map[string]func() (*asset, error){
|
|||
|
||||
"1655456688_add_deleted_at_field_to_bookmarks_table.up.sql": _1655456688_add_deleted_at_field_to_bookmarks_tableUpSql,
|
||||
|
||||
"1655462032_create_bookmarks_deleted_at_index.up.sql": _1655462032_create_bookmarks_deleted_at_indexUpSql,
|
||||
|
||||
"doc.go": docGo,
|
||||
}
|
||||
|
||||
|
|
@ -800,6 +823,7 @@ var _bintree = &bintree{nil, map[string]*bintree{
|
|||
"1655385721_drop_networks_config.up.sql": &bintree{_1655385721_drop_networks_configUpSql, map[string]*bintree{}},
|
||||
"1655385724_networks_chainColor_shortName.up.sql": &bintree{_1655385724_networks_chaincolor_shortnameUpSql, map[string]*bintree{}},
|
||||
"1655456688_add_deleted_at_field_to_bookmarks_table.up.sql": &bintree{_1655456688_add_deleted_at_field_to_bookmarks_tableUpSql, map[string]*bintree{}},
|
||||
"1655462032_create_bookmarks_deleted_at_index.up.sql": &bintree{_1655462032_create_bookmarks_deleted_at_indexUpSql, map[string]*bintree{}},
|
||||
"doc.go": &bintree{docGo, map[string]*bintree{}},
|
||||
}}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
CREATE INDEX deleted_at_bookmarks ON bookmarks (deleted_at) WHERE (deleted_at);
|
||||
|
||||
2
go.mod
2
go.mod
|
|
@ -71,7 +71,7 @@ require (
|
|||
go.uber.org/zap v1.21.0
|
||||
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
|
||||
golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb
|
||||
google.golang.org/protobuf v1.27.1
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
|
||||
gopkg.in/go-playground/validator.v9 v9.31.0
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0
|
||||
|
|
|
|||
|
|
@ -683,6 +683,11 @@ func (m *Messenger) Start() (*MessengerResponse, error) {
|
|||
}
|
||||
}
|
||||
|
||||
err = m.GarbageCollectRemovedBookmarks()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -38,3 +38,7 @@ func (m *Messenger) UpdateBookmark(ctx context.Context, oldURL string, bookmark
|
|||
}
|
||||
return m.SyncBookmark(ctx, &bookmark)
|
||||
}
|
||||
|
||||
func (m *Messenger) GarbageCollectRemovedBookmarks() error {
|
||||
return m.persistence.DeleteSoftRemovedBookmarks(uint64(time.Now().AddDate(0, 0, -30).Unix()))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"crypto/ecdsa"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
gethbridge "github.com/status-im/status-go/eth-node/bridge/geth"
|
||||
"github.com/status-im/status-go/eth-node/crypto"
|
||||
|
|
@ -154,3 +155,53 @@ func (s *MessengerSyncBookmarkSuite) TestSyncBookmark() {
|
|||
s.Require().NoError(theirMessenger.Shutdown())
|
||||
|
||||
}
|
||||
|
||||
func (s *MessengerSyncBookmarkSuite) TestGarbageCollectRemovedBookmarks() {
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Create bookmarks that are flagged as deleted for more than 30 days
|
||||
bookmark1 := browsers.Bookmark{
|
||||
Name: "status official site",
|
||||
URL: "https://status.im",
|
||||
Removed: true,
|
||||
DeletedAt: uint64(now.AddDate(0, 0, -31).Unix()),
|
||||
}
|
||||
|
||||
bookmark2 := browsers.Bookmark{
|
||||
Name: "Uniswap",
|
||||
URL: "https://uniswap.org",
|
||||
Removed: true,
|
||||
DeletedAt: uint64(now.AddDate(0, 0, -31).Unix()),
|
||||
}
|
||||
|
||||
// This one is flagged for deletion less than 30 days
|
||||
bookmark3 := browsers.Bookmark{
|
||||
Name: "Maker DAO",
|
||||
URL: "https://makerdao.com",
|
||||
Removed: true,
|
||||
DeletedAt: uint64(now.AddDate(0, 0, -29).Unix()),
|
||||
}
|
||||
|
||||
// Store bookmarks
|
||||
_, err := s.m.browserDatabase.StoreBookmark(bookmark1)
|
||||
s.Require().NoError(err)
|
||||
|
||||
_, err = s.m.browserDatabase.StoreBookmark(bookmark2)
|
||||
s.Require().NoError(err)
|
||||
|
||||
_, err = s.m.browserDatabase.StoreBookmark(bookmark3)
|
||||
s.Require().NoError(err)
|
||||
|
||||
bookmarks, err := s.m.browserDatabase.GetBookmarks()
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(bookmarks, 3)
|
||||
|
||||
// err = s.m.GarbageCollectRemovedBookmarks(&now)
|
||||
err = s.m.GarbageCollectRemovedBookmarks()
|
||||
s.Require().NoError(err)
|
||||
|
||||
bookmarks, err = s.m.browserDatabase.GetBookmarks()
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(bookmarks, 1)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1081,3 +1081,19 @@ func (db *sqlitePersistence) UpdateBookmark(oldURL string, bookmark browsers.Boo
|
|||
_, err = insert.Exec(bookmark.URL, bookmark.Name, bookmark.ImageURL, bookmark.Removed, bookmark.Clock, bookmark.DeletedAt, oldURL)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *sqlitePersistence) DeleteSoftRemovedBookmarks(threshold uint64) error {
|
||||
tx, err := db.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err == nil {
|
||||
err = tx.Commit()
|
||||
return
|
||||
}
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
_, err = tx.Exec(`DELETE from bookmarks WHERE removed = 1 AND deleted_at < ?`, threshold)
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ type Bookmark struct {
|
|||
ImageURL string `json:"imageUrl"`
|
||||
Removed bool `json:"removed"`
|
||||
Clock uint64 `json:"-"` //used to sync
|
||||
DeletedAt uint64 `json:"deletedAt"`
|
||||
DeletedAt uint64 `json:"deletedAt,omitempty"`
|
||||
}
|
||||
|
||||
func (db *Database) GetBookmarks() ([]*Bookmark, error) {
|
||||
|
|
@ -161,7 +161,7 @@ func (db *Database) GetBookmarks() ([]*Bookmark, error) {
|
|||
}
|
||||
|
||||
func (db *Database) StoreBookmark(bookmark Bookmark) (Bookmark, error) {
|
||||
insert, err := db.db.Prepare("INSERT OR REPLACE INTO bookmarks (url, name, image_url, removed, clock) VALUES (?, ?, ?, ?, ?)")
|
||||
insert, err := db.db.Prepare("INSERT OR REPLACE INTO bookmarks (url, name, image_url, removed, clock, deleted_at) VALUES (?, ?, ?, ?, ?, ?)")
|
||||
|
||||
if err != nil {
|
||||
return bookmark, err
|
||||
|
|
@ -182,7 +182,7 @@ func (db *Database) StoreBookmark(bookmark Bookmark) (Bookmark, error) {
|
|||
log.Error("error getting the bookmark icon", "iconError", iconError)
|
||||
}
|
||||
|
||||
_, err = insert.Exec(bookmark.URL, bookmark.Name, bookmark.ImageURL, bookmark.Removed, bookmark.Clock)
|
||||
_, err = insert.Exec(bookmark.URL, bookmark.Name, bookmark.ImageURL, bookmark.Removed, bookmark.Clock, bookmark.DeletedAt)
|
||||
return bookmark, err
|
||||
}
|
||||
|
||||
|
|
@ -202,7 +202,7 @@ func (db *Database) StoreBookmarkWithoutFetchIcon(bookmark *Bookmark, tx *sql.Tx
|
|||
}()
|
||||
}
|
||||
|
||||
insert, err := tx.Prepare("INSERT OR REPLACE INTO bookmarks (url, name, image_url, removed, clock) VALUES (?, ?, ?, ?, ?)")
|
||||
insert, err := tx.Prepare("INSERT OR REPLACE INTO bookmarks (url, name, image_url, removed, clock, deleted_at) VALUES (?, ?, ?, ?, ?, ?)")
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -210,7 +210,7 @@ func (db *Database) StoreBookmarkWithoutFetchIcon(bookmark *Bookmark, tx *sql.Tx
|
|||
|
||||
defer insert.Close()
|
||||
|
||||
_, err = insert.Exec(bookmark.URL, bookmark.Name, bookmark.ImageURL, bookmark.Removed, bookmark.Clock)
|
||||
_, err = insert.Exec(bookmark.URL, bookmark.Name, bookmark.ImageURL, bookmark.Removed, bookmark.Clock, bookmark.DeletedAt)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue