mirror of
https://github.com/logos-messaging/storenode-messages-counter.git
synced 2026-01-02 14:13:11 +00:00
fix: filter by fleet
This commit is contained in:
Richard Ramos
parent
15d56b9396
commit
75b19fc872
@ -166,7 +166,7 @@ func Execute(ctx context.Context, options Options) error {
|
||||
case <-missingMessagesTimer.C:
|
||||
tmpUUID := uuid.New()
|
||||
runId := hex.EncodeToString(tmpUUID[:])
|
||||
runIdLogger := logger.With(zap.String("runId", runId))
|
||||
runIdLogger := logger.With(zap.String("runId", runId), zap.String("fleet", options.FleetName), zap.Uint("clusterID", options.ClusterID))
|
||||
|
||||
runIdLogger.Info("verifying message history...")
|
||||
shouldResetTimer, err := application.verifyHistory(ctx, runId, storenodeIDs, runIdLogger)
|
||||
@ -400,7 +400,7 @@ func (app *Application) checkMissingMessageStatus(ctx context.Context, storenode
|
||||
from := now.Add(-2 * time.Hour)
|
||||
to := now.Add(-time.Hour)
|
||||
|
||||
logger.Info("rechecking missing messages status", zap.Time("from", from), zap.Time("to", to), zap.Uint("clusterID", options.ClusterID))
|
||||
logger.Info("rechecking missing messages status", zap.Time("from", from), zap.Time("to", to))
|
||||
|
||||
// Get all messages whose status is missing or does not exist, and the column found_on_recheck is false
|
||||
// if found, set found_on_recheck to true
|
||||
|
||||
@ -121,15 +121,6 @@ func NewDBStore(clusterID uint, fleetName string, log *zap.Logger, options ...DB
|
||||
}
|
||||
}
|
||||
|
||||
_, err := result.db.Exec("update missingMessages set fleet = 'status.prod'")
|
||||
log.Info("RESULT", zap.Error(err))
|
||||
|
||||
result.db.Exec("update storeNodeUnavailable set fleet = 'status.prod'")
|
||||
log.Info("RESULT", zap.Error(err))
|
||||
|
||||
result.db.Exec("update syncTopicStatus set fleet = 'status.prod'")
|
||||
log.Info("RESULT", zap.Error(err))
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@ -225,8 +216,8 @@ func (d *DBStore) GetTopicSyncStatus(ctx context.Context, pubsubTopics []string)
|
||||
result[topic] = nil
|
||||
}
|
||||
|
||||
sqlQuery := `SELECT pubsubTopic, lastSyncTimestamp FROM syncTopicStatus WHERE clusterId = $1`
|
||||
rows, err := d.db.QueryContext(ctx, sqlQuery, d.clusterID)
|
||||
sqlQuery := `SELECT pubsubTopic, lastSyncTimestamp FROM syncTopicStatus WHERE fleet = $1 AND clusterId = $2`
|
||||
rows, err := d.db.QueryContext(ctx, sqlQuery, d.fleetName, d.clusterID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -254,7 +245,7 @@ func (d *DBStore) GetTopicSyncStatus(ctx context.Context, pubsubTopics []string)
|
||||
}
|
||||
|
||||
func (d *DBStore) GetMissingMessages(from time.Time, to time.Time) (map[peer.ID][]pb.MessageHash, error) {
|
||||
rows, err := d.db.Query("SELECT messageHash, storenode FROM missingMessages WHERE storedAt >= $1 AND storedAt <= $2 AND clusterId = $3 AND msgStatus = 'does_not_exist' AND foundOnRecheck = false", from.UnixNano(), to.UnixNano(), d.clusterID)
|
||||
rows, err := d.db.Query("SELECT messageHash, storenode FROM missingMessages WHERE storedAt >= $1 AND storedAt <= $2 AND clusterId = $3 AND fleet = $4 AND msgStatus = 'does_not_exist' AND foundOnRecheck = false", from.UnixNano(), to.UnixNano(), d.clusterID, d.fleetName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -288,7 +279,7 @@ func (d *DBStore) GetMissingMessages(from time.Time, to time.Time) (map[peer.ID]
|
||||
}
|
||||
|
||||
func (d *DBStore) UpdateTopicSyncState(tx *sql.Tx, topic string, lastSyncTimestamp time.Time) error {
|
||||
_, err := tx.Exec("INSERT INTO syncTopicStatus(fleet, clusterId, pubsubTopic, lastSyncTimestamp) VALUES ($1, $2, $3, $4) ON CONFLICT(clusterId, pubsubTopic) DO UPDATE SET lastSyncTimestamp = $5", d.fleetName, d.clusterID, topic, lastSyncTimestamp.UnixNano(), lastSyncTimestamp.UnixNano())
|
||||
_, err := tx.Exec("INSERT INTO syncTopicStatus(fleet, clusterId, pubsubTopic, lastSyncTimestamp) VALUES ($1, $2, $3, $4) ON CONFLICT(clusterId, pubsubTopic, fleet) DO UPDATE SET lastSyncTimestamp = $5", d.fleetName, d.clusterID, topic, lastSyncTimestamp.UnixNano(), lastSyncTimestamp.UnixNano())
|
||||
return err
|
||||
}
|
||||
|
||||
@ -315,16 +306,16 @@ func (d *DBStore) MarkMessagesAsFound(peerID peer.ID, messageHashes []pb.Message
|
||||
return nil
|
||||
}
|
||||
|
||||
query := "UPDATE missingMessages SET foundOnRecheck = true WHERE clusterID = $1 AND messageHash IN ("
|
||||
query := "UPDATE missingMessages SET foundOnRecheck = true WHERE fleet = $1 AND clusterID = $2 AND messageHash IN ("
|
||||
for i := range messageHashes {
|
||||
if i > 0 {
|
||||
query += ", "
|
||||
}
|
||||
query += fmt.Sprintf("$%d", i+2)
|
||||
query += fmt.Sprintf("$%d", i+3)
|
||||
}
|
||||
query += ")"
|
||||
|
||||
args := []interface{}{d.clusterID}
|
||||
args := []interface{}{d.fleetName, d.clusterID}
|
||||
for _, messageHash := range messageHashes {
|
||||
args = append(args, messageHash)
|
||||
}
|
||||
@ -340,7 +331,7 @@ func (d *DBStore) RecordStorenodeUnavailable(uuid string, storenode peer.ID) err
|
||||
}
|
||||
|
||||
func (d *DBStore) CountMissingMessages(from time.Time, to time.Time) (map[peer.ID]int, error) {
|
||||
rows, err := d.db.Query("SELECT storenode, count(1) as cnt FROM missingMessages WHERE storedAt >= $1 AND storedAt <= $2 AND clusterId = $3 AND msgStatus = 'does_not_exist' AND foundOnRecheck = false GROUP BY storenode", from.UnixNano(), to.UnixNano(), d.clusterID)
|
||||
rows, err := d.db.Query("SELECT storenode, count(1) as cnt FROM missingMessages WHERE storedAt >= $1 AND storedAt <= $2 AND clusterId = $3 AND fleet = $4 AND msgStatus = 'does_not_exist' AND foundOnRecheck = false GROUP BY storenode", from.UnixNano(), to.UnixNano(), d.clusterID, d.fleetName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -4,6 +4,8 @@
|
||||
// 2_timestamp.up.sql (53B)
|
||||
// 3_found.up.sql (86B)
|
||||
// 4_fleet.up.sql (158B)
|
||||
// 5_indexes.up.sql (779B)
|
||||
// TODO (33B)
|
||||
// doc.go (74B)
|
||||
|
||||
package migrations
|
||||
@ -153,6 +155,46 @@ func _4_fleetUpSql() (*asset, error) {
|
||||
return a, nil
|
||||
}
|
||||
|
||||
var __5_indexesUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x92\xc1\x52\x83\x30\x10\x86\xef\x7d\x8a\x3d\xd6\x19\x6e\x7a\xeb\x29\x42\x3a\x66\x2c\xd0\x09\x39\xb4\x27\x27\x40\x44\x46\x1a\x18\x36\x71\xec\xdb\x3b\xa9\x55\x4b\x9a\xca\x79\x77\xbf\x7c\xff\x6e\x16\x31\xa7\x44\x50\x60\x59\x42\x77\xc0\xd6\x90\xe5\x02\xe8\x8e\x15\xa2\x80\xb6\xfe\x4c\xb1\xb9\x87\x3c\x83\x43\x8b\xd8\xea\x26\x55\x88\xb2\x51\xb8\x44\xd3\x8f\xaa\x26\x06\x12\x5a\xc4\x11\xbc\x76\x4a\x99\xbb\xd5\x3c\xec\x21\x04\x3b\x4d\x47\x50\x75\x16\x8d\x1a\x59\x1d\xc1\xe1\xbb\xf4\x24\xf1\x6d\x96\xba\x76\xd3\x85\xf3\x71\xec\x93\x98\xee\x6b\x65\xb5\xfc\x90\x6d\x27\xcb\x4e\x2d\x7f\xf4\x16\x64\x23\x28\x07\x41\x1e\x37\x14\xf0\xa8\x2b\xd3\x0f\x6d\x85\x46\x1a\x8b\x90\xf0\x7c\x0b\x71\x9e\x15\x82\x13\x96\x09\xbf\xfe\x32\xbc\xab\xe3\xea\x5f\x02\x49\x92\x39\x00\x6c\x39\x4b\x09\xdf\xc3\x33\xdd\xc3\xf2\x22\xf1\x60\x4b\xb4\xa5\x70\xed\x7f\xeb\x9c\xbe\x16\x48\x76\x2d\x1d\x68\x0a\x99\x87\x58\xbe\xfe\x2d\xd4\x34\xc3\x68\xb5\xf3\xff\xed\xbe\x61\xef\xdd\xfc\x4a\xfc\x5c\x3f\x1f\x3e\xb4\x6d\x9f\xe0\xe9\x86\x00\x53\xd3\x8b\x4f\x15\xf4\xfd\x0a\x00\x00\xff\xff\x68\x16\x5f\x47\x0b\x03\x00\x00")
|
||||
|
||||
func _5_indexesUpSqlBytes() ([]byte, error) {
|
||||
return bindataRead(
|
||||
__5_indexesUpSql,
|
||||
"5_indexes.up.sql",
|
||||
)
|
||||
}
|
||||
|
||||
func _5_indexesUpSql() (*asset, error) {
|
||||
bytes, err := _5_indexesUpSqlBytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "5_indexes.up.sql", size: 779, mode: os.FileMode(0664), modTime: time.Unix(1724361674, 0)}
|
||||
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xc7, 0xa5, 0xe7, 0xbd, 0x88, 0x9, 0x6d, 0xaf, 0x1c, 0x69, 0x64, 0x21, 0xb4, 0x6d, 0xda, 0xdc, 0x52, 0x36, 0x78, 0x7a, 0xf2, 0x47, 0xdf, 0xab, 0xfa, 0x1d, 0x7e, 0xc4, 0xcd, 0xfd, 0x46, 0xf7}}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
var _todo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xe2\xe2\xe2\x32\xd1\x53\x70\x2c\xce\x56\x48\x49\x2d\xc8\xc9\xaf\xcc\x4d\xcd\x2b\x51\xc8\x4f\x53\x28\x2e\x49\x4c\xcf\xcc\x4b\xe7\xe2\x02\x04\x00\x00\xff\xff\xdf\x3f\x86\x24\x21\x00\x00\x00")
|
||||
|
||||
func todoBytes() ([]byte, error) {
|
||||
return bindataRead(
|
||||
_todo,
|
||||
"TODO",
|
||||
)
|
||||
}
|
||||
|
||||
func todo() (*asset, error) {
|
||||
bytes, err := todoBytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "TODO", size: 33, mode: os.FileMode(0664), modTime: time.Unix(1724361873, 0)}
|
||||
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x1a, 0x59, 0xaa, 0x5e, 0xd, 0xc8, 0xd5, 0x3e, 0xee, 0x93, 0x7d, 0x20, 0x7a, 0x67, 0x1a, 0x26, 0xeb, 0x18, 0xd3, 0x50, 0xf, 0x65, 0x23, 0x78, 0x50, 0x79, 0x4f, 0x34, 0x9, 0xcc, 0xe3, 0xb5}}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
var _docGo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x2c\xc9\xb1\x0d\xc4\x20\x0c\x05\xd0\x9e\x29\xfe\x02\xd8\xfd\x6d\xe3\x4b\xac\x2f\x44\x82\x09\x78\x7f\xa5\x49\xfd\xa6\x1d\xdd\xe8\xd8\xcf\x55\x8a\x2a\xe3\x47\x1f\xbe\x2c\x1d\x8c\xfa\x6f\xe3\xb4\x34\xd4\xd9\x89\xbb\x71\x59\xb6\x18\x1b\x35\x20\xa2\x9f\x0a\x03\xa2\xe5\x0d\x00\x00\xff\xff\x60\xcd\x06\xbe\x4a\x00\x00\x00")
|
||||
|
||||
func docGoBytes() ([]byte, error) {
|
||||
@ -272,6 +314,10 @@ var _bindata = map[string]func() (*asset, error){
|
||||
|
||||
"4_fleet.up.sql": _4_fleetUpSql,
|
||||
|
||||
"5_indexes.up.sql": _5_indexesUpSql,
|
||||
|
||||
"TODO": todo,
|
||||
|
||||
"doc.go": docGo,
|
||||
}
|
||||
|
||||
@ -320,6 +366,8 @@ var _bintree = &bintree{nil, map[string]*bintree{
|
||||
"2_timestamp.up.sql": &bintree{_2_timestampUpSql, map[string]*bintree{}},
|
||||
"3_found.up.sql": &bintree{_3_foundUpSql, map[string]*bintree{}},
|
||||
"4_fleet.up.sql": &bintree{_4_fleetUpSql, map[string]*bintree{}},
|
||||
"5_indexes.up.sql": &bintree{_5_indexesUpSql, map[string]*bintree{}},
|
||||
"TODO": &bintree{todo, map[string]*bintree{}},
|
||||
"doc.go": &bintree{docGo, map[string]*bintree{}},
|
||||
}}
|
||||
|
||||
|
||||
@ -0,0 +1,13 @@
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idxMsg3 ON missingMessages(storedAt DESC, fleet);
|
||||
CREATE INDEX IF NOT EXISTS idxMsg4 ON missingMessages(fleet, clusterId, messageHash);
|
||||
CREATE INDEX IF NOT EXISTS idxFleetStore ON storenodeunavailable(fleet);
|
||||
|
||||
ALTER TABLE synctopicstatus DROP CONSTRAINT synctopicstatus_pkey;
|
||||
ALTER TABLE synctopicstatus ADD CONSTRAINT synctopicstatus_pkey PRIMARY KEY (clusterId, pubsubTopic, fleet);
|
||||
|
||||
ALTER TABLE storenodeunavailable DROP CONSTRAINT storenodeunavailable_pkey;
|
||||
ALTER TABLE storenodeunavailable ADD CONSTRAINT storenodeunavailable_pkey PRIMARY KEY (runId, storenode, fleet);
|
||||
|
||||
ALTER TABLE missingMessages DROP CONSTRAINT missingmessages_pkey;
|
||||
ALTER TABLE missingMessages ADD CONSTRAINT missingmessages_pkey PRIMARY KEY (messageHash, storenode, fleet);
|
||||
Loading…
x
Reference in New Issue
Block a user