Samuel Hawksby-Robinson 07e46714f0
Anon Metrics Broadcast (#2198)
* Protobufs and adapters

* Added basic anon metric service and config init

* Added fibonacci interval incrementer

* Added basic Client.Start func and integrated interval incrementer

* Added new processed field to app metrics table

* Added id column to app metrics table

* Added migration clean up

* Added appmetrics GetUnprocessed and SetToProcessedByIDs and tests

There was a wierd bug where metrics in the db that did not explicitly insert a  value would be NULL, so could not be found by . In addition I've added a new primary id field to the app_metrics table so that updates could be done against very specific metric rows.

* Updated adaptors and db to handle proto_id

I need a way to distinguish individual metric items from each other so that I can ignore the ones that have been seen before.

* Moved incrementer into dedicated file

* Resolve incrementer test fail

* Finalised the main loop functionality

* Implemented delete loop framework

* Updated adaptors file name

* Added delete loop delay and quit, and tweak on RawMessage gen

* Completed delete loop logic

* Added DBLock to prevent deletion during mainLoop

* Added postgres DB connection, integrated into anonmetrics.Server

* Removed proto_id from SQL migration and model

* Integrated postgres with Server and updated adaptors

* Function name update

* Added sample config files for client and server

* Fixes and testing for low level e2e

* make generate

* Fix lint

* Fix for receiving an anonMetricBatch not in server mode

* Postgres test fixes

* Tidy up, make vendor and make generate

* delinting

* Fixing database tests

* Attempted fix of does:  cannot open `does' (No such file or directory)
not:   cannot open `not' (No such file or directory)
exist: cannot open `exist' (No such file or directory) error on sql resource loas

* Moved all anon metric postgres migration logic and sources into a the protocol/anonmetrics package or sub packages. I don't know if this will fix the does:  cannot open `does' (No such file or directory)
not:   cannot open `not' (No such file or directory)
exist: cannot open `exist' (No such file or directory) error that happens in Jenkins but this could work

* Lint for the lint god

* Why doesn't the linter list all its problems at once?

* test tweaks

* Fix for wakuV2 change

* DB reset change

* Fix for postgres db migrations fails

* More robust implementation of postgres test setup and teardown

* Added block for anon metrics functionality

* Version Bump to 0.84.0

* Added test to check anon metrics broadcast is deactivated

* Protobufs and adapters

* Added basic anon metric service and config init

* Added new processed field to app metrics table

* Added id column to app metrics table

* Added migration clean up

* Added appmetrics GetUnprocessed and SetToProcessedByIDs and tests

There was a wierd bug where metrics in the db that did not explicitly insert a  value would be NULL, so could not be found by . In addition I've added a new primary id field to the app_metrics table so that updates could be done against very specific metric rows.

* Updated adaptors and db to handle proto_id

I need a way to distinguish individual metric items from each other so that I can ignore the ones that have been seen before.

* Added postgres DB connection, integrated into anonmetrics.Server

* Removed proto_id from SQL migration and model

* Integrated postgres with Server and updated adaptors

* Added sample config files for client and server

* Fix lint

* Fix for receiving an anonMetricBatch not in server mode

* Postgres test fixes

* Tidy up, make vendor and make generate

* Moved all anon metric postgres migration logic and sources into a the protocol/anonmetrics package or sub packages. I don't know if this will fix the does:  cannot open `does' (No such file or directory)
not:   cannot open `not' (No such file or directory)
exist: cannot open `exist' (No such file or directory) error that happens in Jenkins but this could work
2021-09-01 13:02:18 +01:00

90 lines
2.5 KiB
Go

package anonmetrics
import (
"crypto/ecdsa"
"fmt"
"github.com/davecgh/go-spew/spew"
"github.com/golang/protobuf/ptypes"
"github.com/status-im/status-go/appmetrics"
"github.com/status-im/status-go/eth-node/crypto"
"github.com/status-im/status-go/eth-node/types"
"github.com/status-im/status-go/protocol/protobuf"
)
// adaptProtoToModel is an adaptor helper function to convert a protobuf.AnonymousMetric into a appmetrics.AppMetric
func adaptProtoToModel(pbAnonMetric *protobuf.AnonymousMetric) (*appmetrics.AppMetric, error) {
t, err := ptypes.Timestamp(pbAnonMetric.CreatedAt)
if err != nil {
return nil, err
}
return &appmetrics.AppMetric{
MessageID: pbAnonMetric.Id,
Event: appmetrics.AppMetricEventType(pbAnonMetric.Event),
Value: pbAnonMetric.Value,
AppVersion: pbAnonMetric.AppVersion,
OS: pbAnonMetric.Os,
SessionID: pbAnonMetric.SessionId,
CreatedAt: t,
}, nil
}
// adaptModelToProto is an adaptor helper function to convert a appmetrics.AppMetric into a protobuf.AnonymousMetric
func adaptModelToProto(modelAnonMetric appmetrics.AppMetric, sendID *ecdsa.PublicKey) (*protobuf.AnonymousMetric, error) {
id := generateProtoID(modelAnonMetric, sendID)
createdAt, err := ptypes.TimestampProto(modelAnonMetric.CreatedAt)
if err != nil {
return nil, err
}
return &protobuf.AnonymousMetric{
Id: id,
Event: string(modelAnonMetric.Event),
Value: modelAnonMetric.Value,
AppVersion: modelAnonMetric.AppVersion,
Os: modelAnonMetric.OS,
SessionId: modelAnonMetric.SessionID,
CreatedAt: createdAt,
}, nil
}
func adaptModelsToProtoBatch(modelAnonMetrics []appmetrics.AppMetric, sendID *ecdsa.PublicKey) (*protobuf.AnonymousMetricBatch, error) {
amb := new(protobuf.AnonymousMetricBatch)
for _, m := range modelAnonMetrics {
p, err := adaptModelToProto(m, sendID)
if err != nil {
return nil, err
}
amb.Metrics = append(amb.Metrics, p)
}
return amb, nil
}
func adaptProtoBatchToModels(protoBatch protobuf.AnonymousMetricBatch) ([]*appmetrics.AppMetric, error) {
var ams []*appmetrics.AppMetric
for _, pm := range protoBatch.Metrics {
m, err := adaptProtoToModel(pm)
if err != nil {
return nil, err
}
ams = append(ams, m)
}
return ams, nil
}
// NEEDED because we don't send individual metrics, we send only batches
func generateProtoID(modelAnonMetric appmetrics.AppMetric, sendID *ecdsa.PublicKey) string {
return types.EncodeHex(crypto.Keccak256([]byte(fmt.Sprintf(
"%s%s",
types.EncodeHex(crypto.FromECDSAPub(sendID)),
spew.Sdump(modelAnonMetric)))))
}