status-im/fathom
2
0
Fork 0
mirror of https://github.com/status-im/fathom.git synced 2025-03-01 03:20:27 +00:00
Code Issues Projects Releases Wiki Activity

add is_finished column indicating whether a pageview is done (ready to be aggregated). closes #123

Browse Source
This commit is contained in:
Danny van Kooten 2018-10-10 10:27:55 +02:00
parent c877319046
commit 3c934b29fc
7 changed files with 56 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/api/collect.go
View File

@ -51,7 +51,6 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
now := time.Now()
// get pageview details
pageview := &models.Pageview{
ID: q.Get("id"),
SiteTrackingID: q.Get("sid"),
@ -62,6 +61,7 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
IsUnique: q.Get("u") == "1",
IsBounce: q.Get("b") != "0",
Referrer: parseReferrer(q.Get("r")),
IsFinished: false,
Duration: 0,
Timestamp: now,
}
@ -96,6 +96,7 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if previousPageview != nil && previousPageview.Timestamp.After(now.Add(-30*time.Minute)) {
previousPageview.Duration = (now.Unix() - previousPageview.Timestamp.Unix())
previousPageview.IsBounce = false
previousPageview.IsFinished = true
// push onto channel to be updated (in batch) later
c.Pageviews <- previousPageview

7
pkg/datastore/sqlstore/migrations/mysql/11_add_pageview_finished_column.sql Normal file
View File

@ -0,0 +1,7 @@
-- +migrate Up
ALTER TABLE pageviews ADD COLUMN is_finished TINYINT(1) NOT NULL DEFAULT 0;
-- +migrate Down
ALTER TABLE pageviews DROP COLUMN is_finished;

7
pkg/datastore/sqlstore/migrations/postgres/12_add_pageview_finished_column.sql Normal file
View File

@ -0,0 +1,7 @@
-- +migrate Up
ALTER TABLE pageviews ADD COLUMN is_finished BOOLEAN NOT NULL DEFAULT FALSE;
-- +migrate Down
ALTER TABLE pageviews DROP COLUMN is_finished;

34
pkg/datastore/sqlstore/migrations/sqlite3/11_add_pageview_finished_column.sql Normal file
View File

@ -0,0 +1,34 @@
-- +migrate Up
DROP TABLE IF EXISTS pageviews;
CREATE TABLE pageviews(
id VARCHAR(31) NOT NULL,
site_tracking_id VARCHAR(8) NOT NULL,
hostname VARCHAR(255) NOT NULL,
pathname VARCHAR(255) NOT NULL,
is_new_visitor TINYINT(1) NOT NULL,
is_new_session TINYINT(1) NOT NULL,
is_unique TINYINT(1) NOT NULL,
is_bounce TINYINT(1) NULL,
is_finished TINYINT(1) NOT NULL DEFAULT 0,
referrer VARCHAR(255) NULL,
duration INTEGER(4) NULL,
timestamp DATETIME NOT NULL
);
-- +migrate Down
DROP TABLE IF EXISTS pageviews;
CREATE TABLE pageviews(
id VARCHAR(31) NOT NULL,
site_tracking_id VARCHAR(8) NOT NULL,
hostname VARCHAR(255) NOT NULL,
pathname VARCHAR(255) NOT NULL,
is_new_visitor TINYINT(1) NOT NULL,
is_new_session TINYINT(1) NOT NULL,
is_unique TINYINT(1) NOT NULL,
is_bounce TINYINT(1) NULL,
referrer VARCHAR(255) NULL,
duration INTEGER(4) NULL,
timestamp DATETIME NOT NULL
);

7
pkg/datastore/sqlstore/pageviews.go
View File

@ -77,14 +77,14 @@ func (db *sqlstore) UpdatePageviews(pageviews []*models.Pageview) error {
return err
}
query := tx.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`)
query := tx.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ?, is_finished = ? WHERE id = ?`)
stmt, err := tx.Preparex(query)
if err != nil {
return err
}
for i := range pageviews {
_, err := stmt.Exec(pageviews[i].IsBounce, pageviews[i].Duration, pageviews[i].ID)
_, err := stmt.Exec(pageviews[i].IsBounce, pageviews[i].Duration, pageviews[i].IsFinished, pageviews[i].ID)
if err != nil {
tx.Rollback()
@ -100,8 +100,7 @@ func (db *sqlstore) UpdatePageviews(pageviews []*models.Pageview) error {
func (db *sqlstore) GetProcessablePageviews() ([]*models.Pageview, error) {
var results []*models.Pageview
thirtyMinsAgo := time.Now().Add(-30 * time.Minute)
// We use FALSE here, even though SQLite has no BOOLEAN value. If it fails, maybe we can roll our own Rebind?
query := db.Rebind(`SELECT * FROM pageviews WHERE ( duration > 0 AND is_bounce = FALSE ) OR timestamp < ? LIMIT 500`)
query := db.Rebind(`SELECT * FROM pageviews WHERE is_finished = TRUE OR timestamp < ? LIMIT 5000`)
err := db.Select(&results, query, thirtyMinsAgo)
return results, err
}

4
pkg/datastore/sqlstore/site_stats.go
View File

@ -113,9 +113,9 @@ func (db *sqlstore) GetRealtimeVisitorCount(siteID int64) (int64, error) {
// for backwards compatibility with tracking snippets without an explicit site tracking ID (< 1.1.0)
if siteID == 1 {
sql = `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR site_tracking_id = '' ) AND ( duration = 0 OR is_bounce = TRUE) AND timestamp > ?`
sql = `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR site_tracking_id = '' ) AND is_finished = FALSE AND timestamp > ?`
} else {
sql = `SELECT COUNT(*) FROM pageviews p WHERE site_tracking_id = ? AND ( duration = 0 OR is_bounce = TRUE) AND timestamp > ?`
sql = `SELECT COUNT(*) FROM pageviews p WHERE site_tracking_id = ? AND is_finished = FALSE AND timestamp > ?`
}
query := db.Rebind(sql)

1
pkg/models/pageview.go
View File

@ -13,6 +13,7 @@ type Pageview struct {
IsNewSession bool `db:"is_new_session"`
IsUnique bool `db:"is_unique"`
IsBounce bool `db:"is_bounce"`
IsFinished bool `db:"is_finished"`
Referrer string `db:"referrer"`
Duration int64 `db:"duration"`
Timestamp time.Time `db:"timestamp"`