diff --git a/pkg/api/collect.go b/pkg/api/collect.go index b26885e..bf35985 100644 --- a/pkg/api/collect.go +++ b/pkg/api/collect.go @@ -51,7 +51,6 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() now := time.Now() - // get pageview details pageview := &models.Pageview{ ID: q.Get("id"), SiteTrackingID: q.Get("sid"), @@ -62,6 +61,7 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) { IsUnique: q.Get("u") == "1", IsBounce: q.Get("b") != "0", Referrer: parseReferrer(q.Get("r")), + IsFinished: false, Duration: 0, Timestamp: now, } @@ -96,6 +96,7 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) { if previousPageview != nil && previousPageview.Timestamp.After(now.Add(-30*time.Minute)) { previousPageview.Duration = (now.Unix() - previousPageview.Timestamp.Unix()) previousPageview.IsBounce = false + previousPageview.IsFinished = true // push onto channel to be updated (in batch) later c.Pageviews <- previousPageview diff --git a/pkg/datastore/sqlstore/migrations/mysql/11_add_pageview_finished_column.sql b/pkg/datastore/sqlstore/migrations/mysql/11_add_pageview_finished_column.sql new file mode 100644 index 0000000..e2d49c5 --- /dev/null +++ b/pkg/datastore/sqlstore/migrations/mysql/11_add_pageview_finished_column.sql @@ -0,0 +1,7 @@ +-- +migrate Up + +ALTER TABLE pageviews ADD COLUMN is_finished TINYINT(1) NOT NULL DEFAULT 0; + +-- +migrate Down + +ALTER TABLE pageviews DROP COLUMN is_finished; diff --git a/pkg/datastore/sqlstore/migrations/postgres/12_add_pageview_finished_column.sql b/pkg/datastore/sqlstore/migrations/postgres/12_add_pageview_finished_column.sql new file mode 100644 index 0000000..6b79689 --- /dev/null +++ b/pkg/datastore/sqlstore/migrations/postgres/12_add_pageview_finished_column.sql @@ -0,0 +1,7 @@ +-- +migrate Up + +ALTER TABLE pageviews ADD COLUMN is_finished BOOLEAN NOT NULL DEFAULT FALSE; + +-- +migrate Down + +ALTER TABLE pageviews DROP COLUMN is_finished; diff --git a/pkg/datastore/sqlstore/migrations/sqlite3/11_add_pageview_finished_column.sql b/pkg/datastore/sqlstore/migrations/sqlite3/11_add_pageview_finished_column.sql new file mode 100644 index 0000000..7018ed7 --- /dev/null +++ b/pkg/datastore/sqlstore/migrations/sqlite3/11_add_pageview_finished_column.sql @@ -0,0 +1,34 @@ +-- +migrate Up + +DROP TABLE IF EXISTS pageviews; +CREATE TABLE pageviews( + id VARCHAR(31) NOT NULL, + site_tracking_id VARCHAR(8) NOT NULL, + hostname VARCHAR(255) NOT NULL, + pathname VARCHAR(255) NOT NULL, + is_new_visitor TINYINT(1) NOT NULL, + is_new_session TINYINT(1) NOT NULL, + is_unique TINYINT(1) NOT NULL, + is_bounce TINYINT(1) NULL, + is_finished TINYINT(1) NOT NULL DEFAULT 0, + referrer VARCHAR(255) NULL, + duration INTEGER(4) NULL, + timestamp DATETIME NOT NULL +); + +-- +migrate Down + +DROP TABLE IF EXISTS pageviews; +CREATE TABLE pageviews( + id VARCHAR(31) NOT NULL, + site_tracking_id VARCHAR(8) NOT NULL, + hostname VARCHAR(255) NOT NULL, + pathname VARCHAR(255) NOT NULL, + is_new_visitor TINYINT(1) NOT NULL, + is_new_session TINYINT(1) NOT NULL, + is_unique TINYINT(1) NOT NULL, + is_bounce TINYINT(1) NULL, + referrer VARCHAR(255) NULL, + duration INTEGER(4) NULL, + timestamp DATETIME NOT NULL +); \ No newline at end of file diff --git a/pkg/datastore/sqlstore/pageviews.go b/pkg/datastore/sqlstore/pageviews.go index 0488108..ba11b9f 100644 --- a/pkg/datastore/sqlstore/pageviews.go +++ b/pkg/datastore/sqlstore/pageviews.go @@ -77,14 +77,14 @@ func (db *sqlstore) UpdatePageviews(pageviews []*models.Pageview) error { return err } - query := tx.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`) + query := tx.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ?, is_finished = ? WHERE id = ?`) stmt, err := tx.Preparex(query) if err != nil { return err } for i := range pageviews { - _, err := stmt.Exec(pageviews[i].IsBounce, pageviews[i].Duration, pageviews[i].ID) + _, err := stmt.Exec(pageviews[i].IsBounce, pageviews[i].Duration, pageviews[i].IsFinished, pageviews[i].ID) if err != nil { tx.Rollback() @@ -100,8 +100,7 @@ func (db *sqlstore) UpdatePageviews(pageviews []*models.Pageview) error { func (db *sqlstore) GetProcessablePageviews() ([]*models.Pageview, error) { var results []*models.Pageview thirtyMinsAgo := time.Now().Add(-30 * time.Minute) - // We use FALSE here, even though SQLite has no BOOLEAN value. If it fails, maybe we can roll our own Rebind? - query := db.Rebind(`SELECT * FROM pageviews WHERE ( duration > 0 AND is_bounce = FALSE ) OR timestamp < ? LIMIT 500`) + query := db.Rebind(`SELECT * FROM pageviews WHERE is_finished = TRUE OR timestamp < ? LIMIT 5000`) err := db.Select(&results, query, thirtyMinsAgo) return results, err } diff --git a/pkg/datastore/sqlstore/site_stats.go b/pkg/datastore/sqlstore/site_stats.go index 824f664..8e21a48 100644 --- a/pkg/datastore/sqlstore/site_stats.go +++ b/pkg/datastore/sqlstore/site_stats.go @@ -113,9 +113,9 @@ func (db *sqlstore) GetRealtimeVisitorCount(siteID int64) (int64, error) { // for backwards compatibility with tracking snippets without an explicit site tracking ID (< 1.1.0) if siteID == 1 { - sql = `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR site_tracking_id = '' ) AND ( duration = 0 OR is_bounce = TRUE) AND timestamp > ?` + sql = `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR site_tracking_id = '' ) AND is_finished = FALSE AND timestamp > ?` } else { - sql = `SELECT COUNT(*) FROM pageviews p WHERE site_tracking_id = ? AND ( duration = 0 OR is_bounce = TRUE) AND timestamp > ?` + sql = `SELECT COUNT(*) FROM pageviews p WHERE site_tracking_id = ? AND is_finished = FALSE AND timestamp > ?` } query := db.Rebind(sql) diff --git a/pkg/models/pageview.go b/pkg/models/pageview.go index cd5a831..3d007ff 100644 --- a/pkg/models/pageview.go +++ b/pkg/models/pageview.go @@ -13,6 +13,7 @@ type Pageview struct { IsNewSession bool `db:"is_new_session"` IsUnique bool `db:"is_unique"` IsBounce bool `db:"is_bounce"` + IsFinished bool `db:"is_finished"` Referrer string `db:"referrer"` Duration int64 `db:"duration"` Timestamp time.Time `db:"timestamp"`