add is_finished column indicating whether a pageview is done (ready to be aggregated). closes #123

pkg/api/collect.go

@@ -51,7 +51,6 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	q := r.URL.Query()
 	now := time.Now()
 
-	// get pageview details
 	pageview := &models.Pageview{
 		ID:             q.Get("id"),
 		SiteTrackingID: q.Get("sid"),
@@ -62,6 +61,7 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		IsUnique:       q.Get("u") == "1",
 		IsBounce:       q.Get("b") != "0",
 		Referrer:       parseReferrer(q.Get("r")),
+		IsFinished:     false,
 		Duration:       0,
 		Timestamp:      now,
 	}
@@ -96,6 +96,7 @@ func (c *Collector) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if previousPageview != nil && previousPageview.Timestamp.After(now.Add(-30*time.Minute)) {
 			previousPageview.Duration = (now.Unix() - previousPageview.Timestamp.Unix())
 			previousPageview.IsBounce = false
+			previousPageview.IsFinished = true
 
 			// push onto channel to be updated (in batch) later
 			c.Pageviews <- previousPageview

pkg/datastore/sqlstore/migrations/mysql/11_add_pageview_finished_column.sql

@@ -0,0 +1,7 @@
+-- +migrate Up
+
+ALTER TABLE pageviews ADD COLUMN is_finished TINYINT(1) NOT NULL DEFAULT 0;
+
+-- +migrate Down
+
+ALTER TABLE pageviews DROP COLUMN is_finished;

pkg/datastore/sqlstore/migrations/postgres/12_add_pageview_finished_column.sql

@@ -0,0 +1,7 @@
+-- +migrate Up
+
+ALTER TABLE pageviews ADD COLUMN is_finished BOOLEAN NOT NULL DEFAULT FALSE;
+
+-- +migrate Down
+
+ALTER TABLE pageviews DROP COLUMN is_finished;

pkg/datastore/sqlstore/migrations/sqlite3/11_add_pageview_finished_column.sql

@@ -0,0 +1,34 @@
+-- +migrate Up
+
+DROP TABLE IF EXISTS pageviews;
+CREATE TABLE pageviews(
+   id VARCHAR(31) NOT NULL,
+   site_tracking_id VARCHAR(8) NOT NULL,
+   hostname VARCHAR(255) NOT NULL,
+   pathname VARCHAR(255) NOT NULL,
+   is_new_visitor TINYINT(1) NOT NULL,
+   is_new_session TINYINT(1) NOT NULL,
+   is_unique TINYINT(1) NOT NULL,
+   is_bounce TINYINT(1) NULL,
+   is_finished TINYINT(1) NOT NULL DEFAULT 0,
+   referrer VARCHAR(255) NULL,
+   duration INTEGER(4) NULL,
+   timestamp DATETIME NOT NULL
+);
+
+-- +migrate Down
+
+DROP TABLE IF EXISTS pageviews;
+CREATE TABLE pageviews(
+   id VARCHAR(31) NOT NULL,
+   site_tracking_id VARCHAR(8) NOT NULL,
+   hostname VARCHAR(255) NOT NULL,
+   pathname VARCHAR(255) NOT NULL,
+   is_new_visitor TINYINT(1) NOT NULL,
+   is_new_session TINYINT(1) NOT NULL,
+   is_unique TINYINT(1) NOT NULL,
+   is_bounce TINYINT(1) NULL,
+   referrer VARCHAR(255) NULL,
+   duration INTEGER(4) NULL,
+   timestamp DATETIME NOT NULL
+);

pkg/datastore/sqlstore/pageviews.go

@@ -77,14 +77,14 @@ func (db *sqlstore) UpdatePageviews(pageviews []*models.Pageview) error {
 		return err
 	}
 
-	query := tx.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`)
+	query := tx.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ?, is_finished = ? WHERE id = ?`)
 	stmt, err := tx.Preparex(query)
 	if err != nil {
 		return err
 	}
 
 	for i := range pageviews {
-		_, err := stmt.Exec(pageviews[i].IsBounce, pageviews[i].Duration, pageviews[i].ID)
+		_, err := stmt.Exec(pageviews[i].IsBounce, pageviews[i].Duration, pageviews[i].IsFinished, pageviews[i].ID)
 
 		if err != nil {
 			tx.Rollback()
@@ -100,8 +100,7 @@ func (db *sqlstore) UpdatePageviews(pageviews []*models.Pageview) error {
 func (db *sqlstore) GetProcessablePageviews() ([]*models.Pageview, error) {
 	var results []*models.Pageview
 	thirtyMinsAgo := time.Now().Add(-30 * time.Minute)
-	// We use FALSE here, even though SQLite has no BOOLEAN value. If it fails, maybe we can roll our own Rebind?
+	query := db.Rebind(`SELECT * FROM pageviews WHERE is_finished = TRUE OR timestamp < ? LIMIT 5000`)
-	query := db.Rebind(`SELECT * FROM pageviews WHERE ( duration > 0 AND is_bounce = FALSE ) OR timestamp < ? LIMIT 500`)
 	err := db.Select(&results, query, thirtyMinsAgo)
 	return results, err
 }

pkg/datastore/sqlstore/site_stats.go

@@ -113,9 +113,9 @@ func (db *sqlstore) GetRealtimeVisitorCount(siteID int64) (int64, error) {
 
 	// for backwards compatibility with tracking snippets without an explicit site tracking ID (< 1.1.0)
 	if siteID == 1 {
-		sql = `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR site_tracking_id = '' ) AND ( duration = 0 OR is_bounce = TRUE) AND  timestamp > ?`
+		sql = `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR site_tracking_id = '' ) AND is_finished = FALSE AND  timestamp > ?`
 	} else {
-		sql = `SELECT COUNT(*) FROM pageviews p WHERE site_tracking_id = ? AND ( duration = 0 OR is_bounce = TRUE) AND timestamp > ?`
+		sql = `SELECT COUNT(*) FROM pageviews p WHERE site_tracking_id = ? AND is_finished = FALSE AND timestamp > ?`
 	}
 
 	query := db.Rebind(sql)

pkg/models/pageview.go

@@ -13,6 +13,7 @@ type Pageview struct {
 	IsNewSession   bool      `db:"is_new_session"`
 	IsUnique       bool      `db:"is_unique"`
 	IsBounce       bool      `db:"is_bounce"`
+	IsFinished     bool      `db:"is_finished"`
 	Referrer       string    `db:"referrer"`
 	Duration       int64     `db:"duration"`
 	Timestamp      time.Time `db:"timestamp"`