get rid of session_id and move knowledge of previous pageview to client. this also gets rid of the (only) index on the pageviews table, allowing for much faster INSERT's. closes #14

assets/src/js/lib/util.js

@@ -11,12 +11,12 @@ function stringifyObject(json) {
       }).join('&');
 }
 
-function generateKey() {
+function randomString(n) {
   var s = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-  return Array(16).join().split(',').map(function() { return s.charAt(Math.floor(Math.random() * s.length)); }).join('');
+  return Array(n).join().split(',').map(() => s.charAt(Math.floor(Math.random() * s.length))).join('');
 }
 
 export { 
-   generateKey, 
+   randomString, 
    stringifyObject
 }

assets/src/js/tracker.js

@@ -13,10 +13,10 @@ const commands = {
 
 function newVisitorData() {
   return {
-    sid: util.generateKey(),
     isNewVisitor: true, 
     isNewSession: true,
     pagesViewed: [],
+    previousPageviewId: '',
     lastSeen: +new Date(),
   }
 }
@@ -97,7 +97,8 @@ function trackPageview() {
 
   let data = getData();
   const d = {
-    sid: data.sid,
+    id: util.randomString(20),
+    pid: data.previousPageviewId || '',
     p: path,
     h: hostname,
     r: referrer,
@@ -113,10 +114,11 @@ function trackPageview() {
     let midnight = new Date(Date.UTC(now.getFullYear(), now.getMonth(), now.getDate(), 24, 0, 0));
     let expires = Math.round((midnight - now) / 1000);
 
+    // update data in cookie
     if( data.pagesViewed.indexOf(path) == -1 ) {
       data.pagesViewed.push(path);
     }
-    
+    data.previousPageviewId = d.id;
     data.isNewVisitor = false;
     data.isNewSession = false;
     data.lastSeen = +new Date();

pkg/api/collect.go

@@ -77,7 +77,7 @@ func (api *API) NewCollectHandler() http.Handler {
 
 		// get pageview details
 		pageview := &models.Pageview{
-			SessionID:    q.Get("sid"),
+			ID:           q.Get("id"),
 			Hostname:     parseHostname(q.Get("h")),
 			Pathname:     parsePathname(q.Get("p")),
 			IsNewVisitor: q.Get("nv") == "1",
@@ -90,8 +90,9 @@ func (api *API) NewCollectHandler() http.Handler {
 		}
 
 		// find previous pageview by same visitor
-		if !pageview.IsNewSession {
+		previousPageviewID := q.Get("pid")
-			previousPageview, err := api.database.GetMostRecentPageviewBySessionID(pageview.SessionID)
+		if !pageview.IsNewSession && previousPageviewID != "" {
+			previousPageview, err := api.database.GetPageview(previousPageviewID)
 			if err != nil && err != datastore.ErrNoResults {
 				return err
 			}

pkg/datastore/datastore.go

@@ -32,7 +32,7 @@ type Datastore interface {
 	// pageviews
 	SavePageview(*models.Pageview) error
 	UpdatePageview(*models.Pageview) error
-	GetMostRecentPageviewBySessionID(string) (*models.Pageview, error)
+	GetPageview(string) (*models.Pageview, error)
 	GetProcessablePageviews() ([]*models.Pageview, error)
 	DeletePageviews([]*models.Pageview) error
 

pkg/datastore/sqlstore/migrations/mysql/4_pageview_id_column.sql

@@ -0,0 +1,14 @@
+-- +migrate Up
+
+ALTER TABLE pageviews DROP COLUMN session_id;
+ALTER TABLE pageviews DROP COLUMN id;
+ALTER TABLE pageviews ADD COLUMN id VARCHAR(31) NOT NULL FIRST;
+
+-- +migrate Down
+
+ALTER TABLE pageviews DROP COLUMN id;
+ALTER TABLE pageviews ADD COLUMN id INT AUTO_INCREMENT PRIMARY KEY NOT NULL FIRST;
+ALTER TABLE pageviews ADD COLUMN session_id VARCHAR(16) NOT NULL AFTER id;
+
+
+

pkg/datastore/sqlstore/migrations/postgres/4_pageview_id_column.sql

@@ -0,0 +1,15 @@
+-- +migrate Up
+
+TRUNCATE pageviews; -- postgres will fail because of NULL values otherwise
+ALTER TABLE pageviews DROP COLUMN session_id;
+ALTER TABLE pageviews DROP COLUMN id;
+ALTER TABLE pageviews ADD COLUMN id VARCHAR(31) NOT NULL;
+
+-- +migrate Down
+
+ALTER TABLE pageviews DROP COLUMN id;
+ALTER TABLE pageviews ADD COLUMN id INT AUTO_INCREMENT PRIMARY KEY NOT NULL;
+ALTER TABLE pageviews ADD COLUMN session_id VARCHAR(16) NOT NULL;
+
+
+

pkg/datastore/sqlstore/migrations/sqlite3/4_pageview_id_column.sql

@@ -0,0 +1,32 @@
+-- +migrate Up
+
+DROP TABLE pageviews;
+CREATE TABLE pageviews(
+   id VARCHAR(31) NOT NULL,
+   hostname VARCHAR(255) NOT NULL,
+   pathname VARCHAR(255) NOT NULL,
+   is_new_visitor TINYINT(1) NOT NULL,
+   is_new_session TINYINT(1) NOT NULL,
+   is_unique TINYINT(1) NOT NULL,
+   is_bounce TINYINT(1) NULL,
+   referrer VARCHAR(255) NULL,
+   duration INTEGER(4) NULL,
+   timestamp DATETIME NOT NULL
+);
+
+-- +migrate Down
+
+DROP TABLE pageviews;
+CREATE TABLE pageviews(
+   id INTEGER PRIMARY KEY,
+   hostname VARCHAR(255) NOT NULL,
+   pathname VARCHAR(255) NOT NULL,
+   session_id VARCHAR(16) NOT NULL,
+   is_new_visitor TINYINT(1) NOT NULL,
+   is_new_session TINYINT(1) NOT NULL,
+   is_unique TINYINT(1) NOT NULL,
+   is_bounce TINYINT(1) NULL,
+   referrer VARCHAR(255) NULL,
+   duration INTEGER(4) NULL,
+   timestamp DATETIME NOT NULL
+);

pkg/datastore/sqlstore/pageviews.go

@@ -2,7 +2,6 @@ package sqlstore
 
 import (
 	"database/sql"
-	"strconv"
 	"strings"
 	"time"
 
@@ -11,26 +10,21 @@ import (
 
 // SavePageview inserts a single pageview model into the connected database
 func (db *sqlstore) SavePageview(p *models.Pageview) error {
-	query := db.Rebind(`INSERT INTO pageviews(hostname, pathname, session_id, is_new_visitor, is_new_session, is_unique, is_bounce, referrer, duration, timestamp) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+	query := db.Rebind(`INSERT INTO pageviews(id, hostname, pathname, is_new_visitor, is_new_session, is_unique, is_bounce, referrer, duration, timestamp) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
-	result, err := db.Exec(query, p.Hostname, p.Pathname, p.SessionID, p.IsNewVisitor, p.IsNewSession, p.IsUnique, p.IsBounce, p.Referrer, p.Duration, p.Timestamp)
+	_, err := db.Exec(query, p.ID, p.Hostname, p.Pathname, p.IsNewVisitor, p.IsNewSession, p.IsUnique, p.IsBounce, p.Referrer, p.Duration, p.Timestamp)
 	if err != nil {
 		return err
 	}
 
-	p.ID, _ = result.LastInsertId()
 	return nil
 }
 
-func (db *sqlstore) UpdatePageview(p *models.Pageview) error {
+// GetPageview selects a single pageview by its string ID
-	query := db.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`)
+func (db *sqlstore) GetPageview(id string) (*models.Pageview, error) {
-	_, err := db.Exec(query, p.IsBounce, p.Duration, p.ID)
-	return err
-}
-
-func (db *sqlstore) GetMostRecentPageviewBySessionID(sessionID string) (*models.Pageview, error) {
 	result := &models.Pageview{}
-	query := db.Rebind(`SELECT * FROM pageviews WHERE session_id = ? ORDER BY id DESC LIMIT 1`)
+	query := db.Rebind(`SELECT * FROM pageviews WHERE id = ? LIMIT 1`)
-	err := db.Get(result, query, sessionID)
+	err := db.Get(result, query, id)
+
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, ErrNoResults
@@ -42,6 +36,13 @@ func (db *sqlstore) GetMostRecentPageviewBySessionID(sessionID string) (*models.
 	return result, nil
 }
 
+func (db *sqlstore) UpdatePageview(p *models.Pageview) error {
+	query := db.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`)
+	_, err := db.Exec(query, p.IsBounce, p.Duration, p.ID)
+	return err
+}
+
+// GetProcessablePageviews selects all pageviews which are "done" (ie not still waiting for bounce flag or duration)
 func (db *sqlstore) GetProcessablePageviews() ([]*models.Pageview, error) {
 	var results []*models.Pageview
 	thirtyMinsAgo := time.Now().Add(-30 * time.Minute)
@@ -54,7 +55,7 @@ func (db *sqlstore) GetProcessablePageviews() ([]*models.Pageview, error) {
 func (db *sqlstore) DeletePageviews(pageviews []*models.Pageview) error {
 	ids := []string{}
 	for _, p := range pageviews {
-		ids = append(ids, strconv.FormatInt(p.ID, 10))
+		ids = append(ids, "'"+p.ID+"'")
 	}
 	query := db.Rebind(`DELETE FROM pageviews WHERE id IN(` + strings.Join(ids, ",") + `)`)
 	_, err := db.Exec(query)

pkg/datastore/sqlstore/site_stats.go

@@ -93,7 +93,7 @@ func (db *sqlstore) GetAverageSiteBounceRate(startDate time.Time, endDate time.T
 }
 
 func (db *sqlstore) GetRealtimeVisitorCount() (int, error) {
-	sql := `SELECT COUNT(DISTINCT(session_id)) FROM pageviews WHERE timestamp > ?`
+	sql := `SELECT COUNT(*) FROM pageviews WHERE ( duration = 0 OR is_bounce = TRUE ) AND  timestamp > ?`
 	query := db.Rebind(sql)
 	var total int
 	err := db.Get(&total, query, time.Now().Add(-5*time.Minute))

pkg/models/pageview.go

@@ -5,8 +5,7 @@ import (
 )
 
 type Pageview struct {
-	ID           int64     `db:"id"`
+	ID           string    `db:"id"`
-	SessionID    string    `db:"session_id"`
 	Hostname     string    `db:"hostname"`
 	Pathname     string    `db:"pathname"`
 	IsNewVisitor bool      `db:"is_new_visitor"`