add known_durations column and use that for calculating duration averages, instead of total # pageviews

pkg/aggregator/aggregator.go

@@ -77,9 +77,9 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
 
 		site.Pageviews += 1
 
-		// TODO: Weight isn't right here because we need the number of pageview with a known time of page, not all pageviews
 		if p.Duration > 0.00 {
-			site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.Pageviews))
+			site.KnownDurations += 1
+			site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.KnownDurations))
 		}
 
 		if p.IsNewVisitor {
@@ -108,7 +108,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
 		}
 
 		if p.Duration > 0.00 {
-			pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.Pageviews))
+			pageStats.KnownDurations += 1
+			pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.KnownDurations))
 		}
 
 		if p.IsNewSession {
@@ -142,7 +143,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
 			}
 
 			if p.Duration > 0.00 {
-				referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.Pageviews))
+				referrerStats.KnownDurations += 1
+				referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.KnownDurations))
 			}
 
 		}

pkg/datastore/sqlstore/migrations/mysql/1_initial_tables.sql

@@ -54,6 +54,7 @@ CREATE UNIQUE INDEX unique_daily_site_stats ON daily_site_stats(date);
 CREATE UNIQUE INDEX unique_daily_page_stats ON daily_page_stats(hostname, pathname, date);
 CREATE UNIQUE INDEX unique_daily_referrer_stats ON daily_referrer_stats(url, date);
 
+
 -- +migrate Down
 
 DROP TABLE IF EXISTS users;

pkg/datastore/sqlstore/migrations/mysql/2_known_durations_column.sql

@@ -0,0 +1,13 @@
+-- +migrate Up
+
+ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+
+-- +migrate Down
+
+ALTER TABLE daily_site_stats DROP COLUMN known_durations;
+ALTER TABLE daily_page_stats DROP COLUMN known_durations;
+ALTER TABLE daily_referrer_stats DROP COLUMN known_durations;
+
+

pkg/datastore/sqlstore/migrations/postgres/2_known_durations_column.sql

@@ -0,0 +1,13 @@
+-- +migrate Up
+
+ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+
+-- +migrate Down
+
+ALTER TABLE daily_site_stats DROP COLUMN known_durations;
+ALTER TABLE daily_page_stats DROP COLUMN known_durations;
+ALTER TABLE daily_referrer_stats DROP COLUMN known_durations;
+
+

pkg/datastore/sqlstore/migrations/sqlite3/2_known_durations_column.sql

@@ -0,0 +1,13 @@
+-- +migrate Up
+
+ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
+
+-- +migrate Down
+
+ALTER TABLE daily_site_stats DROP COLUMN known_durations;
+ALTER TABLE daily_page_stats DROP COLUMN known_durations;
+ALTER TABLE daily_referrer_stats DROP COLUMN known_durations;
+
+

pkg/datastore/sqlstore/page_stats.go

@@ -17,14 +17,14 @@ func (db *sqlstore) GetPageStats(date time.Time, hostname string, pathname strin
 }
 
 func (db *sqlstore) InsertPageStats(s *models.PageStats) error {
-	query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
+	query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)`)
-	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
 	return err
 }
 
 func (db *sqlstore) UpdatePageStats(s *models.PageStats) error {
-	query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE hostname = ? AND pathname = ? AND date = ?`)
+	query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE hostname = ? AND pathname = ? AND date = ?`)
-	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
 	return err
 }
 

pkg/datastore/sqlstore/referrer_stats.go

@@ -18,14 +18,14 @@ func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.Referr
 }
 
 func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error {
-	query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, url, date) VALUES(?, ?, ?, ?, ?, ?)`)
+	query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, url, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
-	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
 	return err
 }
 
 func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error {
-	query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE url = ? AND date = ?`)
+	query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE url = ? AND date = ?`)
-	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
 	return err
 }
 

pkg/datastore/sqlstore/site_stats.go

@@ -17,14 +17,14 @@ func (db *sqlstore) GetSiteStats(date time.Time) (*models.SiteStats, error) {
 }
 
 func (db *sqlstore) InsertSiteStats(s *models.SiteStats) error {
-	query := db.Rebind(`INSERT INTO daily_site_stats(visitors, sessions, pageviews, bounce_rate, avg_duration, date) VALUES(?, ?, ?, ?, ?, ?)`)
+	query := db.Rebind(`INSERT INTO daily_site_stats(visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
-	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
 	return err
 }
 
 func (db *sqlstore) UpdateSiteStats(s *models.SiteStats) error {
-	query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE date = ?`)
+	query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE date = ?`)
-	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
 	return err
 }
 

pkg/datastore/sqlstore/sqlstore.go

@@ -38,7 +38,7 @@ func New(c *Config) *sqlstore {
 func (db *sqlstore) Migrate() {
 	migrationSource := &migrate.PackrMigrationSource{
 		Box: packr.NewBox("./migrations"),
-		Dir: "./" + db.Config.Driver,
+		Dir: db.Config.Driver,
 	}
 	migrate.SetTable("migrations")
 

pkg/models/page_stats.go

@@ -12,5 +12,6 @@ type PageStats struct {
 	Entries        int64     `db:"entries"`
 	BounceRate     float64   `db:"bounce_rate"`
 	AvgDuration    float64   `db:"avg_duration"`
+	KnownDurations int64     `db:"known_durations"`
 	Date           time.Time `db:"date" json:",omitempty"`
 }

pkg/models/referrer_stats.go

@@ -10,5 +10,6 @@ type ReferrerStats struct {
 	Pageviews      int64     `db:"pageviews"`
 	BounceRate     float64   `db:"bounce_rate"`
 	AvgDuration    float64   `db:"avg_duration"`
+	KnownDurations int64     `db:"known_durations"`
 	Date           time.Time `db:"date" json:",omitempty"`
 }

pkg/models/site_stats.go

@@ -10,5 +10,6 @@ type SiteStats struct {
 	Sessions       int64     `db:"sessions"`
 	BounceRate     float64   `db:"bounce_rate"`
 	AvgDuration    float64   `db:"avg_duration"`
+	KnownDurations int64     `db:"known_durations"`
 	Date           time.Time `db:"date" json:",omitempty"`
 }