diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go index ebdd0d6..936bc67 100644 --- a/pkg/aggregator/aggregator.go +++ b/pkg/aggregator/aggregator.go @@ -77,9 +77,9 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results { site.Pageviews += 1 - // TODO: Weight isn't right here because we need the number of pageview with a known time of page, not all pageviews if p.Duration > 0.00 { - site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.Pageviews)) + site.KnownDurations += 1 + site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.KnownDurations)) } if p.IsNewVisitor { @@ -108,7 +108,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results { } if p.Duration > 0.00 { - pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.Pageviews)) + pageStats.KnownDurations += 1 + pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.KnownDurations)) } if p.IsNewSession { @@ -142,7 +143,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results { } if p.Duration > 0.00 { - referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.Pageviews)) + referrerStats.KnownDurations += 1 + referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.KnownDurations)) } } diff --git a/pkg/datastore/sqlstore/migrations/mysql/1_initial_tables.sql b/pkg/datastore/sqlstore/migrations/mysql/1_initial_tables.sql index 3515058..29faa43 100644 --- a/pkg/datastore/sqlstore/migrations/mysql/1_initial_tables.sql +++ b/pkg/datastore/sqlstore/migrations/mysql/1_initial_tables.sql @@ -54,6 +54,7 @@ CREATE UNIQUE INDEX unique_daily_site_stats ON daily_site_stats(date); CREATE UNIQUE INDEX unique_daily_page_stats ON daily_page_stats(hostname, pathname, date); CREATE UNIQUE INDEX unique_daily_referrer_stats ON daily_referrer_stats(url, date); + -- +migrate Down DROP TABLE IF EXISTS users; diff --git a/pkg/datastore/sqlstore/migrations/mysql/2_known_durations_column.sql b/pkg/datastore/sqlstore/migrations/mysql/2_known_durations_column.sql new file mode 100644 index 0000000..e1e476b --- /dev/null +++ b/pkg/datastore/sqlstore/migrations/mysql/2_known_durations_column.sql @@ -0,0 +1,13 @@ +-- +migrate Up + +ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; +ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; +ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; + +-- +migrate Down + +ALTER TABLE daily_site_stats DROP COLUMN known_durations; +ALTER TABLE daily_page_stats DROP COLUMN known_durations; +ALTER TABLE daily_referrer_stats DROP COLUMN known_durations; + + diff --git a/pkg/datastore/sqlstore/migrations/postgres/2_known_durations_column.sql b/pkg/datastore/sqlstore/migrations/postgres/2_known_durations_column.sql new file mode 100644 index 0000000..e1e476b --- /dev/null +++ b/pkg/datastore/sqlstore/migrations/postgres/2_known_durations_column.sql @@ -0,0 +1,13 @@ +-- +migrate Up + +ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; +ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; +ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; + +-- +migrate Down + +ALTER TABLE daily_site_stats DROP COLUMN known_durations; +ALTER TABLE daily_page_stats DROP COLUMN known_durations; +ALTER TABLE daily_referrer_stats DROP COLUMN known_durations; + + diff --git a/pkg/datastore/sqlstore/migrations/sqlite3/2_known_durations_column.sql b/pkg/datastore/sqlstore/migrations/sqlite3/2_known_durations_column.sql new file mode 100644 index 0000000..e1e476b --- /dev/null +++ b/pkg/datastore/sqlstore/migrations/sqlite3/2_known_durations_column.sql @@ -0,0 +1,13 @@ +-- +migrate Up + +ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; +ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; +ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0; + +-- +migrate Down + +ALTER TABLE daily_site_stats DROP COLUMN known_durations; +ALTER TABLE daily_page_stats DROP COLUMN known_durations; +ALTER TABLE daily_referrer_stats DROP COLUMN known_durations; + + diff --git a/pkg/datastore/sqlstore/page_stats.go b/pkg/datastore/sqlstore/page_stats.go index 84c1567..e1d107f 100644 --- a/pkg/datastore/sqlstore/page_stats.go +++ b/pkg/datastore/sqlstore/page_stats.go @@ -17,14 +17,14 @@ func (db *sqlstore) GetPageStats(date time.Time, hostname string, pathname strin } func (db *sqlstore) InsertPageStats(s *models.PageStats) error { - query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`) - _, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) + query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)`) + _, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) return err } func (db *sqlstore) UpdatePageStats(s *models.PageStats) error { - query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE hostname = ? AND pathname = ? AND date = ?`) - _, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) + query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE hostname = ? AND pathname = ? AND date = ?`) + _, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) return err } diff --git a/pkg/datastore/sqlstore/referrer_stats.go b/pkg/datastore/sqlstore/referrer_stats.go index 4de2dfe..244cca6 100644 --- a/pkg/datastore/sqlstore/referrer_stats.go +++ b/pkg/datastore/sqlstore/referrer_stats.go @@ -18,14 +18,14 @@ func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.Referr } func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error { - query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, url, date) VALUES(?, ?, ?, ?, ?, ?)`) - _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02")) + query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, url, date) VALUES(?, ?, ?, ?, ?, ?, ?)`) + _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02")) return err } func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error { - query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE url = ? AND date = ?`) - _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02")) + query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE url = ? AND date = ?`) + _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02")) return err } diff --git a/pkg/datastore/sqlstore/site_stats.go b/pkg/datastore/sqlstore/site_stats.go index b1617f2..c2e7b80 100644 --- a/pkg/datastore/sqlstore/site_stats.go +++ b/pkg/datastore/sqlstore/site_stats.go @@ -17,14 +17,14 @@ func (db *sqlstore) GetSiteStats(date time.Time) (*models.SiteStats, error) { } func (db *sqlstore) InsertSiteStats(s *models.SiteStats) error { - query := db.Rebind(`INSERT INTO daily_site_stats(visitors, sessions, pageviews, bounce_rate, avg_duration, date) VALUES(?, ?, ?, ?, ?, ?)`) - _, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02")) + query := db.Rebind(`INSERT INTO daily_site_stats(visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?)`) + _, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02")) return err } func (db *sqlstore) UpdateSiteStats(s *models.SiteStats) error { - query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE date = ?`) - _, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02")) + query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE date = ?`) + _, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02")) return err } diff --git a/pkg/datastore/sqlstore/sqlstore.go b/pkg/datastore/sqlstore/sqlstore.go index ae0dd78..b77a98c 100644 --- a/pkg/datastore/sqlstore/sqlstore.go +++ b/pkg/datastore/sqlstore/sqlstore.go @@ -38,7 +38,7 @@ func New(c *Config) *sqlstore { func (db *sqlstore) Migrate() { migrationSource := &migrate.PackrMigrationSource{ Box: packr.NewBox("./migrations"), - Dir: "./" + db.Config.Driver, + Dir: db.Config.Driver, } migrate.SetTable("migrations") diff --git a/pkg/models/page_stats.go b/pkg/models/page_stats.go index 8dd3a8c..48d0554 100644 --- a/pkg/models/page_stats.go +++ b/pkg/models/page_stats.go @@ -5,12 +5,13 @@ import ( ) type PageStats struct { - Hostname string `db:"hostname"` - Pathname string `db:"pathname"` - Pageviews int64 `db:"pageviews"` - Visitors int64 `db:"visitors"` - Entries int64 `db:"entries"` - BounceRate float64 `db:"bounce_rate"` - AvgDuration float64 `db:"avg_duration"` - Date time.Time `db:"date" json:",omitempty"` + Hostname string `db:"hostname"` + Pathname string `db:"pathname"` + Pageviews int64 `db:"pageviews"` + Visitors int64 `db:"visitors"` + Entries int64 `db:"entries"` + BounceRate float64 `db:"bounce_rate"` + AvgDuration float64 `db:"avg_duration"` + KnownDurations int64 `db:"known_durations"` + Date time.Time `db:"date" json:",omitempty"` } diff --git a/pkg/models/referrer_stats.go b/pkg/models/referrer_stats.go index 8f28559..c107dc8 100644 --- a/pkg/models/referrer_stats.go +++ b/pkg/models/referrer_stats.go @@ -5,10 +5,11 @@ import ( ) type ReferrerStats struct { - URL string `db:"url"` - Visitors int64 `db:"visitors"` - Pageviews int64 `db:"pageviews"` - BounceRate float64 `db:"bounce_rate"` - AvgDuration float64 `db:"avg_duration"` - Date time.Time `db:"date" json:",omitempty"` + URL string `db:"url"` + Visitors int64 `db:"visitors"` + Pageviews int64 `db:"pageviews"` + BounceRate float64 `db:"bounce_rate"` + AvgDuration float64 `db:"avg_duration"` + KnownDurations int64 `db:"known_durations"` + Date time.Time `db:"date" json:",omitempty"` } diff --git a/pkg/models/site_stats.go b/pkg/models/site_stats.go index 3443ce1..044f3d7 100644 --- a/pkg/models/site_stats.go +++ b/pkg/models/site_stats.go @@ -5,10 +5,11 @@ import ( ) type SiteStats struct { - Visitors int64 `db:"visitors"` - Pageviews int64 `db:"pageviews"` - Sessions int64 `db:"sessions"` - BounceRate float64 `db:"bounce_rate"` - AvgDuration float64 `db:"avg_duration"` - Date time.Time `db:"date" json:",omitempty"` + Visitors int64 `db:"visitors"` + Pageviews int64 `db:"pageviews"` + Sessions int64 `db:"sessions"` + BounceRate float64 `db:"bounce_rate"` + AvgDuration float64 `db:"avg_duration"` + KnownDurations int64 `db:"known_durations"` + Date time.Time `db:"date" json:",omitempty"` }