status-im
/
fathom
mirror of https://github.com/status-im/fathom.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

add known_durations column and use that for calculating duration averages, instead of total # pageviews

This commit is contained in:
Danny 2018-05-28 15:31:40 +02:00
parent 4cf8f029af
commit df7496a80d
12 changed files with 82 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/aggregator/aggregator.go
View File

@ -77,9 +77,9 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
site.Pageviews += 1 site.Pageviews += 1
// TODO: Weight isn't right here because we need the number of pageview with a known time of page, not all pageviews
if p.Duration > 0.00 { if p.Duration > 0.00 {
site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.Pageviews)) site.KnownDurations += 1
site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.KnownDurations))
} }
if p.IsNewVisitor { if p.IsNewVisitor {
@ -108,7 +108,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
} }
if p.Duration > 0.00 { if p.Duration > 0.00 {
pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.Pageviews)) pageStats.KnownDurations += 1
pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.KnownDurations))
} }
if p.IsNewSession { if p.IsNewSession {
@ -142,7 +143,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
} }
if p.Duration > 0.00 { if p.Duration > 0.00 {
referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.Pageviews)) referrerStats.KnownDurations += 1
referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.KnownDurations))
} }
} }

1
pkg/datastore/sqlstore/migrations/mysql/1_initial_tables.sql
View File

@ -54,6 +54,7 @@ CREATE UNIQUE INDEX unique_daily_site_stats ON daily_site_stats(date);
CREATE UNIQUE INDEX unique_daily_page_stats ON daily_page_stats(hostname, pathname, date); CREATE UNIQUE INDEX unique_daily_page_stats ON daily_page_stats(hostname, pathname, date);
CREATE UNIQUE INDEX unique_daily_referrer_stats ON daily_referrer_stats(url, date); CREATE UNIQUE INDEX unique_daily_referrer_stats ON daily_referrer_stats(url, date);
-- +migrate Down -- +migrate Down
DROP TABLE IF EXISTS users; DROP TABLE IF EXISTS users;

13
pkg/datastore/sqlstore/migrations/mysql/2_known_durations_column.sql Normal file
View File

@ -0,0 +1,13 @@
-- +migrate Up
ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
-- +migrate Down
ALTER TABLE daily_site_stats DROP COLUMN known_durations;
ALTER TABLE daily_page_stats DROP COLUMN known_durations;
ALTER TABLE daily_referrer_stats DROP COLUMN known_durations;

13
pkg/datastore/sqlstore/migrations/postgres/2_known_durations_column.sql Normal file
View File

@ -0,0 +1,13 @@
-- +migrate Up
ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
-- +migrate Down
ALTER TABLE daily_site_stats DROP COLUMN known_durations;
ALTER TABLE daily_page_stats DROP COLUMN known_durations;
ALTER TABLE daily_referrer_stats DROP COLUMN known_durations;

13
pkg/datastore/sqlstore/migrations/sqlite3/2_known_durations_column.sql Normal file
View File

@ -0,0 +1,13 @@
-- +migrate Up
ALTER TABLE daily_site_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
ALTER TABLE daily_page_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
ALTER TABLE daily_referrer_stats ADD COLUMN known_durations INTEGER NOT NULL DEFAULT 0;
-- +migrate Down
ALTER TABLE daily_site_stats DROP COLUMN known_durations;
ALTER TABLE daily_page_stats DROP COLUMN known_durations;
ALTER TABLE daily_referrer_stats DROP COLUMN known_durations;

8
pkg/datastore/sqlstore/page_stats.go
View File

@ -17,14 +17,14 @@ func (db *sqlstore) GetPageStats(date time.Time, hostname string, pathname strin
} }
func (db *sqlstore) InsertPageStats(s *models.PageStats) error { func (db *sqlstore) InsertPageStats(s *models.PageStats) error {
query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`) query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) _, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err return err
} }
func (db *sqlstore) UpdatePageStats(s *models.PageStats) error { func (db *sqlstore) UpdatePageStats(s *models.PageStats) error {
query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE hostname = ? AND pathname = ? AND date = ?`) query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE hostname = ? AND pathname = ? AND date = ?`)
_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) _, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err return err
} }

8
pkg/datastore/sqlstore/referrer_stats.go
View File

@ -18,14 +18,14 @@ func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.Referr
} }
func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error { func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error {
query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, url, date) VALUES(?, ?, ?, ?, ?, ?)`) query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, url, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02")) _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
return err return err
} }
func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error { func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error {
query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE url = ? AND date = ?`) query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE url = ? AND date = ?`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02")) _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
return err return err
} }

8
pkg/datastore/sqlstore/site_stats.go
View File

@ -17,14 +17,14 @@ func (db *sqlstore) GetSiteStats(date time.Time) (*models.SiteStats, error) {
} }
func (db *sqlstore) InsertSiteStats(s *models.SiteStats) error { func (db *sqlstore) InsertSiteStats(s *models.SiteStats) error {
query := db.Rebind(`INSERT INTO daily_site_stats(visitors, sessions, pageviews, bounce_rate, avg_duration, date) VALUES(?, ?, ?, ?, ?, ?)`) query := db.Rebind(`INSERT INTO daily_site_stats(visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02")) _, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
return err return err
} }
func (db *sqlstore) UpdateSiteStats(s *models.SiteStats) error { func (db *sqlstore) UpdateSiteStats(s *models.SiteStats) error {
query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE date = ?`) query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE date = ?`)
_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02")) _, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
return err return err
} }

2
pkg/datastore/sqlstore/sqlstore.go
View File

@ -38,7 +38,7 @@ func New(c *Config) *sqlstore {
func (db *sqlstore) Migrate() { func (db *sqlstore) Migrate() {
migrationSource := &migrate.PackrMigrationSource{ migrationSource := &migrate.PackrMigrationSource{
Box: packr.NewBox("./migrations"), Box: packr.NewBox("./migrations"),
Dir: "./" + db.Config.Driver, Dir: db.Config.Driver,
} }
migrate.SetTable("migrations") migrate.SetTable("migrations")

17
pkg/models/page_stats.go
View File

@ -5,12 +5,13 @@ import (
) )
type PageStats struct { type PageStats struct {
Hostname string `db:"hostname"` Hostname string `db:"hostname"`
Pathname string `db:"pathname"` Pathname string `db:"pathname"`
Pageviews int64 `db:"pageviews"` Pageviews int64 `db:"pageviews"`
Visitors int64 `db:"visitors"` Visitors int64 `db:"visitors"`
Entries int64 `db:"entries"` Entries int64 `db:"entries"`
BounceRate float64 `db:"bounce_rate"` BounceRate float64 `db:"bounce_rate"`
AvgDuration float64 `db:"avg_duration"` AvgDuration float64 `db:"avg_duration"`
Date time.Time `db:"date" json:",omitempty"` KnownDurations int64 `db:"known_durations"`
Date time.Time `db:"date" json:",omitempty"`
} }

13
pkg/models/referrer_stats.go
View File

@ -5,10 +5,11 @@ import (
) )
type ReferrerStats struct { type ReferrerStats struct {
URL string `db:"url"` URL string `db:"url"`
Visitors int64 `db:"visitors"` Visitors int64 `db:"visitors"`
Pageviews int64 `db:"pageviews"` Pageviews int64 `db:"pageviews"`
BounceRate float64 `db:"bounce_rate"` BounceRate float64 `db:"bounce_rate"`
AvgDuration float64 `db:"avg_duration"` AvgDuration float64 `db:"avg_duration"`
Date time.Time `db:"date" json:",omitempty"` KnownDurations int64 `db:"known_durations"`
Date time.Time `db:"date" json:",omitempty"`
} }

13
pkg/models/site_stats.go
View File

@ -5,10 +5,11 @@ import (
) )
type SiteStats struct { type SiteStats struct {
Visitors int64 `db:"visitors"` Visitors int64 `db:"visitors"`
Pageviews int64 `db:"pageviews"` Pageviews int64 `db:"pageviews"`
Sessions int64 `db:"sessions"` Sessions int64 `db:"sessions"`
BounceRate float64 `db:"bounce_rate"` BounceRate float64 `db:"bounce_rate"`
AvgDuration float64 `db:"avg_duration"` AvgDuration float64 `db:"avg_duration"`
Date time.Time `db:"date" json:",omitempty"` KnownDurations int64 `db:"known_durations"`
Date time.Time `db:"date" json:",omitempty"`
} }