diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go
index 936bc67..5b4ab81 100644
--- a/pkg/aggregator/aggregator.go
+++ b/pkg/aggregator/aggregator.go
@@ -3,6 +3,7 @@ package aggregator
import (
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
+ "net/url"
log "github.com/sirupsen/logrus"
)
@@ -124,7 +125,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
// referrer stats
if p.Referrer != "" {
- referrerStats, err := agg.getReferrerStats(results, p.Timestamp, p.Referrer)
+ hostname, pathname, _ := parseUrlParts(p.Referrer)
+ referrerStats, err := agg.getReferrerStats(results, p.Timestamp, hostname, pathname)
if err != nil {
log.Error(err)
continue
@@ -153,3 +155,12 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
return results
}
+
+func parseUrlParts(s string) (string, string, error) {
+ u, err := url.Parse(s)
+ if err != nil {
+ return "", "", err
+ }
+
+ return u.Scheme + "://" + u.Host, u.Path, nil
+}
diff --git a/pkg/aggregator/store.go b/pkg/aggregator/store.go
index b7b0631..0090660 100644
--- a/pkg/aggregator/store.go
+++ b/pkg/aggregator/store.go
@@ -64,14 +64,14 @@ func (agg *aggregator) getPageStats(r *results, t time.Time, hostname string, pa
return stats, nil
}
-func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*models.ReferrerStats, error) {
+func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
date := t.Format("2006-01-02")
- if stats, ok := r.Referrers[date+url]; ok {
+ if stats, ok := r.Referrers[date+hostname+pathname]; ok {
return stats, nil
}
// get from db
- stats, err := agg.database.GetReferrerStats(t, url)
+ stats, err := agg.database.GetReferrerStats(t, hostname, pathname)
if err != nil && err != datastore.ErrNoResults {
return nil, err
}
@@ -79,8 +79,9 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*m
// create in db
if stats == nil {
stats = &models.ReferrerStats{
- URL: url,
- Date: t,
+ Hostname: hostname,
+ Pathname: pathname,
+ Date: t,
}
err = agg.database.InsertReferrerStats(stats)
if err != nil {
@@ -88,6 +89,6 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*m
}
}
- r.Referrers[date+url] = stats
+ r.Referrers[date+hostname+pathname] = stats
return stats, nil
}
diff --git a/pkg/datastore/datastore.go b/pkg/datastore/datastore.go
index 28d91b1..a13a090 100644
--- a/pkg/datastore/datastore.go
+++ b/pkg/datastore/datastore.go
@@ -43,7 +43,7 @@ type Datastore interface {
GetAggregatedPageStatsPageviews(time.Time, time.Time) (int, error)
// referrer stats
- GetReferrerStats(time.Time, string) (*models.ReferrerStats, error)
+ GetReferrerStats(time.Time, string, string) (*models.ReferrerStats, error)
InsertReferrerStats(*models.ReferrerStats) error
UpdateReferrerStats(*models.ReferrerStats) error
GetAggregatedReferrerStats(time.Time, time.Time, int) ([]*models.ReferrerStats, error)
diff --git a/pkg/datastore/sqlstore/migrations/mysql/3_referrer_group_column.sql b/pkg/datastore/sqlstore/migrations/mysql/3_referrer_group_column.sql
new file mode 100644
index 0000000..9b19891
--- /dev/null
+++ b/pkg/datastore/sqlstore/migrations/mysql/3_referrer_group_column.sql
@@ -0,0 +1,19 @@
+-- +migrate Up
+
+ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
+ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
+ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
+
+UPDATE daily_referrer_stats SET hostname = SUBSTRING_INDEX( url, "/", 3) WHERE url != "" ANd hostname = "";
+UPDATE daily_referrer_stats SET pathname = CONCAT("/", SUBSTRING_INDEX( url, "/", -1)) WHERE url != "" AND pathname = "";
+
+ALTER TABLE daily_referrer_stats DROP COLUMN url;
+
+-- +migrate Down
+
+ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
+ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
+ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
+
+ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;
+
diff --git a/pkg/datastore/sqlstore/migrations/postgres/3_referrer_group_column.sql b/pkg/datastore/sqlstore/migrations/postgres/3_referrer_group_column.sql
new file mode 100644
index 0000000..b06b5bf
--- /dev/null
+++ b/pkg/datastore/sqlstore/migrations/postgres/3_referrer_group_column.sql
@@ -0,0 +1,18 @@
+-- +migrate Up
+
+ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
+ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
+ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
+
+UPDATE daily_referrer_stats SET hostname = CONCAT( SPLIT_PART(url, '://', 1), '://', SPLIT_PART(SPLIT_PART(url, '://', 2), '/', 1) ) WHERE url != '' AND hostname = '';
+UPDATE daily_referrer_stats SET pathname = SPLIT_PART( url, hostname, 2 ) WHERE url != '' AND pathname = '';
+
+ALTER TABLE daily_referrer_stats DROP COLUMN url;
+
+-- +migrate Down
+
+ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
+ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
+ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
+
+ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;
diff --git a/pkg/datastore/sqlstore/migrations/sqlite3/3_referrer_group_column.sql b/pkg/datastore/sqlstore/migrations/sqlite3/3_referrer_group_column.sql
new file mode 100644
index 0000000..16fff36
--- /dev/null
+++ b/pkg/datastore/sqlstore/migrations/sqlite3/3_referrer_group_column.sql
@@ -0,0 +1,18 @@
+-- +migrate Up
+
+ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
+ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
+ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
+
+UPDATE daily_referrer_stats SET hostname = SUBSTR(url, 0, INSTR(url, '://')+3+INSTR(SUBSTR(url, INSTR(url, '://')+3), '/'-1)) WHERE url != '' AND hostname = '';
+UPDATE daily_referrer_stats SET pathname = SUBSTR(url, LENGTH(hostname)) WHERE url != '' AND pathname = '';
+
+ALTER TABLE daily_referrer_stats DROP COLUMN url;
+
+-- +migrate Down
+
+ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
+ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
+ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
+
+ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;
diff --git a/pkg/datastore/sqlstore/referrer_stats.go b/pkg/datastore/sqlstore/referrer_stats.go
index 244cca6..96566dd 100644
--- a/pkg/datastore/sqlstore/referrer_stats.go
+++ b/pkg/datastore/sqlstore/referrer_stats.go
@@ -7,10 +7,10 @@ import (
"github.com/usefathom/fathom/pkg/models"
)
-func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.ReferrerStats, error) {
+func (db *sqlstore) GetReferrerStats(date time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
stats := &models.ReferrerStats{}
- query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE url = ? AND date = ? LIMIT 1`)
- err := db.Get(stats, query, url, date.Format("2006-01-02"))
+ query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE date = ? AND hostname = ? AND pathname = ? LIMIT 1`)
+ err := db.Get(stats, query, date.Format("2006-01-02"), hostname, pathname)
if err != nil && err == sql.ErrNoRows {
return nil, ErrNoResults
}
@@ -18,20 +18,31 @@ func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.Referr
}
func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error {
- query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, url, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
- _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
+ query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+ _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err
}
func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error {
- query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE url = ? AND date = ?`)
- _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
+ query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ?, groupname = ? WHERE hostname = ? AND pathname = ? AND date = ?`)
+ _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err
}
func (db *sqlstore) GetAggregatedReferrerStats(startDate time.Time, endDate time.Time, limit int) ([]*models.ReferrerStats, error) {
var result []*models.ReferrerStats
- query := db.Rebind(`SELECT url, SUM(visitors) AS visitors, SUM(pageviews) AS pageviews, COALESCE(ROUND(SUM(pageviews*bounce_rate)/SUM(pageviews), 4), 0.00) AS bounce_rate, COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration FROM daily_referrer_stats WHERE date >= ? AND date <= ? GROUP BY url ORDER BY pageviews DESC LIMIT ?`)
+ query := db.Rebind(`
+ SELECT
+ MIN(hostname) AS hostname,
+ MIN(pathname) AS pathname,
+ MIN(COALESCE(groupname, "")) AS groupname,
+ SUM(visitors) AS visitors,
+ SUM(pageviews) AS pageviews,
+ COALESCE(ROUND(SUM(pageviews*bounce_rate)/SUM(pageviews), 4), 0.00) AS bounce_rate,
+ COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration
+ FROM daily_referrer_stats
+ WHERE date >= ? AND date <= ?
+ GROUP BY COALESCE(groupname, CONCAT(hostname, pathname)) ORDER BY pageviews DESC LIMIT ?`)
err := db.Select(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit)
return result, err
}
diff --git a/pkg/models/referrer_stats.go b/pkg/models/referrer_stats.go
index c107dc8..ef8093e 100644
--- a/pkg/models/referrer_stats.go
+++ b/pkg/models/referrer_stats.go
@@ -5,7 +5,9 @@ import (
)
type ReferrerStats struct {
- URL string `db:"url"`
+ Hostname string `db:"hostname"`
+ Pathname string `db:"pathname"`
+ Group string `db:"groupname"`
Visitors int64 `db:"visitors"`
Pageviews int64 `db:"pageviews"`
BounceRate float64 `db:"bounce_rate"`