split hostname and pathname in referrers table

assets/src/js/components/Table.js

@@ -57,21 +57,25 @@ class Table extends Component {
 
   render(props, state) {
     const tableRows = state.records !== null && state.records.length > 0 ? state.records.map((p, i) => {
-      let ahref = document.createElement('a'); 
+
-      ahref.href = (p.Hostname + p.Pathname) || p.URL;
+      let href = (p.Hostname + p.Pathname) || p.URL;
       let classes = "table-row"; 
       if(state.total > 0) {
         classes += " w" + Math.min(98, Math.round(p.Pageviews / state.total * 100 * 2.5));
       }
 
-      let label = ahref.pathname + ahref.search;
+      let label = p.Pathname
       if( props.showHostname ) {
-        label = ahref.hostname.replace('www.', '') + (ahref.pathname.length > 1 ? ahref.pathname : '');
+        if( p.Group) {
+          label = p.Group
+        } else {
+          label = p.Hostname.replace('www.', '').replace('https://', '').replace('http://', '') + (p.Pathname.length > 1 ? p.Pathname : '')
+        }
       }
 
       return(
       <div class={classes}>
-        <div class="cell main-col"><a href={ahref.href}>{label}</a></div>
+        <div class="cell main-col"><a href={href}>{label}</a></div>
         <div class="cell">{p.Pageviews}</div>
         <div class="cell">{p.Visitors||"-"}</div>           
       </div>
@@ -81,8 +85,7 @@ class Table extends Component {
       <div class={(state.loading ? "loading" : '')}>
         <div class="table-row header">
           {props.headers.map((header, i) => {
-            let classes = i === 0 ? 'main-col cell' : 'cell';
+            return (<div class={i === 0 ? 'main-col cell' : 'cell'}>{header}</div>) 
-            return (<div class={classes}>{header}</div>) 
             })}        
         </div>
         <div>

pkg/aggregator/aggregator.go

@@ -3,6 +3,7 @@ package aggregator
 import (
 	"github.com/usefathom/fathom/pkg/datastore"
 	"github.com/usefathom/fathom/pkg/models"
+	"net/url"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -124,7 +125,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
 
 		// referrer stats
 		if p.Referrer != "" {
-			referrerStats, err := agg.getReferrerStats(results, p.Timestamp, p.Referrer)
+			hostname, pathname, _ := parseUrlParts(p.Referrer)
+			referrerStats, err := agg.getReferrerStats(results, p.Timestamp, hostname, pathname)
 			if err != nil {
 				log.Error(err)
 				continue
@@ -153,3 +155,12 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
 
 	return results
 }
+
+func parseUrlParts(s string) (string, string, error) {
+	u, err := url.Parse(s)
+	if err != nil {
+		return "", "", err
+	}
+
+	return u.Scheme + "://" + u.Host, u.Path, nil
+}

pkg/aggregator/store.go

@@ -64,14 +64,14 @@ func (agg *aggregator) getPageStats(r *results, t time.Time, hostname string, pa
 	return stats, nil
 }
 
-func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*models.ReferrerStats, error) {
+func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
 	date := t.Format("2006-01-02")
-	if stats, ok := r.Referrers[date+url]; ok {
+	if stats, ok := r.Referrers[date+hostname+pathname]; ok {
 		return stats, nil
 	}
 
 	// get from db
-	stats, err := agg.database.GetReferrerStats(t, url)
+	stats, err := agg.database.GetReferrerStats(t, hostname, pathname)
 	if err != nil && err != datastore.ErrNoResults {
 		return nil, err
 	}
@@ -79,8 +79,9 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*m
 	// create in db
 	if stats == nil {
 		stats = &models.ReferrerStats{
-			URL:  url,
+			Hostname: hostname,
-			Date: t,
+			Pathname: pathname,
+			Date:     t,
 		}
 		err = agg.database.InsertReferrerStats(stats)
 		if err != nil {
@@ -88,6 +89,6 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*m
 		}
 	}
 
-	r.Referrers[date+url] = stats
+	r.Referrers[date+hostname+pathname] = stats
 	return stats, nil
 }

pkg/datastore/datastore.go

@@ -43,7 +43,7 @@ type Datastore interface {
 	GetAggregatedPageStatsPageviews(time.Time, time.Time) (int, error)
 
 	// referrer stats
-	GetReferrerStats(time.Time, string) (*models.ReferrerStats, error)
+	GetReferrerStats(time.Time, string, string) (*models.ReferrerStats, error)
 	InsertReferrerStats(*models.ReferrerStats) error
 	UpdateReferrerStats(*models.ReferrerStats) error
 	GetAggregatedReferrerStats(time.Time, time.Time, int) ([]*models.ReferrerStats, error)

pkg/datastore/sqlstore/migrations/mysql/3_referrer_group_column.sql

@@ -0,0 +1,19 @@
+-- +migrate Up
+
+ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
+ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
+ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
+
+UPDATE daily_referrer_stats SET hostname = SUBSTRING_INDEX( url, "/", 3) WHERE url != "" ANd hostname = "";
+UPDATE daily_referrer_stats SET pathname = CONCAT("/", SUBSTRING_INDEX( url, "/", -1))  WHERE url != "" AND pathname = "";
+
+ALTER TABLE daily_referrer_stats DROP COLUMN url;
+
+-- +migrate Down
+
+ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
+ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
+ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
+
+ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;
+

pkg/datastore/sqlstore/migrations/postgres/3_referrer_group_column.sql

@@ -0,0 +1,18 @@
+-- +migrate Up
+
+ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
+ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
+ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
+
+UPDATE daily_referrer_stats SET hostname = CONCAT( SPLIT_PART(url, '://', 1), '://', SPLIT_PART(SPLIT_PART(url, '://', 2), '/', 1) ) WHERE url != '' AND hostname = '';
+UPDATE daily_referrer_stats SET pathname = SPLIT_PART( url, hostname, 2 ) WHERE url != '' AND pathname = '';
+
+ALTER TABLE daily_referrer_stats DROP COLUMN url;
+
+-- +migrate Down
+
+ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
+ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
+ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
+
+ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;

pkg/datastore/sqlstore/migrations/sqlite3/3_referrer_group_column.sql

@@ -0,0 +1,18 @@
+-- +migrate Up
+
+ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
+ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
+ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
+
+UPDATE daily_referrer_stats SET hostname = SUBSTR(url, 0, INSTR(url, '://')+3+INSTR(SUBSTR(url, INSTR(url, '://')+3), '/'-1)) WHERE url != '' AND hostname = '';
+UPDATE daily_referrer_stats SET pathname = SUBSTR(url, LENGTH(hostname)) WHERE url != '' AND pathname = '';
+
+ALTER TABLE daily_referrer_stats DROP COLUMN url;
+
+-- +migrate Down
+
+ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
+ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
+ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
+
+ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;

pkg/datastore/sqlstore/referrer_stats.go

@@ -7,10 +7,10 @@ import (
 	"github.com/usefathom/fathom/pkg/models"
 )
 
-func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.ReferrerStats, error) {
+func (db *sqlstore) GetReferrerStats(date time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
 	stats := &models.ReferrerStats{}
-	query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE url = ? AND date = ? LIMIT 1`)
+	query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE date = ? AND hostname = ? AND pathname = ? LIMIT 1`)
-	err := db.Get(stats, query, url, date.Format("2006-01-02"))
+	err := db.Get(stats, query, date.Format("2006-01-02"), hostname, pathname)
 	if err != nil && err == sql.ErrNoRows {
 		return nil, ErrNoResults
 	}
@@ -18,20 +18,31 @@ func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.Referr
 }
 
 func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error {
-	query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, url, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
+	query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)`)
-	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
 	return err
 }
 
 func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error {
-	query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE url = ? AND date = ?`)
+	query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ?, groupname = ? WHERE hostname = ? AND pathname = ? AND date = ?`)
-	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
 	return err
 }
 
 func (db *sqlstore) GetAggregatedReferrerStats(startDate time.Time, endDate time.Time, limit int) ([]*models.ReferrerStats, error) {
 	var result []*models.ReferrerStats
-	query := db.Rebind(`SELECT url, SUM(visitors) AS visitors, SUM(pageviews) AS pageviews, COALESCE(ROUND(SUM(pageviews*bounce_rate)/SUM(pageviews), 4), 0.00) AS bounce_rate, COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration FROM daily_referrer_stats WHERE date >= ? AND date <= ? GROUP BY url ORDER BY pageviews DESC LIMIT ?`)
+	query := db.Rebind(`
+	SELECT 
+		MIN(hostname) AS hostname,
+		MIN(pathname) AS pathname,
+		MIN(COALESCE(groupname, "")) AS groupname, 
+		SUM(visitors) AS visitors, 
+		SUM(pageviews) AS pageviews, 
+		COALESCE(ROUND(SUM(pageviews*bounce_rate)/SUM(pageviews), 4), 0.00) AS bounce_rate, 
+		COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration 
+	FROM daily_referrer_stats 
+	WHERE date >= ? AND date <= ? 
+	GROUP BY COALESCE(groupname, CONCAT(hostname, pathname)) ORDER BY pageviews DESC LIMIT ?`)
 	err := db.Select(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit)
 	return result, err
 }

pkg/models/referrer_stats.go

@@ -5,7 +5,9 @@ import (
 )
 
 type ReferrerStats struct {
-	URL            string    `db:"url"`
+	Hostname       string    `db:"hostname"`
+	Pathname       string    `db:"pathname"`
+	Group          string    `db:"groupname"`
 	Visitors       int64     `db:"visitors"`
 	Pageviews      int64     `db:"pageviews"`
 	BounceRate     float64   `db:"bounce_rate"`