status-im/fathom
2
0
Fork 0
mirror of https://github.com/status-im/fathom.git synced 2025-03-01 11:30:28 +00:00
Code Issues Projects Releases Wiki Activity

split hostname and pathname in referrers table

Browse Source
This commit is contained in:
Danny 2018-05-30 12:50:28 +02:00
parent 2c1edc6aeb
commit 071ea5ace0
9 changed files with 107 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
assets/src/js/components/Table.js
View File

@ -57,21 +57,25 @@ class Table extends Component {
render(props, state) {
const tableRows = state.records !== null && state.records.length > 0 ? state.records.map((p, i) => {
let ahref = document.createElement('a');
ahref.href = (p.Hostname + p.Pathname) || p.URL;
let href = (p.Hostname + p.Pathname) || p.URL;
let classes = "table-row";
if(state.total > 0) {
classes += " w" + Math.min(98, Math.round(p.Pageviews / state.total * 100 * 2.5));
}
let label = ahref.pathname + ahref.search;
let label = p.Pathname
if( props.showHostname ) {
label = ahref.hostname.replace('www.', '') + (ahref.pathname.length > 1 ? ahref.pathname : '');
if( p.Group) {
label = p.Group
} else {
label = p.Hostname.replace('www.', '').replace('https://', '').replace('http://', '') + (p.Pathname.length > 1 ? p.Pathname : '')
}
}
return(
<div class={classes}>
<div class="cell main-col"><a href={ahref.href}>{label}</a></div>
<div class="cell main-col"><a href={href}>{label}</a></div>
<div class="cell">{p.Pageviews}</div>
<div class="cell">{p.Visitors||"-"}</div>
</div>
@ -81,8 +85,7 @@ class Table extends Component {
<div class={(state.loading ? "loading" : '')}>
<div class="table-row header">
{props.headers.map((header, i) => {
let classes = i === 0 ? 'main-col cell' : 'cell';
return (<div class={classes}>{header}</div>)
return (<div class={i === 0 ? 'main-col cell' : 'cell'}>{header}</div>)
})}
</div>
<div>

13
pkg/aggregator/aggregator.go
View File

@ -3,6 +3,7 @@ package aggregator
import (
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
"net/url"
log "github.com/sirupsen/logrus"
)
@ -124,7 +125,8 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
// referrer stats
if p.Referrer != "" {
referrerStats, err := agg.getReferrerStats(results, p.Timestamp, p.Referrer)
hostname, pathname, _ := parseUrlParts(p.Referrer)
referrerStats, err := agg.getReferrerStats(results, p.Timestamp, hostname, pathname)
if err != nil {
log.Error(err)
continue
@ -153,3 +155,12 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
return results
}
func parseUrlParts(s string) (string, string, error) {
u, err := url.Parse(s)
if err != nil {
return "", "", err
}
return u.Scheme + "://" + u.Host, u.Path, nil
}

13
pkg/aggregator/store.go
View File

@ -64,14 +64,14 @@ func (agg *aggregator) getPageStats(r *results, t time.Time, hostname string, pa
return stats, nil
}
func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*models.ReferrerStats, error) {
func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
date := t.Format("2006-01-02")
if stats, ok := r.Referrers[date+url]; ok {
if stats, ok := r.Referrers[date+hostname+pathname]; ok {
return stats, nil
}
// get from db
stats, err := agg.database.GetReferrerStats(t, url)
stats, err := agg.database.GetReferrerStats(t, hostname, pathname)
if err != nil && err != datastore.ErrNoResults {
return nil, err
}
@ -79,8 +79,9 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*m
// create in db
if stats == nil {
stats = &models.ReferrerStats{
URL: url,
Date: t,
Hostname: hostname,
Pathname: pathname,
Date: t,
}
err = agg.database.InsertReferrerStats(stats)
if err != nil {
@ -88,6 +89,6 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, url string) (*m
}
}
r.Referrers[date+url] = stats
r.Referrers[date+hostname+pathname] = stats
return stats, nil
}

2
pkg/datastore/datastore.go
View File

@ -43,7 +43,7 @@ type Datastore interface {
GetAggregatedPageStatsPageviews(time.Time, time.Time) (int, error)
// referrer stats
GetReferrerStats(time.Time, string) (*models.ReferrerStats, error)
GetReferrerStats(time.Time, string, string) (*models.ReferrerStats, error)
InsertReferrerStats(*models.ReferrerStats) error
UpdateReferrerStats(*models.ReferrerStats) error
GetAggregatedReferrerStats(time.Time, time.Time, int) ([]*models.ReferrerStats, error)

19
pkg/datastore/sqlstore/migrations/mysql/3_referrer_group_column.sql Normal file
View File

@ -0,0 +1,19 @@
-- +migrate Up
ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
UPDATE daily_referrer_stats SET hostname = SUBSTRING_INDEX( url, "/", 3) WHERE url != "" ANd hostname = "";
UPDATE daily_referrer_stats SET pathname = CONCAT("/", SUBSTRING_INDEX( url, "/", -1)) WHERE url != "" AND pathname = "";
ALTER TABLE daily_referrer_stats DROP COLUMN url;
-- +migrate Down
ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;

18
pkg/datastore/sqlstore/migrations/postgres/3_referrer_group_column.sql Normal file
View File

@ -0,0 +1,18 @@
-- +migrate Up
ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
UPDATE daily_referrer_stats SET hostname = CONCAT( SPLIT_PART(url, '://', 1), '://', SPLIT_PART(SPLIT_PART(url, '://', 2), '/', 1) ) WHERE url != '' AND hostname = '';
UPDATE daily_referrer_stats SET pathname = SPLIT_PART( url, hostname, 2 ) WHERE url != '' AND pathname = '';
ALTER TABLE daily_referrer_stats DROP COLUMN url;
-- +migrate Down
ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;

18
pkg/datastore/sqlstore/migrations/sqlite3/3_referrer_group_column.sql Normal file
View File

@ -0,0 +1,18 @@
-- +migrate Up
ALTER TABLE daily_referrer_stats ADD COLUMN groupname VARCHAR(255) NULL;
ALTER TABLE daily_referrer_stats ADD COLUMN hostname VARCHAR(255);
ALTER TABLE daily_referrer_stats ADD COLUMN pathname VARCHAR(255);
UPDATE daily_referrer_stats SET hostname = SUBSTR(url, 0, INSTR(url, '://')+3+INSTR(SUBSTR(url, INSTR(url, '://')+3), '/'-1)) WHERE url != '' AND hostname = '';
UPDATE daily_referrer_stats SET pathname = SUBSTR(url, LENGTH(hostname)) WHERE url != '' AND pathname = '';
ALTER TABLE daily_referrer_stats DROP COLUMN url;
-- +migrate Down
ALTER TABLE daily_referrer_stats DROP COLUMN groupname;
ALTER TABLE daily_referrer_stats DROP COLUMN hostname;
ALTER TABLE daily_referrer_stats DROP COLUMN pathname;
ALTER TABLE daily_referrer_stats ADD COLUMN url VARCHAR(255) NOT NULL;

27
pkg/datastore/sqlstore/referrer_stats.go
View File

@ -7,10 +7,10 @@ import (
"github.com/usefathom/fathom/pkg/models"
)
func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.ReferrerStats, error) {
func (db *sqlstore) GetReferrerStats(date time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
stats := &models.ReferrerStats{}
query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE url = ? AND date = ? LIMIT 1`)
err := db.Get(stats, query, url, date.Format("2006-01-02"))
query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE date = ? AND hostname = ? AND pathname = ? LIMIT 1`)
err := db.Get(stats, query, date.Format("2006-01-02"), hostname, pathname)
if err != nil && err == sql.ErrNoRows {
return nil, ErrNoResults
}
@ -18,20 +18,31 @@ func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.Referr
}
func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error {
query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, url, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err
}
func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error {
query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE url = ? AND date = ?`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.URL, s.Date.Format("2006-01-02"))
query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ?, groupname = ? WHERE hostname = ? AND pathname = ? AND date = ?`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err
}
func (db *sqlstore) GetAggregatedReferrerStats(startDate time.Time, endDate time.Time, limit int) ([]*models.ReferrerStats, error) {
var result []*models.ReferrerStats
query := db.Rebind(`SELECT url, SUM(visitors) AS visitors, SUM(pageviews) AS pageviews, COALESCE(ROUND(SUM(pageviews*bounce_rate)/SUM(pageviews), 4), 0.00) AS bounce_rate, COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration FROM daily_referrer_stats WHERE date >= ? AND date <= ? GROUP BY url ORDER BY pageviews DESC LIMIT ?`)
query := db.Rebind(`
SELECT
MIN(hostname) AS hostname,
MIN(pathname) AS pathname,
MIN(COALESCE(groupname, "")) AS groupname,
SUM(visitors) AS visitors,
SUM(pageviews) AS pageviews,
COALESCE(ROUND(SUM(pageviews*bounce_rate)/SUM(pageviews), 4), 0.00) AS bounce_rate,
COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration
FROM daily_referrer_stats
WHERE date >= ? AND date <= ?
GROUP BY COALESCE(groupname, CONCAT(hostname, pathname)) ORDER BY pageviews DESC LIMIT ?`)
err := db.Select(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit)
return result, err
}

4
pkg/models/referrer_stats.go
View File

@ -5,7 +5,9 @@ import (
)
type ReferrerStats struct {
URL string `db:"url"`
Hostname string `db:"hostname"`
Pathname string `db:"pathname"`
Group string `db:"groupname"`
Visitors int64 `db:"visitors"`
Pageviews int64 `db:"pageviews"`
BounceRate float64 `db:"bounce_rate"`