status-im/fathom
2
0
Fork 0
mirror of https://github.com/status-im/fathom.git synced 2025-03-01 03:20:27 +00:00
Code Issues Projects Releases Wiki Activity

get aggregator to take site ID's into account

Browse Source
This commit is contained in:
Danny van Kooten 2018-10-05 14:19:11 +02:00
parent c1367325e6
commit d4176de238
15 changed files with 119 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
assets/src/js/components/Chart.js vendored
View File

@ -101,11 +101,11 @@ class Chart extends Component {
}
componentWillReceiveProps(newProps) {
if(newProps.before == this.props.before && newProps.after == this.props.after) {
if(this.props == newProps) {
return;
}
this.fetchData(newProps.before, newProps.after);
this.fetchData(newProps);
}
@bind
@ -212,17 +212,17 @@ class Chart extends Component {
}
@bind
fetchData(before, after) {
fetchData(props) {
this.setState({ loading: true })
Client.request(`/sites/${this.props.site.id}/stats/site/groupby/day?before=${before}&after=${after}`)
Client.request(`/sites/${props.site.id}/stats/site/groupby/day?before=${props.before}&after=${props.after}`)
.then((d) => {
// request finished; check if timestamp range is still the one user wants to see
if( this.props.before != before || this.props.after != after ) {
// request finished; check if args changed in the meantime
if( props != this.props) {
return;
}
let chartData = prepareData(after, before, d);
let chartData = prepareData(props.after, props.before, d);
this.setState({
loading: false,
data: chartData,

12
assets/src/js/components/Sidebar.js
View File

@ -17,22 +17,24 @@ class Sidebar extends Component {
}
componentWillReceiveProps(newProps, newState) {
if(newProps.before == this.props.before && newProps.after == this.props.after) {
if(newProps == this.props) {
return;
}
this.fetchData(newProps.before, newProps.after);
this.fetchData(newProps);
}
@bind
fetchData(before, after) {
fetchData(props) {
this.setState({ loading: true })
Client.request(`/sites/${this.props.site.id}/stats/site?before=${before}&after=${after}`)
Client.request(`/sites/${props.site.id}/stats/site?before=${props.before}&after=${props.after}`)
.then((data) => {
// request finished; check if timestamp range is still the one user wants to see
if( this.props.before != before || this.props.after != after ) {
if( props != this.props ) {
return;
}

2
assets/src/js/components/SiteSettings.js
View File

@ -89,7 +89,7 @@ class SiteSettings extends Component {
</fieldset>
<fieldset>
<label>Add this code to your website</label>
<label>Add this code to your website <small class="right">(site ID = {props.site.trackingId})</small></label>
<textarea ref={(el) => { this.textarea = el }} onClick={this.onTextareaClick} readonly="readonly">{`<!-- Fathom - simple website analytics - https://github.com/usefathom/fathom -->
<script>
(function(f, a, t, h, o, m){

12
assets/src/js/components/Table.js
View File

@ -22,21 +22,21 @@ class Table extends Component {
}
componentWillReceiveProps(newProps) {
if(newProps.before == this.props.before && newProps.after == this.props.after) {
if(this.props == newProps) {
return;
}
this.fetchRecords(newProps.before, newProps.after);
this.fetchData(newProps);
}
@bind
fetchRecords(before, after) {
fetchData(props) {
this.setState({ loading: true });
Client.request(`/sites/${this.props.site.id}/${this.props.endpoint}?before=${before}&after=${after}&limit=${this.state.limit}`)
Client.request(`/sites/${props.site.id}/${props.endpoint}?before=${props.before}&after=${props.after}&limit=${this.state.limit}`)
.then((d) => {
// request finished; check if timestamp range is still the one user wants to see
if( this.props.before != before || this.props.after != after ) {
if( this.props != props ) {
return;
}
@ -47,7 +47,7 @@ class Table extends Component {
});
// fetch totals too
Client.request(`/sites/${this.props.site.id}/${this.props.endpoint}/pageviews?before=${before}&after=${after}`)
Client.request(`/sites/${props.site.id}/${props.endpoint}/pageviews?before=${props.before}&after=${props.after}`)
.then((d) => {
this.setState({
total: d

7
assets/src/sass/util.scss
View File

@ -3,6 +3,13 @@
margin-bottom: 20px;
}
.right {
float: right;
}
.left {
float: left;
}
.notification {
position: fixed;

33
pkg/aggregator/aggregator.go
View File

@ -39,21 +39,21 @@ func (agg *aggregator) Run() int {
// update stats
for _, site := range results.Sites {
err = agg.database.UpdateSiteStats(site)
err = agg.database.SaveSiteStats(site)
if err != nil {
log.Error(err)
}
}
for _, pageStats := range results.Pages {
err = agg.database.UpdatePageStats(pageStats)
err = agg.database.SavePageStats(pageStats)
if err != nil {
log.Error(err)
}
}
for _, referrerStats := range results.Referrers {
err = agg.database.UpdateReferrerStats(referrerStats)
err = agg.database.SaveReferrerStats(referrerStats)
if err != nil {
log.Error(err)
}
@ -73,15 +73,36 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
log.Debugf("processing %d pageviews", len(pageviews))
results := newResults()
sites, err := agg.database.GetSites()
if err != nil {
log.Error(err)
return nil
}
// create map of public tracking ID's => site ID
trackingIDMap := make(map[string]int64, len(sites)+1)
trackingIDMap["0"] = 0
for _, s := range sites {
trackingIDMap[s.TrackingID] = s.ID
}
for _, p := range pageviews {
site, err := agg.getSiteStats(results, p.Timestamp)
// discard pageview if site tracking ID is unknown
siteID, ok := trackingIDMap[p.SiteTrackingID]
if !ok {
continue
}
// get existing site stats so we can add this pageview to it
site, err := agg.getSiteStats(results, siteID, p.Timestamp)
if err != nil {
log.Error(err)
continue
}
site.HandlePageview(p)
pageStats, err := agg.getPageStats(results, p.Timestamp, p.Hostname, p.Pathname)
pageStats, err := agg.getPageStats(results, siteID, p.Timestamp, p.Hostname, p.Pathname)
if err != nil {
log.Error(err)
continue
@ -97,7 +118,7 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
continue
}
referrerStats, err := agg.getReferrerStats(results, p.Timestamp, hostname, pathname)
referrerStats, err := agg.getReferrerStats(results, siteID, p.Timestamp, hostname, pathname)
if err != nil {
log.Error(err)
continue

52
pkg/aggregator/store.go
View File

@ -1,6 +1,7 @@
package aggregator
import (
"fmt"
"strings"
"time"
@ -8,16 +9,16 @@ import (
"github.com/usefathom/fathom/pkg/models"
)
func (agg *aggregator) getSiteStats(r *results, t time.Time) (*models.SiteStats, error) {
func (agg *aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*models.SiteStats, error) {
// get from map
date := t.Format("2006-01-02")
if stats, ok := r.Sites[date]; ok {
cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02"))
if stats, ok := r.Sites[cacheKey]; ok {
return stats, nil
}
// get from db
stats, err := agg.database.GetSiteStats(0, t)
stats, err := agg.database.GetSiteStats(siteID, t)
if err != nil && err != datastore.ErrNoResults {
return nil, err
}
@ -25,54 +26,50 @@ func (agg *aggregator) getSiteStats(r *results, t time.Time) (*models.SiteStats,
// create in db
if stats == nil {
stats = &models.SiteStats{
SiteID: siteID,
Date: t,
}
err = agg.database.InsertSiteStats(stats)
if err != nil {
return nil, err
New: true,
}
}
r.Sites[date] = stats
r.Sites[cacheKey] = stats
return stats, nil
}
func (agg *aggregator) getPageStats(r *results, t time.Time, hostname string, pathname string) (*models.PageStats, error) {
date := t.Format("2006-01-02")
if stats, ok := r.Pages[date+hostname+pathname]; ok {
func (agg *aggregator) getPageStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.PageStats, error) {
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
if stats, ok := r.Pages[cacheKey]; ok {
return stats, nil
}
stats, err := agg.database.GetPageStats(0, t, hostname, pathname)
stats, err := agg.database.GetPageStats(siteID, t, hostname, pathname)
if err != nil && err != datastore.ErrNoResults {
return nil, err
}
if stats == nil {
stats = &models.PageStats{
SiteID: siteID,
New: true,
Hostname: hostname,
Pathname: pathname,
Date: t,
}
err = agg.database.InsertPageStats(stats)
if err != nil {
return nil, err
}
}
r.Pages[date+hostname+pathname] = stats
r.Pages[cacheKey] = stats
return stats, nil
}
func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
date := t.Format("2006-01-02")
if stats, ok := r.Referrers[date+hostname+pathname]; ok {
func (agg *aggregator) getReferrerStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
if stats, ok := r.Referrers[cacheKey]; ok {
return stats, nil
}
// get from db
stats, err := agg.database.GetReferrerStats(0, t, hostname, pathname)
stats, err := agg.database.GetReferrerStats(siteID, t, hostname, pathname)
if err != nil && err != datastore.ErrNoResults {
return nil, err
}
@ -80,6 +77,8 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string
// create in db
if stats == nil {
stats = &models.ReferrerStats{
SiteID: siteID,
New: true,
Hostname: hostname,
Pathname: pathname,
Date: t,
@ -90,13 +89,8 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string
if strings.Contains(stats.Hostname, "www.google.") {
stats.Group = "Google"
}
err = agg.database.InsertReferrerStats(stats)
if err != nil {
return nil, err
}
}
r.Referrers[date+hostname+pathname] = stats
r.Referrers[cacheKey] = stats
return stats, nil
}

2
pkg/api/sites.go
View File

@ -37,6 +37,8 @@ func (api *API) SaveSiteHandler(w http.ResponseWriter, r *http.Request) error {
return err
}
// TODO: If we just created the first site, add existing data (with site_id = 0) to the site we just created
return respond(w, http.StatusOK, envelope{Data: s})
}

9
pkg/datastore/datastore.go
View File

@ -27,8 +27,7 @@ type Datastore interface {
// site stats
GetSiteStats(int64, time.Time) (*models.SiteStats, error)
GetSiteStatsPerDay(int64, time.Time, time.Time) ([]*models.SiteStats, error)
InsertSiteStats(*models.SiteStats) error
UpdateSiteStats(*models.SiteStats) error
SaveSiteStats(*models.SiteStats) error
GetAggregatedSiteStats(int64, time.Time, time.Time) (*models.SiteStats, error)
GetTotalSiteViews(int64, time.Time, time.Time) (int64, error)
GetTotalSiteVisitors(int64, time.Time, time.Time) (int64, error)
@ -46,15 +45,13 @@ type Datastore interface {
// page stats
GetPageStats(int64, time.Time, string, string) (*models.PageStats, error)
InsertPageStats(*models.PageStats) error
UpdatePageStats(*models.PageStats) error
SavePageStats(*models.PageStats) error
GetAggregatedPageStats(int64, time.Time, time.Time, int64) ([]*models.PageStats, error)
GetAggregatedPageStatsPageviews(int64, time.Time, time.Time) (int64, error)
// referrer stats
GetReferrerStats(int64, time.Time, string, string) (*models.ReferrerStats, error)
InsertReferrerStats(*models.ReferrerStats) error
UpdateReferrerStats(*models.ReferrerStats) error
SaveReferrerStats(*models.ReferrerStats) error
GetAggregatedReferrerStats(int64, time.Time, time.Time, int64) ([]*models.ReferrerStats, error)
GetAggregatedReferrerStatsPageviews(int64, time.Time, time.Time) (int64, error)

12
pkg/datastore/sqlstore/page_stats.go
View File

@ -17,13 +17,21 @@ func (db *sqlstore) GetPageStats(siteID int64, date time.Time, hostname string,
return stats, err
}
func (db *sqlstore) InsertPageStats(s *models.PageStats) error {
func (db *sqlstore) SavePageStats(s *models.PageStats) error {
if s.New {
return db.insertPageStats(s)
}
return db.updatePageStats(s)
}
func (db *sqlstore) insertPageStats(s *models.PageStats) error {
query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, site_id, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err
}
func (db *sqlstore) UpdatePageStats(s *models.PageStats) error {
func (db *sqlstore) updatePageStats(s *models.PageStats) error {
query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE site_id = ? AND hostname = ? AND pathname = ? AND date = ?`)
_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err

12
pkg/datastore/sqlstore/referrer_stats.go
View File

@ -17,13 +17,21 @@ func (db *sqlstore) GetReferrerStats(siteID int64, date time.Time, hostname stri
return stats, err
}
func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error {
func (db *sqlstore) SaveReferrerStats(s *models.ReferrerStats) error {
if s.New {
return db.insertReferrerStats(s)
}
return db.updateReferrerStats(s)
}
func (db *sqlstore) insertReferrerStats(s *models.ReferrerStats) error {
query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, site_id, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err
}
func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error {
func (db *sqlstore) updateReferrerStats(s *models.ReferrerStats) error {
query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ?, groupname = ? WHERE site_id = ? AND hostname = ? AND pathname = ? AND date = ?`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
return err

16
pkg/datastore/sqlstore/site_stats.go
View File

@ -8,7 +8,7 @@ import (
)
func (db *sqlstore) GetSiteStats(siteID int64, date time.Time) (*models.SiteStats, error) {
stats := &models.SiteStats{}
stats := &models.SiteStats{New: false}
query := db.Rebind(`SELECT * FROM daily_site_stats WHERE site_id = ? AND date = ? LIMIT 1`)
err := db.Get(stats, query, siteID, date.Format("2006-01-02"))
if err != nil && err == sql.ErrNoRows {
@ -17,13 +17,21 @@ func (db *sqlstore) GetSiteStats(siteID int64, date time.Time) (*models.SiteStat
return stats, err
}
func (db *sqlstore) InsertSiteStats(s *models.SiteStats) error {
query := db.Rebind(`INSERT INTO daily_site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
func (db *sqlstore) SaveSiteStats(s *models.SiteStats) error {
if s.New {
return db.insertSiteStats(s)
}
return db.updateSiteStats(s)
}
func (db *sqlstore) insertSiteStats(s *models.SiteStats) error {
query := db.Rebind(`INSERT INTO daily_site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.SiteID, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
return err
}
func (db *sqlstore) UpdateSiteStats(s *models.SiteStats) error {
func (db *sqlstore) updateSiteStats(s *models.SiteStats) error {
query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE site_id = ? AND date = ?`)
_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Date.Format("2006-01-02"))
return err

1
pkg/models/page_stats.go
View File

@ -5,6 +5,7 @@ import (
)
type PageStats struct {
New bool `db:"-" json:"-"`
SiteID int64 `db:"site_id"`
Hostname string `db:"hostname"`
Pathname string `db:"pathname"`

1
pkg/models/referrer_stats.go
View File

@ -5,6 +5,7 @@ import (
)
type ReferrerStats struct {
New bool `db:"-" json:"-"`
SiteID int64 `db:"site_id"`
Hostname string `db:"hostname"`
Pathname string `db:"pathname"`

1
pkg/models/site_stats.go
View File

@ -6,6 +6,7 @@ import (
)
type SiteStats struct {
New bool `db:"-" json:"-" `
SiteID int64 `db:"site_id"`
Visitors int64 `db:"visitors"`
Pageviews int64 `db:"pageviews"`