get aggregator to take site ID's into account

assets/src/js/components/Chart.js

@@ -101,11 +101,11 @@ class Chart extends Component {
   }
 
   componentWillReceiveProps(newProps) {
-    if(newProps.before == this.props.before && newProps.after == this.props.after) {
+    if(this.props == newProps) {
       return;
     }
 
-    this.fetchData(newProps.before, newProps.after);
+    this.fetchData(newProps);
   }
 
   @bind
@@ -212,17 +212,17 @@ class Chart extends Component {
   }
 
   @bind
-  fetchData(before, after) {
+  fetchData(props) {
     this.setState({ loading: true })
 
-    Client.request(`/sites/${this.props.site.id}/stats/site/groupby/day?before=${before}&after=${after}`)
+    Client.request(`/sites/${props.site.id}/stats/site/groupby/day?before=${props.before}&after=${props.after}`)
       .then((d) => { 
-        // request finished; check if timestamp range is still the one user wants to see
+        // request finished; check if args changed in the meantime
-        if( this.props.before != before || this.props.after != after ) {
+        if( props != this.props) {
           return;
         }
 
-        let chartData = prepareData(after, before, d);
+        let chartData = prepareData(props.after, props.before, d);
         this.setState({ 
           loading: false,
           data: chartData,

assets/src/js/components/Sidebar.js

@@ -17,22 +17,24 @@ class Sidebar extends Component {
   }
 
   componentWillReceiveProps(newProps, newState) {
-    if(newProps.before == this.props.before && newProps.after == this.props.after) {
+    if(newProps == this.props) {
       return;
     }
 
-    this.fetchData(newProps.before, newProps.after);
+    this.fetchData(newProps);
   }
 
 
+
+
   @bind
-  fetchData(before, after) {
+  fetchData(props) {
     this.setState({ loading: true })
 
-    Client.request(`/sites/${this.props.site.id}/stats/site?before=${before}&after=${after}`)
+    Client.request(`/sites/${props.site.id}/stats/site?before=${props.before}&after=${props.after}`)
       .then((data) => { 
         // request finished; check if timestamp range is still the one user wants to see
-        if( this.props.before != before || this.props.after != after ) {
+        if( props != this.props ) {
           return;
         }
 

assets/src/js/components/SiteSettings.js

@@ -89,7 +89,7 @@ class SiteSettings extends Component {
                     </fieldset>
 
                     <fieldset>
-                        <label>Add this code to your website</label>
+                        <label>Add this code to your website    <small class="right">(site ID = {props.site.trackingId})</small></label>
                         <textarea ref={(el) => { this.textarea = el }} onClick={this.onTextareaClick} readonly="readonly">{`<!-- Fathom - simple website analytics - https://github.com/usefathom/fathom -->
 <script>
 (function(f, a, t, h, o, m){

assets/src/js/components/Table.js

@@ -22,21 +22,21 @@ class Table extends Component {
   }
 
   componentWillReceiveProps(newProps) {
-      if(newProps.before == this.props.before && newProps.after == this.props.after) {
+      if(this.props == newProps) {
         return;
       }
 
-      this.fetchRecords(newProps.before, newProps.after);
+      this.fetchData(newProps);
   }
 
   @bind
-  fetchRecords(before, after) {
+  fetchData(props) {
     this.setState({ loading: true });
 
-    Client.request(`/sites/${this.props.site.id}/${this.props.endpoint}?before=${before}&after=${after}&limit=${this.state.limit}`)
+    Client.request(`/sites/${props.site.id}/${props.endpoint}?before=${props.before}&after=${props.after}&limit=${this.state.limit}`)
       .then((d) => {
          // request finished; check if timestamp range is still the one user wants to see
-        if( this.props.before != before || this.props.after != after ) {
+        if( this.props != props ) {
           return;
         }
 
@@ -47,7 +47,7 @@ class Table extends Component {
       });
 
      // fetch totals too
-     Client.request(`/sites/${this.props.site.id}/${this.props.endpoint}/pageviews?before=${before}&after=${after}`)
+     Client.request(`/sites/${props.site.id}/${props.endpoint}/pageviews?before=${props.before}&after=${props.after}`)
       .then((d) => {
         this.setState({
           total: d

assets/src/sass/util.scss

@@ -3,6 +3,13 @@
    margin-bottom: 20px;
 }
 
+.right {
+  float: right;
+}
+
+.left {
+  float: left;
+}
 
 .notification {
   position: fixed;

pkg/aggregator/aggregator.go

@@ -39,21 +39,21 @@ func (agg *aggregator) Run() int {
 
 	// update stats
 	for _, site := range results.Sites {
-		err = agg.database.UpdateSiteStats(site)
+		err = agg.database.SaveSiteStats(site)
 		if err != nil {
 			log.Error(err)
 		}
 	}
 
 	for _, pageStats := range results.Pages {
-		err = agg.database.UpdatePageStats(pageStats)
+		err = agg.database.SavePageStats(pageStats)
 		if err != nil {
 			log.Error(err)
 		}
 	}
 
 	for _, referrerStats := range results.Referrers {
-		err = agg.database.UpdateReferrerStats(referrerStats)
+		err = agg.database.SaveReferrerStats(referrerStats)
 		if err != nil {
 			log.Error(err)
 		}
@@ -73,15 +73,36 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
 	log.Debugf("processing %d pageviews", len(pageviews))
 	results := newResults()
 
+	sites, err := agg.database.GetSites()
+	if err != nil {
+		log.Error(err)
+		return nil
+	}
+
+	// create map of public tracking ID's => site ID
+	trackingIDMap := make(map[string]int64, len(sites)+1)
+	trackingIDMap["0"] = 0
+	for _, s := range sites {
+		trackingIDMap[s.TrackingID] = s.ID
+	}
+
 	for _, p := range pageviews {
-		site, err := agg.getSiteStats(results, p.Timestamp)
+
+		// discard pageview if site tracking ID is unknown
+		siteID, ok := trackingIDMap[p.SiteTrackingID]
+		if !ok {
+			continue
+		}
+
+		// get existing site stats so we can add this pageview to it
+		site, err := agg.getSiteStats(results, siteID, p.Timestamp)
 		if err != nil {
 			log.Error(err)
 			continue
 		}
 		site.HandlePageview(p)
 
-		pageStats, err := agg.getPageStats(results, p.Timestamp, p.Hostname, p.Pathname)
+		pageStats, err := agg.getPageStats(results, siteID, p.Timestamp, p.Hostname, p.Pathname)
 		if err != nil {
 			log.Error(err)
 			continue
@@ -97,7 +118,7 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
 				continue
 			}
 
-			referrerStats, err := agg.getReferrerStats(results, p.Timestamp, hostname, pathname)
+			referrerStats, err := agg.getReferrerStats(results, siteID, p.Timestamp, hostname, pathname)
 			if err != nil {
 				log.Error(err)
 				continue

pkg/aggregator/store.go

@@ -1,6 +1,7 @@
 package aggregator
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
@@ -8,16 +9,16 @@ import (
 	"github.com/usefathom/fathom/pkg/models"
 )
 
-func (agg *aggregator) getSiteStats(r *results, t time.Time) (*models.SiteStats, error) {
+func (agg *aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*models.SiteStats, error) {
 	// get from map
-	date := t.Format("2006-01-02")
+	cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02"))
-	if stats, ok := r.Sites[date]; ok {
+	if stats, ok := r.Sites[cacheKey]; ok {
 		return stats, nil
 
 	}
 
 	// get from db
-	stats, err := agg.database.GetSiteStats(0, t)
+	stats, err := agg.database.GetSiteStats(siteID, t)
 	if err != nil && err != datastore.ErrNoResults {
 		return nil, err
 	}
@@ -25,54 +26,50 @@ func (agg *aggregator) getSiteStats(r *results, t time.Time) (*models.SiteStats,
 	// create in db
 	if stats == nil {
 		stats = &models.SiteStats{
+			SiteID: siteID,
 			Date:   t,
-		}
+			New:    true,
-
-		err = agg.database.InsertSiteStats(stats)
-		if err != nil {
-			return nil, err
 		}
 	}
 
-	r.Sites[date] = stats
+	r.Sites[cacheKey] = stats
 	return stats, nil
 }
 
-func (agg *aggregator) getPageStats(r *results, t time.Time, hostname string, pathname string) (*models.PageStats, error) {
+func (agg *aggregator) getPageStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.PageStats, error) {
-	date := t.Format("2006-01-02")
+	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
-	if stats, ok := r.Pages[date+hostname+pathname]; ok {
+	if stats, ok := r.Pages[cacheKey]; ok {
 		return stats, nil
 	}
 
-	stats, err := agg.database.GetPageStats(0, t, hostname, pathname)
+	stats, err := agg.database.GetPageStats(siteID, t, hostname, pathname)
 	if err != nil && err != datastore.ErrNoResults {
 		return nil, err
 	}
 
 	if stats == nil {
 		stats = &models.PageStats{
+			SiteID:   siteID,
+			New:      true,
 			Hostname: hostname,
 			Pathname: pathname,
 			Date:     t,
 		}
-		err = agg.database.InsertPageStats(stats)
+
-		if err != nil {
-			return nil, err
-		}
 	}
 
-	r.Pages[date+hostname+pathname] = stats
+	r.Pages[cacheKey] = stats
 	return stats, nil
 }
 
-func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
+func (agg *aggregator) getReferrerStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
-	date := t.Format("2006-01-02")
+	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
-	if stats, ok := r.Referrers[date+hostname+pathname]; ok {
+	if stats, ok := r.Referrers[cacheKey]; ok {
 		return stats, nil
 	}
 
 	// get from db
-	stats, err := agg.database.GetReferrerStats(0, t, hostname, pathname)
+	stats, err := agg.database.GetReferrerStats(siteID, t, hostname, pathname)
 	if err != nil && err != datastore.ErrNoResults {
 		return nil, err
 	}
@@ -80,6 +77,8 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string
 	// create in db
 	if stats == nil {
 		stats = &models.ReferrerStats{
+			SiteID:   siteID,
+			New:      true,
 			Hostname: hostname,
 			Pathname: pathname,
 			Date:     t,
@@ -90,13 +89,8 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string
 		if strings.Contains(stats.Hostname, "www.google.") {
 			stats.Group = "Google"
 		}
-
-		err = agg.database.InsertReferrerStats(stats)
-		if err != nil {
-			return nil, err
-		}
 	}
 
-	r.Referrers[date+hostname+pathname] = stats
+	r.Referrers[cacheKey] = stats
 	return stats, nil
 }

pkg/api/sites.go

@@ -37,6 +37,8 @@ func (api *API) SaveSiteHandler(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
+	// TODO: If we just created the first site, add existing data (with site_id = 0) to the site we just created
+
 	return respond(w, http.StatusOK, envelope{Data: s})
 }
 

pkg/datastore/datastore.go

@@ -27,8 +27,7 @@ type Datastore interface {
 	// site stats
 	GetSiteStats(int64, time.Time) (*models.SiteStats, error)
 	GetSiteStatsPerDay(int64, time.Time, time.Time) ([]*models.SiteStats, error)
-	InsertSiteStats(*models.SiteStats) error
+	SaveSiteStats(*models.SiteStats) error
-	UpdateSiteStats(*models.SiteStats) error
 	GetAggregatedSiteStats(int64, time.Time, time.Time) (*models.SiteStats, error)
 	GetTotalSiteViews(int64, time.Time, time.Time) (int64, error)
 	GetTotalSiteVisitors(int64, time.Time, time.Time) (int64, error)
@@ -46,15 +45,13 @@ type Datastore interface {
 
 	// page stats
 	GetPageStats(int64, time.Time, string, string) (*models.PageStats, error)
-	InsertPageStats(*models.PageStats) error
+	SavePageStats(*models.PageStats) error
-	UpdatePageStats(*models.PageStats) error
 	GetAggregatedPageStats(int64, time.Time, time.Time, int64) ([]*models.PageStats, error)
 	GetAggregatedPageStatsPageviews(int64, time.Time, time.Time) (int64, error)
 
 	// referrer stats
 	GetReferrerStats(int64, time.Time, string, string) (*models.ReferrerStats, error)
-	InsertReferrerStats(*models.ReferrerStats) error
+	SaveReferrerStats(*models.ReferrerStats) error
-	UpdateReferrerStats(*models.ReferrerStats) error
 	GetAggregatedReferrerStats(int64, time.Time, time.Time, int64) ([]*models.ReferrerStats, error)
 	GetAggregatedReferrerStatsPageviews(int64, time.Time, time.Time) (int64, error)
 

pkg/datastore/sqlstore/page_stats.go

@@ -17,13 +17,21 @@ func (db *sqlstore) GetPageStats(siteID int64, date time.Time, hostname string,
 	return stats, err
 }
 
-func (db *sqlstore) InsertPageStats(s *models.PageStats) error {
+func (db *sqlstore) SavePageStats(s *models.PageStats) error {
+	if s.New {
+		return db.insertPageStats(s)
+	}
+
+	return db.updatePageStats(s)
+}
+
+func (db *sqlstore) insertPageStats(s *models.PageStats) error {
 	query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, site_id, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
 	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
 	return err
 }
 
-func (db *sqlstore) UpdatePageStats(s *models.PageStats) error {
+func (db *sqlstore) updatePageStats(s *models.PageStats) error {
 	query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE site_id = ? AND hostname = ? AND pathname = ? AND date = ?`)
 	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
 	return err

pkg/datastore/sqlstore/referrer_stats.go

@@ -17,13 +17,21 @@ func (db *sqlstore) GetReferrerStats(siteID int64, date time.Time, hostname stri
 	return stats, err
 }
 
-func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error {
+func (db *sqlstore) SaveReferrerStats(s *models.ReferrerStats) error {
+	if s.New {
+		return db.insertReferrerStats(s)
+	}
+
+	return db.updateReferrerStats(s)
+}
+
+func (db *sqlstore) insertReferrerStats(s *models.ReferrerStats) error {
 	query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, site_id, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
 	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
 	return err
 }
 
-func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error {
+func (db *sqlstore) updateReferrerStats(s *models.ReferrerStats) error {
 	query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ?, groupname = ? WHERE site_id = ? AND hostname = ? AND pathname = ? AND date = ?`)
 	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.Hostname, s.Pathname, s.Date.Format("2006-01-02"))
 	return err

pkg/datastore/sqlstore/site_stats.go

@@ -8,7 +8,7 @@ import (
 )
 
 func (db *sqlstore) GetSiteStats(siteID int64, date time.Time) (*models.SiteStats, error) {
-	stats := &models.SiteStats{}
+	stats := &models.SiteStats{New: false}
 	query := db.Rebind(`SELECT * FROM daily_site_stats WHERE site_id = ? AND date = ? LIMIT 1`)
 	err := db.Get(stats, query, siteID, date.Format("2006-01-02"))
 	if err != nil && err == sql.ErrNoRows {
@@ -17,13 +17,21 @@ func (db *sqlstore) GetSiteStats(siteID int64, date time.Time) (*models.SiteStat
 	return stats, err
 }
 
-func (db *sqlstore) InsertSiteStats(s *models.SiteStats) error {
+func (db *sqlstore) SaveSiteStats(s *models.SiteStats) error {
-	query := db.Rebind(`INSERT INTO daily_site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?)`)
+	if s.New {
+		return db.insertSiteStats(s)
+	}
+
+	return db.updateSiteStats(s)
+}
+
+func (db *sqlstore) insertSiteStats(s *models.SiteStats) error {
+	query := db.Rebind(`INSERT INTO daily_site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
 	_, err := db.Exec(query, s.SiteID, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
 	return err
 }
 
-func (db *sqlstore) UpdateSiteStats(s *models.SiteStats) error {
+func (db *sqlstore) updateSiteStats(s *models.SiteStats) error {
 	query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4), known_durations = ? WHERE site_id = ? AND date = ?`)
 	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Date.Format("2006-01-02"))
 	return err

pkg/models/page_stats.go

@@ -5,6 +5,7 @@ import (
 )
 
 type PageStats struct {
+	New            bool      `db:"-" json:"-"`
 	SiteID         int64     `db:"site_id"`
 	Hostname       string    `db:"hostname"`
 	Pathname       string    `db:"pathname"`

pkg/models/referrer_stats.go

@@ -5,6 +5,7 @@ import (
 )
 
 type ReferrerStats struct {
+	New            bool      `db:"-" json:"-"`
 	SiteID         int64     `db:"site_id"`
 	Hostname       string    `db:"hostname"`
 	Pathname       string    `db:"pathname"`

pkg/models/site_stats.go

@@ -6,6 +6,7 @@ import (
 )
 
 type SiteStats struct {
+	New            bool      `db:"-" json:"-" `
 	SiteID         int64     `db:"site_id"`
 	Visitors       int64     `db:"visitors"`
 	Pageviews      int64     `db:"pageviews"`