set group during aggregation

pkg/aggregator/aggregator.go

@@ -70,85 +70,22 @@ func (agg *aggregator) Process(pageviews []*models.Pageview) *results {
 	results := newResults()
 
 	for _, p := range pageviews {
-		site, err := agg.getSiteStats(results, p.Timestamp)
+		err := agg.handleSiteview(results, p)
 		if err != nil {
-			log.Error(err)
 			continue
 		}
 
-		site.Pageviews += 1
+		err = agg.handlePageview(results, p)
-
-		if p.Duration > 0.00 {
-			site.KnownDurations += 1
-			site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.KnownDurations))
-		}
-
-		if p.IsNewVisitor {
-			site.Visitors += 1
-		}
-
-		if p.IsNewSession {
-			site.Sessions += 1
-
-			if p.IsBounce {
-				site.BounceRate = ((float64(site.Sessions-1) * site.BounceRate) + 1) / (float64(site.Sessions))
-			} else {
-				site.BounceRate = ((float64(site.Sessions-1) * site.BounceRate) + 0) / (float64(site.Sessions))
-			}
-		}
-
-		pageStats, err := agg.getPageStats(results, p.Timestamp, p.Hostname, p.Pathname)
 		if err != nil {
-			log.Error(err)
 			continue
 		}
 
-		pageStats.Pageviews += 1
-		if p.IsUnique {
-			pageStats.Visitors += 1
-		}
-
-		if p.Duration > 0.00 {
-			pageStats.KnownDurations += 1
-			pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.KnownDurations))
-		}
-
-		if p.IsNewSession {
-			pageStats.Entries += 1
-
-			if p.IsBounce {
-				pageStats.BounceRate = ((float64(pageStats.Entries-1) * pageStats.BounceRate) + 1.00) / (float64(pageStats.Entries))
-			} else {
-				pageStats.BounceRate = ((float64(pageStats.Entries-1) * pageStats.BounceRate) + 0.00) / (float64(pageStats.Entries))
-			}
-		}
-
 		// referrer stats
 		if p.Referrer != "" {
-			hostname, pathname, _ := parseUrlParts(p.Referrer)
+			err := agg.handleReferral(results, p)
-			referrerStats, err := agg.getReferrerStats(results, p.Timestamp, hostname, pathname)
 			if err != nil {
-				log.Error(err)
 				continue
 			}
-
-			referrerStats.Pageviews += 1
-
-			if p.IsNewVisitor {
-				referrerStats.Visitors += 1
-			}
-
-			if p.IsBounce {
-				referrerStats.BounceRate = ((float64(referrerStats.Pageviews-1) * referrerStats.BounceRate) + 1.00) / (float64(referrerStats.Pageviews))
-			} else {
-				referrerStats.BounceRate = ((float64(referrerStats.Pageviews-1) * referrerStats.BounceRate) + 0.00) / (float64(referrerStats.Pageviews))
-			}
-
-			if p.Duration > 0.00 {
-				referrerStats.KnownDurations += 1
-				referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.KnownDurations))
-			}
-
 		}
 
 	}

pkg/aggregator/pageviews.go

@@ -0,0 +1,36 @@
+package aggregator
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/usefathom/fathom/pkg/models"
+)
+
+func (agg *aggregator) handlePageview(results *results, p *models.Pageview) error {
+	pageStats, err := agg.getPageStats(results, p.Timestamp, p.Hostname, p.Pathname)
+	if err != nil {
+		log.Error(err)
+		return err
+	}
+
+	pageStats.Pageviews += 1
+	if p.IsUnique {
+		pageStats.Visitors += 1
+	}
+
+	if p.Duration > 0.00 {
+		pageStats.KnownDurations += 1
+		pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.KnownDurations))
+	}
+
+	if p.IsNewSession {
+		pageStats.Entries += 1
+
+		if p.IsBounce {
+			pageStats.BounceRate = ((float64(pageStats.Entries-1) * pageStats.BounceRate) + 1.00) / (float64(pageStats.Entries))
+		} else {
+			pageStats.BounceRate = ((float64(pageStats.Entries-1) * pageStats.BounceRate) + 0.00) / (float64(pageStats.Entries))
+		}
+	}
+
+	return nil
+}

pkg/aggregator/referrers.go

@@ -0,0 +1,38 @@
+package aggregator
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/usefathom/fathom/pkg/models"
+)
+
+func (agg *aggregator) handleReferral(results *results, p *models.Pageview) error {
+	hostname, pathname, err := parseUrlParts(p.Referrer)
+	if err != nil {
+		return err
+	}
+
+	referrerStats, err := agg.getReferrerStats(results, p.Timestamp, hostname, pathname)
+	if err != nil {
+		log.Error(err)
+		return err
+	}
+
+	referrerStats.Pageviews += 1
+
+	if p.IsNewVisitor {
+		referrerStats.Visitors += 1
+	}
+
+	if p.IsBounce {
+		referrerStats.BounceRate = ((float64(referrerStats.Pageviews-1) * referrerStats.BounceRate) + 1.00) / (float64(referrerStats.Pageviews))
+	} else {
+		referrerStats.BounceRate = ((float64(referrerStats.Pageviews-1) * referrerStats.BounceRate) + 0.00) / (float64(referrerStats.Pageviews))
+	}
+
+	if p.Duration > 0.00 {
+		referrerStats.KnownDurations += 1
+		referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.KnownDurations))
+	}
+
+	return nil
+}

pkg/aggregator/siteviews.go

@@ -0,0 +1,37 @@
+package aggregator
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/usefathom/fathom/pkg/models"
+)
+
+func (agg *aggregator) handleSiteview(results *results, p *models.Pageview) error {
+	site, err := agg.getSiteStats(results, p.Timestamp)
+	if err != nil {
+		log.Error(err)
+		return err
+	}
+
+	site.Pageviews += 1
+
+	if p.Duration > 0.00 {
+		site.KnownDurations += 1
+		site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.KnownDurations))
+	}
+
+	if p.IsNewVisitor {
+		site.Visitors += 1
+	}
+
+	if p.IsNewSession {
+		site.Sessions += 1
+
+		if p.IsBounce {
+			site.BounceRate = ((float64(site.Sessions-1) * site.BounceRate) + 1) / (float64(site.Sessions))
+		} else {
+			site.BounceRate = ((float64(site.Sessions-1) * site.BounceRate) + 0) / (float64(site.Sessions))
+		}
+	}
+
+	return nil
+}

pkg/aggregator/store.go

@@ -1,6 +1,7 @@
 package aggregator
 
 import (
+	"strings"
 	"time"
 
 	"github.com/usefathom/fathom/pkg/datastore"
@@ -64,7 +65,6 @@ func (agg *aggregator) getPageStats(r *results, t time.Time, hostname string, pa
 	return stats, nil
 }
 
-// TODO: Set referrers group here.
 func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
 	date := t.Format("2006-01-02")
 	if stats, ok := r.Referrers[date+hostname+pathname]; ok {
@@ -83,7 +83,14 @@ func (agg *aggregator) getReferrerStats(r *results, t time.Time, hostname string
 			Hostname: hostname,
 			Pathname: pathname,
 			Date:     t,
+			Group:    "",
 		}
+
+		// TODO: Abstract this
+		if strings.Contains(stats.Hostname, "www.google.") {
+			stats.Group = "Google"
+		}
+
 		err = agg.database.InsertReferrerStats(stats)
 		if err != nil {
 			return nil, err