package counter import ( "time" "github.com/usefathom/fathom/pkg/datastore" "github.com/usefathom/fathom/pkg/models" log "github.com/sirupsen/logrus" ) func Aggregate() error { // Get unprocessed pageviews pageviews, err := datastore.GetProcessablePageviews() if err != nil && err != datastore.ErrNoResults { return err } // Do we have anything to process? if len(pageviews) == 0 { return nil } sites := map[string]*models.SiteStats{} pages := map[string]*models.PageStats{} referrers := map[string]*models.ReferrerStats{} for _, p := range pageviews { date := p.Timestamp.Format("2006-01-02") var site *models.SiteStats var ok bool if site, ok = sites[date]; !ok { site, err = getSiteStats(p.Timestamp) if err != nil { log.Error(err) continue // TODO: Pageview should not be deleted if this happens } sites[date] = site } // site stats site.Pageviews += 1 site.AvgDuration = site.AvgDuration + ((float64(p.Duration) - site.AvgDuration) * 1 / float64(site.Pageviews)) if p.IsNewVisitor { site.Visitors += 1 } if p.IsNewSession { site.Sessions += 1 if p.IsBounce { site.BounceRate = ((float64(site.Sessions-1) * site.BounceRate) + 1) / (float64(site.Sessions)) } else { site.BounceRate = ((float64(site.Sessions-1) * site.BounceRate) + 0) / (float64(site.Sessions)) } } // page stats var pageStats *models.PageStats if pageStats, ok = pages[date+p.Pathname]; !ok { pageStats, err = getPageStats(p.Timestamp, p.Hostname, p.Pathname) if err != nil { log.Error(err) continue } pages[date+p.Hostname+p.Pathname] = pageStats } pageStats.Pageviews += 1 if p.IsUnique { pageStats.Visitors += 1 } pageStats.AvgDuration = pageStats.AvgDuration + ((float64(p.Duration) - pageStats.AvgDuration) * 1 / float64(pageStats.Pageviews)) if p.IsNewSession { pageStats.Entries += 1 if p.IsBounce { pageStats.BounceRate = ((float64(pageStats.Entries-1) * pageStats.BounceRate) + 1.00) / (float64(pageStats.Entries)) } else { pageStats.BounceRate = ((float64(pageStats.Entries-1) * pageStats.BounceRate) + 0.00) / (float64(pageStats.Entries)) } } // referrer stats if p.Referrer != "" { var referrerStats *models.ReferrerStats var ok bool if referrerStats, ok = referrers[date+p.Referrer]; !ok { referrerStats, err = getReferrerStats(p.Timestamp, p.Referrer) if err != nil { log.Error(err) continue } referrers[date+p.Referrer] = referrerStats } referrerStats.Pageviews += 1 if p.IsNewVisitor { referrerStats.Visitors += 1 } if p.IsBounce { referrerStats.BounceRate = ((float64(referrerStats.Pageviews-1) * referrerStats.BounceRate) + 1.00) / (float64(referrerStats.Pageviews)) } else { referrerStats.BounceRate = ((float64(referrerStats.Pageviews-1) * referrerStats.BounceRate) + 0.00) / (float64(referrerStats.Pageviews)) } referrerStats.AvgDuration = referrerStats.AvgDuration + ((float64(p.Duration) - referrerStats.AvgDuration) * 1 / float64(referrerStats.Pageviews)) } } // update stats for _, site := range sites { err = datastore.UpdateSiteStats(site) if err != nil { log.Error(err) } } for _, pageStats := range pages { err = datastore.UpdatePageStats(pageStats) if err != nil { log.Error(err) return err } } for _, referrerStats := range referrers { err = datastore.UpdateReferrerStats(referrerStats) if err != nil { log.Error(err) return err } } // finally, remove pageviews that we just processed err = datastore.DeletePageviews(pageviews) if err != nil { log.Error(err) } return nil } func getSiteStats(date time.Time) (*models.SiteStats, error) { stats, err := datastore.GetSiteStats(date) if err != nil && err != datastore.ErrNoResults { return nil, err } if stats != nil { return stats, nil } stats = &models.SiteStats{ Date: date, } err = datastore.InsertSiteStats(stats) return stats, err } func getPageStats(date time.Time, hostname string, pathname string) (*models.PageStats, error) { stats, err := datastore.GetPageStats(date, hostname, pathname) if err != nil && err != datastore.ErrNoResults { return nil, err } if stats != nil { return stats, nil } stats = &models.PageStats{ Hostname: hostname, Pathname: pathname, Date: date, } err = datastore.InsertPageStats(stats) return stats, err } func getReferrerStats(date time.Time, url string) (*models.ReferrerStats, error) { stats, err := datastore.GetReferrerStats(date, url) if err != nil && err != datastore.ErrNoResults { return nil, err } if stats != nil { return stats, nil } stats = &models.ReferrerStats{ URL: url, Date: date, } err = datastore.InsertReferrerStats(stats) return stats, err }