[wip] move bulk of storage to client-side for improved scalability, simplicity and privacy. see #14.

assets/js/tracker.js

@@ -1,13 +1,6 @@
 'use strict';
 
-/* cookies.js - https://github.com/ScottHamper/Cookies */
+var cookies = require('cookies-js');
-(function(d,f){"use strict";var h=function(d){if("object"!==typeof d.document)throw Error("Cookies.js requires a `window` with a `document` object");var b=function(a,e,c){return 1===arguments.length?b.get(a):b.set(a,e,c)};b._document=d.document;b._cacheKeyPrefix="cookey.";b._maxExpireDate=new Date("Fri, 31 Dec 9999 23:59:59 UTC");b.defaults={path:"/",secure:!1};b.get=function(a){b._cachedDocumentCookie!==b._document.cookie&&b._renewCache();a=b._cache[b._cacheKeyPrefix+a];return a===f?f:decodeURIComponent(a)};
-b.set=function(a,e,c){c=b._getExtendedOptions(c);c.expires=b._getExpiresDate(e===f?-1:c.expires);b._document.cookie=b._generateCookieString(a,e,c);return b};b.expire=function(a,e){return b.set(a,f,e)};b._getExtendedOptions=function(a){return{path:a&&a.path||b.defaults.path,domain:a&&a.domain||b.defaults.domain,expires:a&&a.expires||b.defaults.expires,secure:a&&a.secure!==f?a.secure:b.defaults.secure}};b._isValidDate=function(a){return"[object Date]"===Object.prototype.toString.call(a)&&!isNaN(a.getTime())};
-b._getExpiresDate=function(a,e){e=e||new Date;"number"===typeof a?a=Infinity===a?b._maxExpireDate:new Date(e.getTime()+1E3*a):"string"===typeof a&&(a=new Date(a));if(a&&!b._isValidDate(a))throw Error("`expires` parameter cannot be converted to a valid Date instance");return a};b._generateCookieString=function(a,b,c){a=a.replace(/[^#$&+\^`|]/g,encodeURIComponent);a=a.replace(/\(/g,"%28").replace(/\)/g,"%29");b=(b+"").replace(/[^!#$&-+\--:<-\[\]-~]/g,encodeURIComponent);c=c||{};a=a+"="+b+(c.path?";path="+
-c.path:"");a+=c.domain?";domain="+c.domain:"";a+=c.expires?";expires="+c.expires.toUTCString():"";return a+=c.secure?";secure":""};b._getCacheFromString=function(a){var e={};a=a?a.split("; "):[];for(var c=0;c<a.length;c++){var d=b._getKeyValuePairFromCookieString(a[c]);e[b._cacheKeyPrefix+d.key]===f&&(e[b._cacheKeyPrefix+d.key]=d.value)}return e};b._getKeyValuePairFromCookieString=function(a){var b=a.indexOf("="),b=0>b?a.length:b,c=a.substr(0,b),d;try{d=decodeURIComponent(c)}catch(k){console&&"function"===
-typeof console.error&&console.error('Could not decode cookie with key "'+c+'"',k)}return{key:d,value:a.substr(b+1)}};b._renewCache=function(){b._cache=b._getCacheFromString(b._document.cookie);b._cachedDocumentCookie=b._document.cookie};b._areEnabled=function(){var a="1"===b.set("cookies.js",1).get("cookies.js");b.expire("cookies.js");return a};b.enabled=b._areEnabled();return b},g=d&&"object"===typeof d.document?h(d):h;"function"===typeof define&&define.amd?define(function(){return g}):"object"===
-typeof exports?("object"===typeof module&&"object"===typeof module.exports&&(exports=module.exports=g),exports.Cookies=g):d.Cookies=g})("undefined"===typeof window?this:window);
-
 var queue = window.fathom.q || [];
 var trackerUrl = '';
 var commands = {
@@ -19,11 +12,6 @@ var commands = {
 function stringifyObject(json) {
   var keys = Object.keys(json);
 
-  // omit empty
-  keys = keys.filter(function(k) {
-    return json[k].length > 0;
-  });
-
   return '?' +
       keys.map(function(k) {
           return encodeURIComponent(k) + '=' +
@@ -36,6 +24,25 @@ function generateKey() {
   return Array(16).join().split(',').map(function() { return s.charAt(Math.floor(Math.random() * s.length)); }).join('');
 }
 
+function getData() {
+  var data = Cookies.get('_fathom');
+
+  if(data) {
+    try{
+      data = JSON.parse(data);
+      return data;
+    } catch(e) {
+      console.error(e);
+    }
+  }
+
+  return {
+    sid: generateKey(),
+    isNew: true, 
+    pagesViewed: [],
+  }
+}
+
 function setTrackerUrl(v) {
   trackerUrl = v;
 }
@@ -51,12 +58,6 @@ function trackPageview() {
     return;
   }
 
-  // set cookie with random visitor key, valid for 24 hours.
-  // visitor can delete this cookie in order to be forgotten
-  if(!Cookies.get('_fathom')) {
-    Cookies.set('_fathom', generateKey(), { expires: 60 * 60 * 24 });
-  }
-
   // get the path or canonical
   var path = location.pathname + location.search;
 
@@ -68,21 +69,31 @@ function trackPageview() {
     path = a.pathname;
   }
 
+  let referrer = '';
+  if(document.referrer.indexOf(location.hostname) < 0) {
+    referrer = document.referrer;
+  }
+
+  let data = getData();
+
   var d = {
-    vk: Cookies.get('_fathom'),
+    sid: data.sid,
-    h: location.hostname,
-    t: document.title,
-    l: navigator.language,
     p: path,
-    sr: screen.width + "x" + screen.height,
     t: document.title,
-    ru: document.referrer,
+    r: referrer,
-    rk: "",
     scheme: location.protocol.substring(0, location.protocol.length - 1),
+    u: data.pagesViewed.indexOf(path) == -1 ? 1 : 0,
+    b: data.isNew ? 1 : 0, // because only new visitors can bounce. we update this server-side.
+    n: data.isNew ? 1 : 0, 
   };
 
   var i = document.createElement('img');
   i.src = trackerUrl + stringifyObject(d);
+  i.addEventListener('load', function() {
+    data.pagesViewed.push(path);
+    data.isNew = false;
+    Cookies.set('_fathom', JSON.stringify(data), { expires: 60 * 60 * 24});
+  });
   document.body.appendChild(i);
 }
 

fathom.go

@@ -1,13 +1,13 @@
 package main
 
 import (
+	"os"
+
 	"github.com/joho/godotenv"
-	"github.com/robfig/cron"
 	"github.com/usefathom/fathom/pkg/commands"
-	"github.com/usefathom/fathom/pkg/count"
+	"github.com/usefathom/fathom/pkg/counter"
 	"github.com/usefathom/fathom/pkg/datastore"
 	"gopkg.in/alecthomas/kingpin.v2"
-	"os"
 
 	"github.com/kelseyhightower/envconfig"
 	log "github.com/sirupsen/logrus"
@@ -50,11 +50,6 @@ func main() {
 	db := datastore.Init(dbcfg.Driver, dbcfg.Host, dbcfg.Name, dbcfg.User, dbcfg.Password)
 	defer db.Close()
 
-	// setup cron to run count.Archive every hour
-	c := cron.New()
-	c.AddFunc("@hourly", count.Archive)
-	c.Start()
-
 	// parse & run cli commands
 	app.Version("1.0")
 	app.UsageTemplate(kingpin.CompactUsageTemplate)
@@ -66,8 +61,10 @@ func main() {
 		commands.Server(*serverPort, *serverWebRoot)
 
 	case "archive":
-		commands.Archive()
+		err := counter.Aggregate()
-
+		if err != nil {
+			log.Warn(err)
+		}
 	}
 
 }

gulpfile.js

@@ -10,9 +10,10 @@ const gutil = require('gulp-util')
 const sass = require('gulp-sass')
 const uglify = require('gulp-uglify')
 const pump = require('pump')
+const es = require('event-stream');
 const debug = process.env.NODE_ENV !== 'production';
 
-let defaultTasks = [ 'browserify', 'sass', 'tracker', 'html', 'img' ] ;
+let defaultTasks = [ 'browserify', 'sass', 'html', 'img' ] ;
 if( ! debug ) {
   defaultTasks.push( 'minify' );
 }
@@ -20,29 +21,33 @@ if( ! debug ) {
 gulp.task('default', defaultTasks);
 
 gulp.task('browserify', function () {
-  return browserify({
+  let files = [
-      entries: './assets/js/script.js',
+    './assets/js/script.js',
-      debug: debug
+    './assets/js/tracker.js',
-  })
+  ];
-  .transform("babelify", {
+
-    presets: ["es2015"],
+  var tasks = files.map(function(entry) {
-    plugins: [ 
+      return browserify({
-      "transform-decorators-legacy", 
+            entries: entry,
-      ["transform-react-jsx", { "pragma":"h" } ] 
+            debug: debug
-    ]
+        })
-  })
+        .transform("babelify", {
-  .bundle()
+          presets: ["es2015"],
-  .on('error', function(err){
+          plugins: [ 
-    console.log(err.message);
+            "transform-decorators-legacy", 
-    this.emit('end');
+            ["transform-react-jsx", { "pragma":"h" } ] 
-  })
+          ]
-  .pipe(source('script.js'))
+        })
-  .pipe(buffer())
+        .bundle()  
-  .pipe(gulp.dest('./build/js/'))
+        .pipe(source(entry.split('/').pop()))
+        .pipe(gulp.dest('./build/js/'))
+      });
+      // create a merged stream
+      return es.merge.apply(null, tasks);
 });
 
 gulp.task('minify', function(cb) {
-  process.env.NODE_ENV = 'production';
+  process.env.NODE_ENV = 'production'; // why is this here?
 
   pump([
     gulp.src('./build/js/*.js'),
@@ -61,11 +66,6 @@ gulp.task('html', function() {
     .pipe(gulp.dest('./build'))
 });
 
-gulp.task('tracker', function() {
-  return gulp.src('./assets/js/tracker.js')
-    .pipe(gulp.dest('./build/js'))
-});
-
 gulp.task('sass', function () {
 	var files = './assets/sass/[^_]*.scss';
 	return gulp.src(files)
@@ -76,7 +76,7 @@ gulp.task('sass', function () {
 });
 
 gulp.task('watch', ['default'], function() {
-  gulp.watch(['./assets/js/**/*.js'], ['browserify', 'tracker'] );
+  gulp.watch(['./assets/js/**/*.js'], ['browserify'] );
   gulp.watch(['./assets/sass/**/**/*.scss'], ['sass'] );
   gulp.watch(['./assets/**/*.html'], ['html'] );
   gulp.watch(['./assets/img/**/*'], ['img'] );

package.json

@@ -1,8 +1,8 @@
 {
   "license": "MIT",
-  "repository": { 
+  "repository": {
-    "type" : "git"
+    "type": "git",
-  , "url" : "https://github.com/usefathom/fathom.git"
+    "url": "https://github.com/usefathom/fathom.git"
   },
   "devDependencies": {
     "babel-core": "^6.26.0",
@@ -11,6 +11,7 @@
     "babel-preset-es2015": "^6.18.0",
     "babelify": "^8.0.0",
     "browserify": "^16.2.0",
+    "event-stream": "^3.3.4",
     "gulp": "^3.9.1",
     "gulp-rename": "^1.2.2",
     "gulp-sass": "^4.0.1",
@@ -19,6 +20,7 @@
     "vinyl-source-stream": "^2.0.0"
   },
   "dependencies": {
+    "cookies-js": "^1.2.3",
     "d3": "^5.1.0",
     "d3-tip": "^0.7.1",
     "decko": "^1.2.0",

pkg/api/bounces.go

@@ -1,18 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-)
-
-// URL: /api/bounces/count
-var GetBouncesCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	result, err := datastore.TotalBounces(before, after)
-	if err != nil {
-		return err
-	}
-
-	return respond(w, envelope{Data: result})
-})

pkg/api/browsers.go

@@ -1,16 +0,0 @@
-package api
-
-import (
-	"github.com/usefathom/fathom/pkg/count"
-	"net/http"
-)
-
-// URL: /api/browsers
-var GetBrowsersHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	results, err := count.Browsers(before, after, getRequestedLimit(r))
-	if err != nil {
-		return err
-	}
-	return respond(w, envelope{Data: results})
-})

pkg/api/collect.go

@@ -1,11 +1,8 @@
 package api
 
 import (
-	"crypto/md5"
 	"encoding/base64"
-	"encoding/hex"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/mssola/user_agent"
@@ -15,13 +12,13 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-var buffer []*models.Pageview
+var buffer []*models.RawPageview
-var bufferSize = 250
+var bufferSize = 50
 var timeout = 200 * time.Millisecond
 
 func persistPageviews() {
 	if len(buffer) > 0 {
-		err := datastore.SavePageviews(buffer)
+		err := datastore.SaveRawPageviews(buffer)
 		if err != nil {
 			log.Errorf("error saving pageviews: %s", err)
 		}
@@ -31,7 +28,7 @@ func persistPageviews() {
 	}
 }
 
-func processBuffer(pv chan *models.Pageview) {
+func processBuffer(pv chan *models.RawPageview) {
 	for {
 		select {
 		case pageview := <-pv:
@@ -47,11 +44,10 @@ func processBuffer(pv chan *models.Pageview) {
 
 /* middleware */
 func NewCollectHandler() http.Handler {
-	pageviews := make(chan *models.Pageview, bufferSize)
+	pageviews := make(chan *models.RawPageview, bufferSize)
 	go processBuffer(pageviews)
 
 	return HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-
 		// abort if this is a bot.
 		userAgent := r.UserAgent()
 		ua := user_agent.New(userAgent)
@@ -60,96 +56,26 @@ func NewCollectHandler() http.Handler {
 		}
 
 		q := r.URL.Query()
-
-		// find page
-		page, err := datastore.GetPageByHostnameAndPath(q.Get("h"), q.Get("p"))
-		if err != nil && err != datastore.ErrNoResults {
-			return err
-		}
-
-		// page does not exist yet, get details & save it
-		if page == nil {
-			page = &models.Page{
-				Scheme:   "http",
-				Hostname: q.Get("h"),
-				Path:     q.Get("p"),
-				Title:    q.Get("t"),
-			}
-
-			if scheme := q.Get("scheme"); scheme != "" {
-				page.Scheme = scheme
-			}
-
-			err = datastore.SavePage(page)
-			if err != nil {
-				return err
-			}
-		}
-
-		// find visitor by anonymized key from query params
 		now := time.Now()
-		visitorKey := q.Get("vk")
-		visitorKey = enhanceVisitorKey(visitorKey, now.Format("2006-01-02"), userAgent, q.Get("l"), q.Get("sr"))
-		visitor, err := datastore.GetVisitorByKey(visitorKey)
-		if err != nil && err != datastore.ErrNoResults {
-			return err
-		}
-
-		// visitor is new, save it
-		if visitor == nil {
-			visitor = &models.Visitor{
-				BrowserLanguage:  q.Get("l"),
-				ScreenResolution: q.Get("sr"),
-				DeviceOS:         ua.OS(),
-				Country:          "",
-				Key:              visitorKey,
-			}
-
-			// add browser details
-			visitor.BrowserName, visitor.BrowserVersion = ua.Browser()
-
-			// get rid of exact browser versions
-			visitor.BrowserVersion = parseMajorMinor(visitor.BrowserVersion)
-			err = datastore.SaveVisitor(visitor)
-			if err != nil {
-				return err
-			}
-		} else {
-			lastPageview, err := datastore.GetLastPageviewForVisitor(visitor.ID)
-			if err != nil && err != datastore.ErrNoResults {
-				return err
-			}
-
-			if lastPageview != nil && lastPageview.Timestamp.After(now.Add(-30*time.Minute)) {
-				lastPageview.Bounced = false
-				lastPageview.TimeOnPage = now.Unix() - lastPageview.Timestamp.Unix()
-
-				// TODO: Delay storage until in buffer?
-				err := datastore.UpdatePageview(lastPageview)
-				if err != nil {
-					return err
-				}
-			}
-		}
 
 		// get pageview details
-		pageview := &models.Pageview{
+		pageview := &models.RawPageview{
-			PageID:          page.ID,
+			SessionID:    q.Get("sid"),
-			VisitorID:       visitor.ID,
+			Pathname:     q.Get("p"),
-			ReferrerUrl:     q.Get("ru"),
+			IsNewVisitor: q.Get("n") == "1",
-			ReferrerKeyword: q.Get("rk"),
+			IsUnique:     q.Get("u") == "1",
-			TimeOnPage:      0,
+			IsBounce:     q.Get("b") != "0",
-			Bounced:         true, // TODO: Only mark as bounced if no other pageviews in this session
+			Referrer:     q.Get("r"),
-			Timestamp:       now,
+			Duration:     0,
+			Timestamp:    now,
 		}
 
-		// only store referrer URL if not coming from own site
+		err := datastore.SaveRawPageview(pageview)
-		if strings.Contains(pageview.ReferrerUrl, page.Hostname) {
+		if err != nil {
-			pageview.ReferrerUrl = ""
+			return err
 		}
-
 		// push onto channel
-		pageviews <- pageview
+		//pageviews <- pageview
 
 		// don't you cache this
 		w.Header().Set("Content-Type", "image/gif")
@@ -164,9 +90,3 @@ func NewCollectHandler() http.Handler {
 		return nil
 	})
 }
-
-// generateVisitorKey generates the "unique" visitor key from date, user agent + screen resolution
-func enhanceVisitorKey(key string, date string, userAgent string, lang string, screenRes string) string {
-	byteKey := md5.Sum([]byte(date + userAgent + lang + screenRes))
-	return hex.EncodeToString(byteKey[:])
-}

pkg/api/languages.go

@@ -1,18 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/usefathom/fathom/pkg/count"
-)
-
-// URL: /api/languages
-var GetLanguagesHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	results, err := count.Languages(before, after, getRequestedLimit(r))
-	if err != nil {
-		return err
-	}
-
-	return respond(w, envelope{Data: results})
-})

pkg/api/page_stats.go

@@ -0,0 +1,14 @@
+package api
+
+import (
+	"github.com/usefathom/fathom/pkg/models"
+	"net/http"
+)
+
+// URL: /api/stats/page
+var GetPageStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+	//before, after := getRequestedPeriods(r)
+	//limit := getRequestedLimit(r)
+	var result []*models.PageStats
+	return respond(w, envelope{Data: result})
+})

pkg/api/pageviews.go

@@ -1,42 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/usefathom/fathom/pkg/count"
-	"github.com/usefathom/fathom/pkg/datastore"
-)
-
-// URL: /api/pageviews
-var GetPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	limit := getRequestedLimit(r)
-	results, err := count.Pageviews(before, after, limit)
-	if err != nil {
-		return err
-	}
-
-	return respond(w, envelope{Data: results})
-})
-
-// URL: /api/pageviews/count
-var GetPageviewsCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	result, err := datastore.TotalPageviews(before, after)
-	if err != nil {
-		return err
-	}
-
-	return respond(w, envelope{Data: result})
-})
-
-// URL: /api/pageviews/group/day
-var GetPageviewsPeriodCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	results, err := datastore.TotalPageviewsPerDay(before, after)
-	if err != nil {
-		return err
-	}
-
-	return respond(w, envelope{Data: results})
-})

pkg/api/params.go

@@ -7,13 +7,15 @@ import (
 	"time"
 )
 
+// TODO: Move params into Params struct (with defaults)
+
 const defaultPeriod = 7
 const defaultLimit = 10
 
 func getRequestedLimit(r *http.Request) int64 {
 	limit, err := strconv.ParseInt(r.URL.Query().Get("limit"), 10, 64)
 	if err != nil || limit == 0 {
-		limit = 10
+		limit = defaultLimit
 	}
 
 	return limit
@@ -30,7 +32,7 @@ func getRequestedPeriods(r *http.Request) (int64, int64) {
 
 	after, err = strconv.ParseInt(r.URL.Query().Get("after"), 10, 64)
 	if err != nil || before == 0 {
-		after = time.Now().AddDate(0, 0, -7).Unix()
+		after = time.Now().AddDate(0, 0, -defaultPeriod).Unix()
 	}
 
 	return before, after

pkg/api/referrer_stats.go

@@ -0,0 +1,14 @@
+package api
+
+import (
+	"github.com/usefathom/fathom/pkg/models"
+	"net/http"
+)
+
+// URL: /api/stats/referrer
+var GetReferrerStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+	// before, after := getRequestedPeriods(r)
+	// limit := getRequestedLimit(r)
+	var result []*models.ReferrerStats
+	return respond(w, envelope{Data: result})
+})

pkg/api/referrers.go

@@ -1,17 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/usefathom/fathom/pkg/count"
-)
-
-// URL: /api/referrers
-var GetReferrersHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	results, err := count.Referrers(before, after, getRequestedLimit(r))
-	if err != nil {
-		return err
-	}
-	return respond(w, envelope{Data: results})
-})

pkg/api/routes.go

@@ -0,0 +1,20 @@
+package api
+
+import (
+	"github.com/gorilla/mux"
+	"net/http"
+)
+
+func Routes(webroot string) *mux.Router {
+	// register routes
+	r := mux.NewRouter()
+	r.Handle("/collect", NewCollectHandler()).Methods(http.MethodGet)
+	r.Handle("/api/session", LoginHandler).Methods(http.MethodPost)
+	r.Handle("/api/session", LogoutHandler).Methods(http.MethodDelete)
+
+	r.Handle("/api/stats/page", Authorize(GetPageStatsHandler)).Methods(http.MethodGet)
+
+	r.Path("/tracker.js").Handler(http.FileServer(http.Dir(webroot + "/js/")))
+	r.PathPrefix("/").Handler(http.FileServer(http.Dir(webroot)))
+	return r
+}

pkg/api/screens.go

@@ -1,17 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/usefathom/fathom/pkg/count"
-)
-
-// URL: /api/screen-resolutions
-var GetScreenResolutionsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	results, err := count.Screens(before, after, getRequestedLimit(r))
-	if err != nil {
-		return err
-	}
-	return respond(w, envelope{Data: results})
-})

pkg/api/site_stats.go

@@ -0,0 +1,44 @@
+package api
+
+import (
+	"github.com/usefathom/fathom/pkg/models"
+	"net/http"
+)
+
+// URL: /api/stats/site
+var GetSiteStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+	// before, after := getRequestedPeriods(r)
+	// limit := getRequestedLimit(r)
+
+	var results []*models.SiteStats
+	return respond(w, envelope{Data: results})
+})
+
+// URL: /api/stats/site/pageviews
+var GetSiteStatsPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+	// before, after := getRequestedPeriods(r)
+	// limit := getRequestedLimit(r)
+	var result int
+	return respond(w, envelope{Data: result})
+})
+
+// URL: /api/stats/site/visitors
+var GetSiteStatsVisitorsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+	// before, after := getRequestedPeriods(r)
+	var result int
+	return respond(w, envelope{Data: result})
+})
+
+// URL: /api/stats/site/avg-duration
+var GetSiteStatsAvgDurationHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+	// before, after := getRequestedPeriods(r)
+	var result int
+	return respond(w, envelope{Data: result})
+})
+
+// URL: /api/stats/site/avg-bounce
+var GetSiteStatusAvgBounceHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+	// before, after := getRequestedPeriods(r)
+	var result int
+	return respond(w, envelope{Data: result})
+})

pkg/api/time.go

@@ -1,20 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-)
-
-// TODO: Come up with more consistent URL names.
-// URL: /api/time-on-site/count
-var GetTimeOnSiteCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	result, err := datastore.AvgTimeOnSite(before, after)
-
-	if err != nil {
-		return err
-	}
-
-	return respond(w, envelope{Data: result})
-})

pkg/api/visitors.go

@@ -1,38 +0,0 @@
-package api
-
-import (
-	"net/http"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-)
-
-// URL: /api/visitors/count
-var GetVisitorsCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	result, err := datastore.TotalVisitors(before, after)
-	if err != nil {
-		return err
-	}
-
-	return respond(w, envelope{Data: result})
-})
-
-// URL: /api/visitors/count/realtime
-var GetVisitorsRealtimeCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	result, err := datastore.RealtimeVisitors()
-	if err != nil {
-		return err
-	}
-	return respond(w, envelope{Data: result})
-})
-
-// URL: /api/visitors/count/group/:period
-var GetVisitorsPeriodCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-	before, after := getRequestedPeriods(r)
-	results, err := datastore.TotalVisitorsPerDay(before, after)
-	if err != nil {
-		return err
-	}
-
-	return respond(w, envelope{Data: results})
-})

pkg/commands/archive.go

@@ -1,10 +0,0 @@
-package commands
-
-import (
-	"github.com/usefathom/fathom/pkg/count"
-)
-
-// Archive processes unarchived data (pageviews to aggregated count tables)
-func Archive() {
-	count.Archive()
-}

pkg/commands/server.go

@@ -2,40 +2,21 @@ package commands
 
 import (
 	"fmt"
-	"github.com/gorilla/handlers"
-	"github.com/gorilla/mux"
-	"github.com/usefathom/fathom/pkg/api"
 	"log"
 	"net/http"
 	"os"
+
+	"github.com/gorilla/handlers"
+	"github.com/usefathom/fathom/pkg/api"
 )
 
 // Server starts the HTTP server, listening on the given port
 func Server(port int, webroot string) {
-
+	r := api.Routes(webroot)
-	// register routes
-	r := mux.NewRouter()
-	r.Handle("/collect", api.NewCollectHandler()).Methods("GET")
-	r.Handle("/api/session", api.LoginHandler).Methods("POST")
-	r.Handle("/api/session", api.LogoutHandler).Methods("DELETE")
-	r.Handle("/api/visitors/count", api.Authorize(api.GetVisitorsCountHandler)).Methods("GET")
-	r.Handle("/api/visitors/count/group/{period}", api.Authorize(api.GetVisitorsPeriodCountHandler)).Methods("GET")
-	r.Handle("/api/visitors/count/realtime", api.Authorize(api.GetVisitorsRealtimeCountHandler)).Methods("GET")
-	r.Handle("/api/pageviews/count", api.Authorize(api.GetPageviewsCountHandler)).Methods("GET")
-	r.Handle("/api/pageviews/count/group/{period}", api.Authorize(api.GetPageviewsPeriodCountHandler)).Methods("GET")
-	r.Handle("/api/pageviews", api.Authorize(api.GetPageviewsHandler)).Methods("GET")
-	r.Handle("/api/languages", api.Authorize(api.GetLanguagesHandler)).Methods("GET")
-	r.Handle("/api/referrers", api.Authorize(api.GetReferrersHandler)).Methods("GET")
-	r.Handle("/api/screen-resolutions", api.Authorize(api.GetScreenResolutionsHandler)).Methods("GET")
-	//r.Handle("/api/countries", api.Authorize(api.GetCountriesHandler)).Methods("GET")
-	r.Handle("/api/browsers", api.Authorize(api.GetBrowsersHandler)).Methods("GET")
-	r.Handle("/api/bounces/count", api.Authorize(api.GetBouncesCountHandler)).Methods("GET")
-	r.Handle("/api/time-on-site/count", api.Authorize(api.GetTimeOnSiteCountHandler)).Methods("GET")
-
-	r.Path("/tracker.js").Handler(http.FileServer(http.Dir(webroot + "/js/")))
-	r.PathPrefix("/").Handler(http.FileServer(http.Dir(webroot)))
-
 	log.Printf("Now serving %s on port %d/\n", webroot, port)
+
 	err := http.ListenAndServe(fmt.Sprintf(":%d", port), handlers.LoggingHandler(os.Stdout, r))
-	log.Println(err)
+	if err != nil {
+		log.Println(err)
+	}
 }

pkg/count/bounces.go

@@ -1,22 +0,0 @@
-package count
-
-import (
-	"log"
-	"time"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-)
-
-// CreateBouncesTotals aggregates pageview data for each page into daily totals
-func CreateBouncesTotals(since string) {
-	tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
-	totals, err := datastore.BouncesCountPerPageAndDay(tomorrow, since)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = datastore.SavePageTotals("bounced", totals)
-	if err != nil {
-		log.Fatal(err)
-	}
-}

pkg/count/browsers.go

@@ -1,39 +0,0 @@
-package count
-
-import (
-	"log"
-	"time"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-// Browsers returns a point slice containing browser data per browser name
-func Browsers(before int64, after int64, limit int64) ([]*models.Total, error) {
-	points, err := datastore.TotalsPerBrowser(before, after, limit)
-	if err != nil {
-		return nil, err
-	}
-
-	total, err := datastore.TotalBrowsers(before, after)
-	if err != nil {
-		return nil, err
-	}
-
-	points = calculatePercentagesOfTotal(points, total)
-	return points, nil
-}
-
-// CreateBrowserTotals aggregates screen data into daily totals
-func CreateBrowserTotals(since string) {
-	tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
-	totals, err := datastore.BrowserCountPerDay(tomorrow, since)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = datastore.SaveTotals("browser_names", totals)
-	if err != nil {
-		log.Fatal(err)
-	}
-}

pkg/count/count.go

@@ -1,46 +0,0 @@
-package count
-
-import (
-	"time"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-func getLastArchivedDate() string {
-	value, _ := datastore.GetOption("last_archived")
-	if value == "" {
-		return time.Now().AddDate(-1, 0, 0).Format("2006-01-02")
-	}
-
-	return value
-}
-
-// Archive aggregates data into daily totals
-func Archive() {
-	start := time.Now()
-
-	lastArchived := getLastArchivedDate()
-	CreatePageviewTotals(lastArchived)
-	CreateVisitorTotals(lastArchived)
-	CreateScreenTotals(lastArchived)
-	CreateLanguageTotals(lastArchived)
-	CreateBrowserTotals(lastArchived)
-	CreateReferrerTotals(lastArchived)
-	CreateBouncesTotals(lastArchived)
-	datastore.SetOption("last_archived", time.Now().Format("2006-01-02"))
-
-	end := time.Now()
-	log.Infof("finished aggregating metrics. ran for %dms.", (end.UnixNano()-start.UnixNano())/1000000)
-}
-
-func calculatePercentagesOfTotal(totals []*models.Total, total int) []*models.Total {
-	// calculate percentage values for each point
-	for _, p := range totals {
-		p.PercentageOfTotal = float64(p.Count) / float64(total) * 100.00
-	}
-
-	return totals
-}

pkg/count/count_test.go

@@ -1,22 +0,0 @@
-package count
-
-import (
-	"testing"
-
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-func TestCalculatePointPercentages(t *testing.T) {
-	totals := []*models.Total{
-		&models.Total{
-			Value: "Foo",
-			Count: 5,
-		},
-	}
-
-	totals = calculatePercentagesOfTotal(totals, 100)
-
-	if totals[0].PercentageOfTotal != 5.00 {
-		t.Errorf("Percentage value should be 5.00, is %.2f", totals[0].PercentageOfTotal)
-	}
-}

pkg/count/languages.go

@@ -1,39 +0,0 @@
-package count
-
-import (
-	"log"
-	"time"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-// Languages returns a point slice containing language data per language
-func Languages(before int64, after int64, limit int64) ([]*models.Total, error) {
-	points, err := datastore.TotalsPerLanguage(before, after, limit)
-	if err != nil {
-		return nil, err
-	}
-
-	total, err := datastore.TotalLanguages(before, after)
-	if err != nil {
-		return nil, err
-	}
-
-	points = calculatePercentagesOfTotal(points, total)
-	return points, nil
-}
-
-// CreateLanguageTotals aggregates screen data into daily totals
-func CreateLanguageTotals(since string) {
-	tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
-	totals, err := datastore.LanguageCountPerDay(tomorrow, since)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = datastore.SaveTotals("browser_languages", totals)
-	if err != nil {
-		log.Fatal(err)
-	}
-}

pkg/count/pageviews.go

@@ -1,39 +0,0 @@
-package count
-
-import (
-	"log"
-	"time"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-// Pageviews returns a point slice containing language data per language
-func Pageviews(before int64, after int64, limit int64) ([]*models.Total, error) {
-	points, err := datastore.TotalPageviewsPerPage(before, after, limit)
-	if err != nil {
-		return nil, err
-	}
-
-	total, err := datastore.TotalPageviews(before, after)
-	if err != nil {
-		return nil, err
-	}
-
-	points = calculatePercentagesOfTotal(points, total)
-	return points, nil
-}
-
-// CreatePageviewTotals aggregates pageview data for each page into daily totals
-func CreatePageviewTotals(since string) {
-	tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
-	totals, err := datastore.PageviewCountPerPageAndDay(tomorrow, since)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = datastore.SavePageTotals("pageviews", totals)
-	if err != nil {
-		log.Fatal(err)
-	}
-}

pkg/count/referrers.go

@@ -1,39 +0,0 @@
-package count
-
-import (
-	"log"
-	"time"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-// Referrers returns a point slice containing browser data per browser name
-func Referrers(before int64, after int64, limit int64) ([]*models.Total, error) {
-	points, err := datastore.TotalsPerReferrer(before, after, limit)
-	if err != nil {
-		return nil, err
-	}
-
-	total, err := datastore.TotalReferrers(before, after)
-	if err != nil {
-		return nil, err
-	}
-
-	points = calculatePercentagesOfTotal(points, total)
-	return points, nil
-}
-
-// CreateReferrerTotals aggregates screen data into daily totals
-func CreateReferrerTotals(since string) {
-	tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
-	totals, err := datastore.ReferrerCountPerDay(tomorrow, since)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = datastore.SaveTotals("referrers", totals)
-	if err != nil {
-		log.Fatal(err)
-	}
-}

pkg/count/screens.go

@@ -1,39 +0,0 @@
-package count
-
-import (
-	"log"
-	"time"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-// Screens returns a point slice containing screen data per size
-func Screens(before int64, after int64, limit int64) ([]*models.Total, error) {
-	points, err := datastore.TotalsPerScreen(before, after, limit)
-	if err != nil {
-		return nil, err
-	}
-
-	total, err := datastore.TotalScreens(before, after)
-	if err != nil {
-		return nil, err
-	}
-
-	points = calculatePercentagesOfTotal(points, total)
-	return points, nil
-}
-
-// CreateScreenTotals aggregates screen data into daily totals
-func CreateScreenTotals(since string) {
-	tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
-	totals, err := datastore.ScreenCountPerDay(tomorrow, since)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = datastore.SaveTotals("screens", totals)
-	if err != nil {
-		log.Fatal(err)
-	}
-}

pkg/count/visitors.go

@@ -1,23 +0,0 @@
-package count
-
-import (
-	"log"
-	"time"
-
-	"github.com/usefathom/fathom/pkg/datastore"
-)
-
-// CreateVisitorTotals aggregates visitor data into daily totals
-func CreateVisitorTotals(since string) {
-	tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
-	totals, err := datastore.VisitorCountPerDay(tomorrow, since)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = datastore.SaveVisitorTotals(totals)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-}

pkg/counter/counter.go

@@ -0,0 +1,71 @@
+package counter
+
+import (
+	"time"
+
+	"github.com/usefathom/fathom/pkg/datastore"
+	"github.com/usefathom/fathom/pkg/models"
+)
+
+func Aggregate() error {
+
+	// Get unprocessed pageviews
+	pageviews, err := datastore.GetRawPageviews()
+	if err != nil && err != datastore.ErrNoResults {
+		return err
+	}
+
+	//  Do we have anything to process?
+	if len(pageviews) == 0 {
+		return nil
+	}
+
+	// site stats
+	date := time.Now()
+	siteStats, err := getSiteStats(date)
+	if err != nil {
+		return err
+	}
+
+	for _, p := range pageviews {
+		siteStats.Pageviews += 1
+
+		if p.IsNewVisitor {
+			siteStats.Visitors += 1
+			siteStats.BouncedN += 1
+
+			if p.IsBounce {
+				siteStats.Bounced += 1
+			}
+
+			// TODO: duration
+		}
+	}
+
+	// TODO: page stats
+
+	// TODO: referrer stats
+
+	err = datastore.SaveSiteStats(siteStats)
+	if err != nil {
+		return err
+	}
+
+	// TODO: delete data
+	return nil
+}
+
+func getSiteStats(date time.Time) (*models.SiteStats, error) {
+	stats, err := datastore.GetSiteStats(date)
+	if err != nil && err != datastore.ErrNoResults {
+		return nil, err
+	}
+
+	if stats == nil {
+		return &models.SiteStats{
+			Date: date,
+		}, nil
+	}
+
+	return stats, nil
+}

pkg/datastore/bounces.go

@@ -1,17 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-func BouncesCountPerPageAndDay(before string, after string) ([]*models.Total, error) {
-	query := dbx.Rebind(`SELECT
-		pv.page_id,
-		( COUNT(*) * 100 ) DIV ( SELECT ( COUNT(*) ) FROM pageviews WHERE page_id = pv.page_id AND bounced IS NOT NULL ) AS count,
-		( COUNT(DISTINCT(pv.visitor_id)) * 100 ) DIV ( SELECT ( COUNT(*) ) FROM pageviews WHERE page_id = pv.page_id AND bounced IS NOT NULL ) AS count_unique,
-		DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
-		FROM pageviews pv
-		WHERE pv.bounced = 1 AND pv.bounced IS NOT NULL AND pv.timestamp < ? AND pv.timestamp > ? 
-		GROUP BY pv.page_id, date_group`)
-	var results []*models.Total
-	err := dbx.Select(&results, query, before, after)
-	return results, err
-}

pkg/datastore/browsers.go

@@ -1,21 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-func BrowserCountPerDay(before string, after string) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-		SELECT
-		  v.browser_name AS value,
-		  COUNT(*) AS count,
-		  COUNT(DISTINCT(pv.visitor_id)) AS count_unique,
-		  DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
-		FROM pageviews pv
-		LEFT JOIN visitors v ON v.id = pv.visitor_id
-		WHERE pv.timestamp < ? AND pv.timestamp > ?
-		GROUP BY date_group, v.browser_name`)
-
-	err := dbx.Select(&results, query, before, after)
-	return results, err
-}

pkg/datastore/datastore.go

@@ -44,6 +44,7 @@ func getDSN(driver string, host string, name string, user string, password strin
 	return dsn
 }
 
+// TODO: Move to command (but still auto-run on boot).
 func runMigrations(driver string) {
 	migrations := migrate.FileMigrationSource{
 		Dir: "pkg/datastore/migrations", // TODO: Move to bindata

pkg/datastore/languages.go

@@ -1,21 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-func LanguageCountPerDay(before string, after string) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-		SELECT
-		   v.browser_language AS value,
-		   COUNT(*) AS count,
-		   COUNT(DISTINCT(pv.visitor_id)) AS count_unique,
-		   DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
-		FROM pageviews pv
-		LEFT JOIN visitors v ON v.id = pv.visitor_id
-		WHERE pv.timestamp < ? AND pv.timestamp > ?
-		GROUP BY date_group, v.browser_language`)
-
-	err := dbx.Select(&results, query, before, after)
-	return results, err
-}

pkg/datastore/migrations/6_raw_pageviews.sql

@@ -0,0 +1,17 @@
+-- +migrate Up
+
+CREATE TABLE raw_pageviews(
+   id INT AUTO_INCREMENT PRIMARY KEY NOT NULL,
+   session_id VARCHAR(16) NOT NULL,
+   pathname VARCHAR(255) NOT NULL,
+   is_new_visitor TINYINT(1) NOT NULL,
+   is_unique TINYINT(1) NOT NULL,
+   is_bounce TINYINT(1) NULL,
+   referrer VARCHAR(255) NULL,
+   duration INT(4) NULL,
+   timestamp DATETIME NOT NULL
+);
+
+-- +migrate Down
+
+DROP TABLE IF EXISTS raw_pageviews;

pkg/datastore/migrations/7_daily_page_stats.sql

@@ -0,0 +1,16 @@
+-- +migrate Up
+
+CREATE TABLE daily_page_stats(
+   pathname VARCHAR(255) NOT NULL,
+   views INT NOT NULL,
+   unique_views INT NOT NULL,
+   bounced INT NOT NULL,
+   bounced_n INT NOT NULL,
+   avg_duration INT NOT NULL,
+   avg_duration_n INT NOT NULL,
+   date DATE NOT NULL
+);
+
+-- +migrate Down
+
+DROP TABLE IF EXISTS daily_page_stats;

pkg/datastore/migrations/8_daily_site_stats.sql

@@ -0,0 +1,15 @@
+-- +migrate Up
+
+CREATE TABLE daily_site_stats(
+   visitors INT NOT NULL,
+   pageviews INT NOT NULL,
+   bounced INT NOT NULL,
+   bounced_n INT NOT NULL,
+   avg_duration INT NOT NULL,
+   avg_duration_n INT NOT NULL,
+   date DATE NOT NULL
+);
+
+-- +migrate Down
+
+DROP TABLE IF EXISTS daily_site_stats;

pkg/datastore/migrations/9_daily_referrer_stats.sql

@@ -0,0 +1,16 @@
+-- +migrate Up
+
+CREATE TABLE daily_referrer_stats(
+   url VARCHAR(255) NOT NULL,
+   visitors INT NOT NULL,
+   pageviews INT NOT NULL,
+   bounced INT NOT NULL,
+   bounced_n INT NOT NULL,
+   avg_duration INT NOT NULL,
+   avg_duration_n INT NOT NULL,
+   date DATE NOT NULL
+);
+
+-- +migrate Down
+
+DROP TABLE IF EXISTS daily_referrer_stats;

pkg/datastore/page_stats.go

@@ -0,0 +1 @@
+package datastore

pkg/datastore/pages.go

@@ -1,34 +0,0 @@
-package datastore
-
-import (
-	"database/sql"
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-// GetPageByHostnameAndPath retrieves a page from the connected database
-func GetPageByHostnameAndPath(hostname, path string) (*models.Page, error) {
-	p := &models.Page{}
-	query := dbx.Rebind(`SELECT * FROM pages WHERE hostname = ? AND path = ? LIMIT 1`)
-	err := dbx.Get(p, query, hostname, path)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, ErrNoResults
-		}
-
-		return nil, err
-	}
-
-	return p, nil
-}
-
-// SavePage inserts the page model in the connected database
-func SavePage(p *models.Page) error {
-	query := dbx.Rebind(`INSERT INTO pages(scheme, hostname, path) VALUES(?, ?, ?)`)
-	result, err := dbx.Exec(query, p.Scheme, p.Hostname, p.Path)
-	if err != nil {
-		return err
-	}
-
-	p.ID, _ = result.LastInsertId()
-	return nil
-}

pkg/datastore/pageviews.go

@@ -1,5 +1,6 @@
 package datastore
 
+/*
 import (
 	"database/sql"
 
@@ -95,3 +96,5 @@ func PageviewCountPerPageAndDay(before string, after string) ([]*models.Total, e
 	err := dbx.Select(&results, query, before, after)
 	return results, err
 }
+
+*/

pkg/datastore/raw_pageviews.go

@@ -0,0 +1,32 @@
+package datastore
+
+import (
+	"github.com/usefathom/fathom/pkg/models"
+)
+
+const sqlInsertRawPageview = `INSERT INTO raw_pageviews(session_id, pathname, is_new_visitor, is_unique, is_bounce, referrer, duration, timestamp) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`
+const sqlSelectRawPageviews = `SELECT * FROM raw_pageviews`
+
+// SaveRawPageview inserts a single pageview model into the connected database
+func SaveRawPageview(p *models.RawPageview) error {
+	query := dbx.Rebind(sqlInsertRawPageview)
+	result, err := dbx.Exec(query, p.SessionID, p.Pathname, p.IsNewVisitor, p.IsUnique, p.IsBounce, p.Referrer, p.Duration, p.Timestamp)
+	if err != nil {
+		return err
+	}
+
+	p.ID, _ = result.LastInsertId()
+	return nil
+}
+
+// SaveRawPageviews inserts multiple pageviews
+func SaveRawPageviews(p []*models.RawPageview) error {
+	return nil // TODO: Implement this method
+}
+
+func GetRawPageviews() ([]*models.RawPageview, error) {
+	var results []*models.RawPageview
+	query := dbx.Rebind(sqlSelectRawPageviews)
+	err := dbx.Select(&results, query)
+	return results, err
+}

pkg/datastore/referrer_stats.go

@@ -0,0 +1 @@
+package datastore

pkg/datastore/referrers.go

@@ -1,22 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-func ReferrerCountPerDay(before string, after string) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-		SELECT
-	      pv.referrer_url AS value,
-		  COUNT(*) AS count,
-	      COUNT(DISTINCT(pv.visitor_id)) AS count_unique,
-	      DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
-	    FROM pageviews pv
-	    WHERE pv.referrer_url IS NOT NULL
-	    AND pv.referrer_url != ''
-	    AND pv.timestamp < ? AND pv.timestamp > ? 
-	    GROUP BY date_group, pv.referrer_url`)
-
-	err := dbx.Select(&results, query, before, after)
-	return results, err
-}

pkg/datastore/screens.go

@@ -1,21 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-func ScreenCountPerDay(before string, after string) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-		SELECT
-		  v.screen_resolution AS value,
-		  COUNT(*) AS count,
-		  COUNT(DISTINCT(pv.visitor_id)) AS count_unique,
-		  DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
-		FROM pageviews pv
-		LEFT JOIN visitors v ON v.id = pv.visitor_id
-		WHERE pv.timestamp < ? AND pv.timestamp > ?
-		GROUP BY date_group, v.screen_resolution`)
-
-	err := dbx.Select(&results, query, before, after)
-	return results, err
-}

pkg/datastore/site_stats.go

@@ -0,0 +1,35 @@
+package datastore
+
+import (
+	"database/sql"
+	"github.com/usefathom/fathom/pkg/models"
+	"time"
+)
+
+const sqlSelectSiteStat = `SELECT * FROM daily_site_stats WHERE date = ? LIMIT 1`
+const sqlInsertSiteStats = `INSERT INTO daily_site_stats(visitors, pageviews, bounced, bounced_n, avg_duration, avg_duration_n, date) VALUES(?, ?, ?, ?, ?, ?, ?)`
+
+/*
+visitors INT NOT NULL,
+pageviews INT NOT NULL,
+bounced INT NOT NULL,
+bounced_n INT NOT NULL,
+avg_duration INT NOT NULL,
+avg_duration_n INT NOT NULL,
+date DATE NOT NULL
+*/
+func GetSiteStats(date time.Time) (*models.SiteStats, error) {
+	stats := &models.SiteStats{}
+	query := dbx.Rebind(sqlSelectSiteStat)
+	err := dbx.Get(stats, query, date)
+	if err != nil && err == sql.ErrNoRows {
+		return nil, ErrNoResults
+	}
+	return stats, err
+}
+
+func SaveSiteStats(s *models.SiteStats) error {
+	query := dbx.Rebind(sqlInsertSiteStats)
+	_, err := dbx.Exec(query, s.Visitors, s.Pageviews, s.Bounced, s.BouncedN, s.AvgDuration, s.AvgDurationN, s.Date)
+	return err
+}

pkg/datastore/time_on_site.go

@@ -1,20 +0,0 @@
-package datastore
-
-// TODO: Store in visitors table? Or other table?
-func AvgTimeOnSite(before int64, after int64) (int64, error) {
-	query := dbx.Rebind(`
-   SELECT ROUND(AVG(time_on_site)) FROM ( 
-      SELECT SUM(time_on_page) AS time_on_site 
-      FROM pageviews 
-      WHERE time_on_page > 0 AND UNIX_TIMESTAMP(timestamp) < ? AND UNIX_TIMESTAMP(timestamp) > ?
-      GROUP BY visitor_id 
-   ) AS time_on_site_query`)
-
-	var total int64
-	err := dbx.Get(&total, query, before, after)
-	if err != nil {
-		return 0, err
-	}
-
-	return total, nil
-}

pkg/datastore/total.go

@@ -1,35 +0,0 @@
-package datastore
-
-import (
-	"fmt"
-
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-func SaveTotals(metric string, totals []*models.Total) error {
-	query := fmt.Sprintf(`INSERT INTO total_%s( value, count, count_unique, date) VALUES( ?, ?, ?, ? ) ON DUPLICATE KEY UPDATE count = ?, count_unique = ?`, metric)
-	query = dbx.Rebind(query)
-
-	tx, err := dbx.Begin()
-	if err != nil {
-		return err
-	}
-
-	stmt, err := tx.Prepare(query)
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-
-	for _, t := range totals {
-		result, err := stmt.Exec(t.Value, t.Count, t.CountUnique, t.Date, t.Count, t.CountUnique)
-		if err != nil {
-			return err
-		}
-
-		t.ID, _ = result.LastInsertId()
-	}
-
-	err = tx.Commit()
-	return err
-}

pkg/datastore/total_bounces.go

@@ -1,33 +0,0 @@
-package datastore
-
-// TotalBounces returns the total number of pageviews between the given timestamps
-func TotalBounces(before int64, after int64) (int64, error) {
-	var total int64
-
-	query := dbx.Rebind(`
-		SELECT COALESCE(ROUND(AVG(t.count), 0), 0)
-		FROM total_bounced t 
-		WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	if err != nil {
-		return 0, err
-	}
-
-	return total, nil
-}
-
-// TotalUniqueBounces returns the total number of unique pageviews between the given timestamps
-func TotalUniqueBounces(before int64, after int64) (int64, error) {
-	var total int64
-
-	query := dbx.Rebind(`
-		SELECT COALESCE(AVG(t.count_unique), 0)
-		FROM total_bounced t 
-		WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	if err != nil {
-		return 0, err
-	}
-
-	return total, nil
-}

pkg/datastore/total_browsers.go

@@ -1,47 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-// TotalBrowsers returns the total # of browsers between two given timestamps
-func TotalBrowsers(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-       SELECT
-         COALESCE(SUM(t.count), 0)
-       FROM total_browser_names t
-       WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-// TotalUniqueBrowsers returns the total # of unique browsers between two given timestamps
-func TotalUniqueBrowsers(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-       SELECT
-         COALESCE(SUM(t.count_unique), 0)
-       FROM total_browser_names t
-       WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-func TotalsPerBrowser(before int64, after int64, limit int64) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-      SELECT
-        t.value AS value,
-        COALESCE(SUM(t.count), 0) AS count,
-        COALESCE(SUM(t.count_unique), 0) AS count_unique
-      FROM total_browser_names t
-      WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-      GROUP BY t.value
-      ORDER BY count DESC
-      LIMIT ?`)
-
-	err := dbx.Select(&results, query, before, after, limit)
-	return results, err
-}

pkg/datastore/total_languages.go

@@ -1,49 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-// TotalLanguages returns the total # of browser languages between two given timestamps
-func TotalLanguages(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-    SELECT
-      COALESCE(SUM(t.count), 0)
-    FROM total_browser_languages t
-    WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-// TotalUniqueLanguages returns the total # of unique browser languages between two given timestamps
-func TotalUniqueLanguages(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-    SELECT
-      COALESCE(SUM(t.count_unique), 0)
-    FROM total_browser_languages t
-    WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-func TotalsPerLanguage(before int64, after int64, limit int64) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-		SELECT
-	      t.value AS value,
-	      COALESCE(SUM(t.count), 0) AS count,
-          COALESCE(SUM(t.count_unique), 0) AS count_unique
-	   FROM total_browser_languages t
-	   WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-	   GROUP BY t.value
-	   ORDER BY count DESC
-	   LIMIT ?`)
-
-	err := dbx.Select(&results, query, before, after, limit)
-	return results, err
-}

pkg/datastore/total_pageviews.go

@@ -1,106 +0,0 @@
-package datastore
-
-import (
-	"fmt"
-
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-// TotalPageviews returns the total number of pageviews between the given timestamps
-func TotalPageviews(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-		SELECT COALESCE(SUM(t.count), 0)
-		FROM total_pageviews t 
-		WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	if err != nil {
-		return 0, err
-	}
-
-	return total, nil
-}
-
-// TotalUniquePageviews returns the total number of unique pageviews between the given timestamps
-func TotalUniquePageviews(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-		SELECT COALESCE(SUM(t.count_unique), 0)
-		FROM total_pageviews t 
-		WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	if err != nil {
-		return 0, err
-	}
-
-	return total, nil
-}
-
-// TotalPageviewsPerDay returns a slice of data points representing the number of pageviews per day
-func TotalPageviewsPerDay(before int64, after int64) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-		SELECT
-	      CONCAT(p.scheme, "://", p.hostname, p.path) AS value,
-		  COALESCE(SUM(t.count), 0) AS count,
-		  COALESCE(SUM(t.count_unique), 0) AS count_unique,
-	      DATE_FORMAT(t.date, '%Y-%m-%d') AS label
-	    FROM total_pageviews t
-	    WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-	    GROUP BY label, p.hostname, p.path, p.scheme`)
-
-	err := dbx.Select(&results, query, before, after)
-	if err != nil {
-		return results, err
-	}
-
-	return results, nil
-}
-
-// TotalPageviewsPerPage returns a set of pageview counts, grouped by page (hostname + path)
-func TotalPageviewsPerPage(before int64, after int64, limit int64) ([]*models.Total, error) {
-	var results []*models.Total
-	query := dbx.Rebind(`
-		SELECT
-			CONCAT(p.scheme, "://", p.hostname, p.path) AS value,
-			COALESCE(SUM(t.count), 0) AS count,
-			COALESCE(SUM(t.count_unique), 0) AS count_unique
-		FROM total_pageviews t
-		LEFT JOIN pages p ON p.id = t.page_id
-		WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-		GROUP BY p.hostname, p.path, p.scheme
-		ORDER BY count DESC
-		LIMIT ?`)
-	err := dbx.Select(&results, query, before, after, limit)
-	if err != nil {
-		return results, err
-	}
-
-	return results, nil
-}
-
-// SavePageviewTotals saves the given totals in the connected database
-// Differs slightly from the metric specific totals because of the normalized pages (to save storage)
-func SavePageTotals(metric string, totals []*models.Total) error {
-	tx, err := dbx.Begin()
-	if err != nil {
-		return nil
-	}
-
-	query := fmt.Sprintf(`INSERT INTO total_%s( page_id, count, count_unique, date ) VALUES( ?, ?, ?, ? ) ON DUPLICATE KEY UPDATE count = ?, count_unique = ?`, metric)
-	query = dbx.Rebind(query)
-	stmt, err := tx.Prepare(query)
-	if err != nil {
-		return err
-	}
-
-	for _, t := range totals {
-		_, err = stmt.Exec(t.PageID, t.Count, t.CountUnique, t.Date, t.Count, t.CountUnique)
-	}
-
-	err = tx.Commit()
-	return err
-}

pkg/datastore/total_referrers.go

@@ -1,49 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-// TotalReferrers returns the total # of referrers between two given timestamps
-func TotalReferrers(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-       SELECT
-         COALESCE(SUM(t.count), 0)
-       FROM total_referrers t
-       WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-// TotalUniqueReferrers returns the total # of unique referrers between two given timestamps
-func TotalUniqueReferrers(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-       SELECT
-         COALESCE(SUM(t.count_unique), 0)
-       FROM total_referrers t
-       WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-func TotalsPerReferrer(before int64, after int64, limit int64) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-      SELECT
-         t.value,
-         COALESCE(SUM(t.count), 0) AS count,
-         COALESCE(SUM(t.count_unique), 0) AS count_unique
-      FROM total_referrers t
-      WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-      GROUP BY t.value
-      ORDER BY count DESC
-      LIMIT ?`)
-
-	err := dbx.Select(&results, query, before, after, limit)
-	return results, err
-}

pkg/datastore/total_screens.go

@@ -1,47 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-// TotalScreens returns the total # of screens between two given timestamps
-func TotalScreens(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-       SELECT
-         COALESCE(SUM(t.count), 0)
-       FROM total_screens t
-       WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-// TotalUniqueScreens returns the total # of unique screens between two given timestamps
-func TotalUniqueScreens(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-       SELECT
-         COALESCE(SUM(t.count_unique), 0)
-       FROM total_screens t
-       WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-func TotalsPerScreen(before int64, after int64, limit int64) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-      SELECT
-        t.value AS value,
-        COALESCE(SUM(t.count), 0) AS count,
-        COALESCE(SUM(t.count_unique), 0) AS count_unique
-      FROM total_screens t
-      WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-      GROUP BY t.value
-      ORDER BY count DESC
-      LIMIT ?`)
-
-	err := dbx.Select(&results, query, before, after, limit)
-	return results, err
-}

pkg/datastore/total_visitors.go

@@ -1,52 +0,0 @@
-package datastore
-
-import "github.com/usefathom/fathom/pkg/models"
-
-// TotalVisitors returns the number of total visitors between the given timestamps
-func TotalVisitors(before int64, after int64) (int, error) {
-	var total int
-
-	query := dbx.Rebind(`
-    SELECT COALESCE(SUM(t.count), 0)
-    FROM total_visitors t
-    WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	err := dbx.Get(&total, query, before, after)
-	return total, err
-}
-
-// TotalVisitorsPerDay returns a point slice containing visitor data per day
-func TotalVisitorsPerDay(before int64, after int64) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`SELECT
-      COALESCE(SUM(t.count), 0) AS count,
-	  COALESCE(SUM(t.count_unique), 0) AS count_unique,
-      DATE_FORMAT(t.date, '%Y-%m-%d') AS label
-    FROM total_visitors t
-    WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-    GROUP BY label`)
-
-	err := dbx.Select(&results, query, before, after)
-	return results, err
-}
-
-// SaveVisitorTotals saves the given totals in the connected datastore
-func SaveVisitorTotals(totals []*models.Total) error {
-	tx, err := dbx.Begin()
-	if err != nil {
-		return nil
-	}
-
-	query := dbx.Rebind(`INSERT INTO total_visitors( count, date ) VALUES( ?, ? ) ON DUPLICATE KEY UPDATE count = ?`)
-	stmt, err := tx.Prepare(query)
-	if err != nil {
-		return err
-	}
-
-	for _, t := range totals {
-		_, err = stmt.Exec(t.Count, t.Date, t.Count)
-	}
-
-	err = tx.Commit()
-	return err
-}

pkg/datastore/visitors.go

@@ -1,62 +0,0 @@
-package datastore
-
-import (
-	"database/sql"
-
-	"github.com/usefathom/fathom/pkg/models"
-)
-
-// GetVisitorByKey ...
-func GetVisitorByKey(key string) (*models.Visitor, error) {
-	v := &models.Visitor{}
-	query := dbx.Rebind(`SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1`)
-	err := dbx.Get(v, query, key)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, ErrNoResults
-		}
-
-		return nil, err
-	}
-
-	return v, nil
-}
-
-// SaveVisitor inserts a single visitor model into the connected database
-func SaveVisitor(v *models.Visitor) error {
-	query := dbx.Rebind(`INSERT INTO visitors(visitor_key, device_os, browser_name, browser_version, browser_language, screen_resolution, country) VALUES( ?, ?, ?, ?, ?, ?, ? )`)
-	result, err := dbx.Exec(query, v.Key, v.DeviceOS, v.BrowserName, v.BrowserVersion, v.BrowserLanguage, v.ScreenResolution, v.Country)
-	if err != nil {
-		return err
-	}
-
-	v.ID, _ = result.LastInsertId()
-	return nil
-}
-
-// RealtimeVisitors returns the total number of visitors in the last 3 minutes
-// TODO: Query visitors table instead, using a last_seen column.
-func RealtimeVisitors() (int, error) {
-	var result int
-	query := dbx.Rebind(`
-		SELECT COUNT(DISTINCT(pv.visitor_id))
-		FROM pageviews pv
-		WHERE pv.timestamp >= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 3 HOUR_MINUTE) AND pv.timestamp <= CURRENT_TIMESTAMP`)
-	err := dbx.Get(&result, query)
-	return result, err
-}
-
-func VisitorCountPerDay(before string, after string) ([]*models.Total, error) {
-	var results []*models.Total
-
-	query := dbx.Rebind(`
-		SELECT
-		  COUNT(DISTINCT(pv.visitor_id)) AS count,
-		  DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
-		FROM pageviews pv
-		WHERE pv.timestamp < ? AND pv.timestamp > ?
-		GROUP BY date_group`)
-
-	err := dbx.Select(&results, query, before, after)
-	return results, err
-}

pkg/models/count.go

@@ -1,8 +0,0 @@
-package models
-
-type Count struct {
-	URL            string  `json:"url"`
-	Views          int64   `json:"views"`
-	Uniques        int64   `json:"uniques"`
-	PercentOfTotal float64 `json:"percent_of_total"`
-}

pkg/models/page.go

@@ -1,9 +0,0 @@
-package models
-
-type Page struct {
-	ID       int64  `json:"-"`
-	Scheme   string `json:"scheme"`
-	Hostname string `json:"hostname"`
-	Path     string `json:"path"`
-	Title    string `json:"title"`
-}

pkg/models/page_stats.go

@@ -0,0 +1,16 @@
+package models
+
+import (
+	"time"
+)
+
+type PageStats struct {
+	Pathname     string    `db:"pathname"`
+	Views        int64     `db:"views"`
+	UniqueViews  int64     `db:"unique_views"`
+	Bounced      int64     `db:"bounced"`
+	BouncedN     int64     `db:"bounced_n"`
+	AvgDuration  int64     `db:"avg_duration"`
+	AvgDurationN int64     `db:"avg_duration_n"`
+	Date         time.Time `db:"date"`
+}

pkg/models/raw_pageview.go

@@ -0,0 +1,17 @@
+package models
+
+import (
+	"time"
+)
+
+type RawPageview struct {
+	ID           int64     `db:"id"`
+	SessionID    string    `db:"session_id"`
+	Pathname     string    `db:"pathname"`
+	IsNewVisitor bool      `db:"is_new_visitor"`
+	IsUnique     bool      `db:"is_unique"`
+	IsBounce     bool      `db:"is_bounce"`
+	Referrer     string    `db:"referrer"`
+	Duration     int64     `db:"duration"`
+	Timestamp    time.Time `db:"timestamp"`
+}

pkg/models/referrer_stats.go

@@ -0,0 +1,16 @@
+package models
+
+import (
+	"time"
+)
+
+type ReferrerStats struct {
+	URL          string    `db:"url"`
+	Visitors     int64     `db:"visitors"`
+	Pageviews    int64     `db:"pageviews"`
+	Bounced      int64     `db:"bounced"`
+	BouncedN     int64     `db:"bounced_n"`
+	AvgDuration  int64     `db:"avg_duration"`
+	AvgDurationN int64     `db:"avg_duration_n"`
+	Date         time.Time `db:"date"`
+}

pkg/models/site_stats.go

@@ -0,0 +1,15 @@
+package models
+
+import (
+	"time"
+)
+
+type SiteStats struct {
+	Visitors     int64     `db:"visitors"`
+	Pageviews    int64     `db:"pageviews"`
+	Bounced      int64     `db:"bounced"`
+	BouncedN     int64     `db:"bounced_n"`
+	AvgDuration  int64     `db:"avg_duration"`
+	AvgDurationN int64     `db:"avg_duration_n"`
+	Date         time.Time `db:"date"`
+}

pkg/models/total.go

@@ -1,12 +0,0 @@
-package models
-
-// Total represents a daily aggregated total for a metric
-type Total struct {
-	ID                int64   `json:"-"`
-	PageID            int64   `db:"page_id" json:"-"`
-	Value             string  `db:"value" json:"value"`
-	Count             int64   `db:"count" json:"count"`
-	CountUnique       int64   `db:"count_unique" json:"count_unique"`
-	PercentageOfTotal float64 `db:"-" json:"percentage_of_total"`
-	Date              string  `db:"date_group" json:"date,omitempty"`
-}

pkg/models/visitor.go

@@ -1,12 +0,0 @@
-package models
-
-type Visitor struct {
-	ID               int64
-	Key              string
-	BrowserName      string
-	BrowserVersion   string
-	BrowserLanguage  string
-	Country          string
-	DeviceOS         string
-	ScreenResolution string
-}