status-im
/
fathom
mirror of https://github.com/status-im/fathom.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

[wip] move bulk of storage to client-side for improved scalability, simplicity and privacy. see #14.

This commit is contained in:
Danny van Kooten 2018-05-06 11:53:19 +02:00
parent 5a403d536a
commit 082073afb8
65 changed files with 1688 additions and 2527 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
assets/js/tracker.js
View File

@ -1,13 +1,6 @@
'use strict';
/* cookies.js - https://github.com/ScottHamper/Cookies */
(function(d,f){"use strict";var h=function(d){if("object"!==typeof d.document)throw Error("Cookies.js requires a `window` with a `document` object");var b=function(a,e,c){return 1===arguments.length?b.get(a):b.set(a,e,c)};b._document=d.document;b._cacheKeyPrefix="cookey.";b._maxExpireDate=new Date("Fri, 31 Dec 9999 23:59:59 UTC");b.defaults={path:"/",secure:!1};b.get=function(a){b._cachedDocumentCookie!==b._document.cookie&&b._renewCache();a=b._cache[b._cacheKeyPrefix+a];return a===f?f:decodeURIComponent(a)};
b.set=function(a,e,c){c=b._getExtendedOptions(c);c.expires=b._getExpiresDate(e===f?-1:c.expires);b._document.cookie=b._generateCookieString(a,e,c);return b};b.expire=function(a,e){return b.set(a,f,e)};b._getExtendedOptions=function(a){return{path:a&&a.path||b.defaults.path,domain:a&&a.domain||b.defaults.domain,expires:a&&a.expires||b.defaults.expires,secure:a&&a.secure!==f?a.secure:b.defaults.secure}};b._isValidDate=function(a){return"[object Date]"===Object.prototype.toString.call(a)&&!isNaN(a.getTime())};
b._getExpiresDate=function(a,e){e=e||new Date;"number"===typeof a?a=Infinity===a?b._maxExpireDate:new Date(e.getTime()+1E3*a):"string"===typeof a&&(a=new Date(a));if(a&&!b._isValidDate(a))throw Error("`expires` parameter cannot be converted to a valid Date instance");return a};b._generateCookieString=function(a,b,c){a=a.replace(/[^#$&+\^`|]/g,encodeURIComponent);a=a.replace(/\(/g,"%28").replace(/\)/g,"%29");b=(b+"").replace(/[^!#$&-+\--:<-\[\]-~]/g,encodeURIComponent);c=c||{};a=a+"="+b+(c.path?";path="+
c.path:"");a+=c.domain?";domain="+c.domain:"";a+=c.expires?";expires="+c.expires.toUTCString():"";return a+=c.secure?";secure":""};b._getCacheFromString=function(a){var e={};a=a?a.split("; "):[];for(var c=0;c<a.length;c++){var d=b._getKeyValuePairFromCookieString(a[c]);e[b._cacheKeyPrefix+d.key]===f&&(e[b._cacheKeyPrefix+d.key]=d.value)}return e};b._getKeyValuePairFromCookieString=function(a){var b=a.indexOf("="),b=0>b?a.length:b,c=a.substr(0,b),d;try{d=decodeURIComponent(c)}catch(k){console&&"function"===
typeof console.error&&console.error('Could not decode cookie with key "'+c+'"',k)}return{key:d,value:a.substr(b+1)}};b._renewCache=function(){b._cache=b._getCacheFromString(b._document.cookie);b._cachedDocumentCookie=b._document.cookie};b._areEnabled=function(){var a="1"===b.set("cookies.js",1).get("cookies.js");b.expire("cookies.js");return a};b.enabled=b._areEnabled();return b},g=d&&"object"===typeof d.document?h(d):h;"function"===typeof define&&define.amd?define(function(){return g}):"object"===
typeof exports?("object"===typeof module&&"object"===typeof module.exports&&(exports=module.exports=g),exports.Cookies=g):d.Cookies=g})("undefined"===typeof window?this:window);
var cookies = require('cookies-js');
var queue = window.fathom.q || [];
var trackerUrl = '';
var commands = {
@ -19,11 +12,6 @@ var commands = {
function stringifyObject(json) {
var keys = Object.keys(json);
// omit empty
keys = keys.filter(function(k) {
return json[k].length > 0;
});
return '?' +
keys.map(function(k) {
return encodeURIComponent(k) + '=' +
@ -36,6 +24,25 @@ function generateKey() {
return Array(16).join().split(',').map(function() { return s.charAt(Math.floor(Math.random() * s.length)); }).join('');
}
function getData() {
var data = Cookies.get('_fathom');
if(data) {
try{
data = JSON.parse(data);
return data;
} catch(e) {
console.error(e);
}
}
return {
sid: generateKey(),
isNew: true,
pagesViewed: [],
}
}
function setTrackerUrl(v) {
trackerUrl = v;
}
@ -51,12 +58,6 @@ function trackPageview() {
return;
}
// set cookie with random visitor key, valid for 24 hours.
// visitor can delete this cookie in order to be forgotten
if(!Cookies.get('_fathom')) {
Cookies.set('_fathom', generateKey(), { expires: 60 * 60 * 24 });
}
// get the path or canonical
var path = location.pathname + location.search;
@ -68,21 +69,31 @@ function trackPageview() {
path = a.pathname;
}
let referrer = '';
if(document.referrer.indexOf(location.hostname) < 0) {
referrer = document.referrer;
}
let data = getData();
var d = {
vk: Cookies.get('_fathom'),
h: location.hostname,
t: document.title,
l: navigator.language,
sid: data.sid,
p: path,
sr: screen.width + "x" + screen.height,
t: document.title,
ru: document.referrer,
rk: "",
r: referrer,
scheme: location.protocol.substring(0, location.protocol.length - 1),
u: data.pagesViewed.indexOf(path) == -1 ? 1 : 0,
b: data.isNew ? 1 : 0, // because only new visitors can bounce. we update this server-side.
n: data.isNew ? 1 : 0,
};
var i = document.createElement('img');
i.src = trackerUrl + stringifyObject(d);
i.addEventListener('load', function() {
data.pagesViewed.push(path);
data.isNew = false;
Cookies.set('_fathom', JSON.stringify(data), { expires: 60 * 60 * 24});
});
document.body.appendChild(i);
}

17
fathom.go
View File

@ -1,13 +1,13 @@
package main
import (
"os"
"github.com/joho/godotenv"
"github.com/robfig/cron"
"github.com/usefathom/fathom/pkg/commands"
"github.com/usefathom/fathom/pkg/count"
"github.com/usefathom/fathom/pkg/counter"
"github.com/usefathom/fathom/pkg/datastore"
"gopkg.in/alecthomas/kingpin.v2"
"os"
"github.com/kelseyhightower/envconfig"
log "github.com/sirupsen/logrus"
@ -50,11 +50,6 @@ func main() {
db := datastore.Init(dbcfg.Driver, dbcfg.Host, dbcfg.Name, dbcfg.User, dbcfg.Password)
defer db.Close()
// setup cron to run count.Archive every hour
c := cron.New()
c.AddFunc("@hourly", count.Archive)
c.Start()
// parse & run cli commands
app.Version("1.0")
app.UsageTemplate(kingpin.CompactUsageTemplate)
@ -66,8 +61,10 @@ func main() {
commands.Server(*serverPort, *serverWebRoot)
case "archive":
commands.Archive()
err := counter.Aggregate()
if err != nil {
log.Warn(err)
}
}
}

54
gulpfile.js
View File

@ -10,9 +10,10 @@ const gutil = require('gulp-util')
const sass = require('gulp-sass')
const uglify = require('gulp-uglify')
const pump = require('pump')
const es = require('event-stream');
const debug = process.env.NODE_ENV !== 'production';
let defaultTasks = [ 'browserify', 'sass', 'tracker', 'html', 'img' ] ;
let defaultTasks = [ 'browserify', 'sass', 'html', 'img' ] ;
if( ! debug ) {
defaultTasks.push( 'minify' );
}
@ -20,29 +21,33 @@ if( ! debug ) {
gulp.task('default', defaultTasks);
gulp.task('browserify', function () {
return browserify({
entries: './assets/js/script.js',
debug: debug
})
.transform("babelify", {
presets: ["es2015"],
plugins: [
"transform-decorators-legacy",
["transform-react-jsx", { "pragma":"h" } ]
]
})
.bundle()
.on('error', function(err){
console.log(err.message);
this.emit('end');
})
.pipe(source('script.js'))
.pipe(buffer())
.pipe(gulp.dest('./build/js/'))
let files = [
'./assets/js/script.js',
'./assets/js/tracker.js',
];
var tasks = files.map(function(entry) {
return browserify({
entries: entry,
debug: debug
})
.transform("babelify", {
presets: ["es2015"],
plugins: [
"transform-decorators-legacy",
["transform-react-jsx", { "pragma":"h" } ]
]
})
.bundle()
.pipe(source(entry.split('/').pop()))
.pipe(gulp.dest('./build/js/'))
});
// create a merged stream
return es.merge.apply(null, tasks);
});
gulp.task('minify', function(cb) {
process.env.NODE_ENV = 'production';
process.env.NODE_ENV = 'production'; // why is this here?
pump([
gulp.src('./build/js/*.js'),
@ -61,11 +66,6 @@ gulp.task('html', function() {
.pipe(gulp.dest('./build'))
});
gulp.task('tracker', function() {
return gulp.src('./assets/js/tracker.js')
.pipe(gulp.dest('./build/js'))
});
gulp.task('sass', function () {
var files = './assets/sass/[^_]*.scss';
return gulp.src(files)
@ -76,7 +76,7 @@ gulp.task('sass', function () {
});
gulp.task('watch', ['default'], function() {
gulp.watch(['./assets/js/**/*.js'], ['browserify', 'tracker'] );
gulp.watch(['./assets/js/**/*.js'], ['browserify'] );
gulp.watch(['./assets/sass/**/**/*.scss'], ['sass'] );
gulp.watch(['./assets/**/*.html'], ['html'] );
gulp.watch(['./assets/img/**/*'], ['img'] );

2371
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

8
package.json
View File

@ -1,8 +1,8 @@
{
"license": "MIT",
"repository": {
"type" : "git"
, "url" : "https://github.com/usefathom/fathom.git"
"repository": {
"type": "git",
"url": "https://github.com/usefathom/fathom.git"
},
"devDependencies": {
"babel-core": "^6.26.0",
@ -11,6 +11,7 @@
"babel-preset-es2015": "^6.18.0",
"babelify": "^8.0.0",
"browserify": "^16.2.0",
"event-stream": "^3.3.4",
"gulp": "^3.9.1",
"gulp-rename": "^1.2.2",
"gulp-sass": "^4.0.1",
@ -19,6 +20,7 @@
"vinyl-source-stream": "^2.0.0"
},
"dependencies": {
"cookies-js": "^1.2.3",
"d3": "^5.1.0",
"d3-tip": "^0.7.1",
"decko": "^1.2.0",

18
pkg/api/bounces.go
View File

@ -1,18 +0,0 @@
package api
import (
"net/http"
"github.com/usefathom/fathom/pkg/datastore"
)
// URL: /api/bounces/count
var GetBouncesCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
result, err := datastore.TotalBounces(before, after)
if err != nil {
return err
}
return respond(w, envelope{Data: result})
})

16
pkg/api/browsers.go
View File

@ -1,16 +0,0 @@
package api
import (
"github.com/usefathom/fathom/pkg/count"
"net/http"
)
// URL: /api/browsers
var GetBrowsersHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results, err := count.Browsers(before, after, getRequestedLimit(r))
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

116
pkg/api/collect.go
View File

@ -1,11 +1,8 @@
package api
import (
"crypto/md5"
"encoding/base64"
"encoding/hex"
"net/http"
"strings"
"time"
"github.com/mssola/user_agent"
@ -15,13 +12,13 @@ import (
log "github.com/sirupsen/logrus"
)
var buffer []*models.Pageview
var bufferSize = 250
var buffer []*models.RawPageview
var bufferSize = 50
var timeout = 200 * time.Millisecond
func persistPageviews() {
if len(buffer) > 0 {
err := datastore.SavePageviews(buffer)
err := datastore.SaveRawPageviews(buffer)
if err != nil {
log.Errorf("error saving pageviews: %s", err)
}
@ -31,7 +28,7 @@ func persistPageviews() {
}
}
func processBuffer(pv chan *models.Pageview) {
func processBuffer(pv chan *models.RawPageview) {
for {
select {
case pageview := <-pv:
@ -47,11 +44,10 @@ func processBuffer(pv chan *models.Pageview) {
/* middleware */
func NewCollectHandler() http.Handler {
pageviews := make(chan *models.Pageview, bufferSize)
pageviews := make(chan *models.RawPageview, bufferSize)
go processBuffer(pageviews)
return HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
// abort if this is a bot.
userAgent := r.UserAgent()
ua := user_agent.New(userAgent)
@ -60,96 +56,26 @@ func NewCollectHandler() http.Handler {
}
q := r.URL.Query()
// find page
page, err := datastore.GetPageByHostnameAndPath(q.Get("h"), q.Get("p"))
if err != nil && err != datastore.ErrNoResults {
return err
}
// page does not exist yet, get details & save it
if page == nil {
page = &models.Page{
Scheme: "http",
Hostname: q.Get("h"),
Path: q.Get("p"),
Title: q.Get("t"),
}
if scheme := q.Get("scheme"); scheme != "" {
page.Scheme = scheme
}
err = datastore.SavePage(page)
if err != nil {
return err
}
}
// find visitor by anonymized key from query params
now := time.Now()
visitorKey := q.Get("vk")
visitorKey = enhanceVisitorKey(visitorKey, now.Format("2006-01-02"), userAgent, q.Get("l"), q.Get("sr"))
visitor, err := datastore.GetVisitorByKey(visitorKey)
if err != nil && err != datastore.ErrNoResults {
return err
}
// visitor is new, save it
if visitor == nil {
visitor = &models.Visitor{
BrowserLanguage: q.Get("l"),
ScreenResolution: q.Get("sr"),
DeviceOS: ua.OS(),
Country: "",
Key: visitorKey,
}
// add browser details
visitor.BrowserName, visitor.BrowserVersion = ua.Browser()
// get rid of exact browser versions
visitor.BrowserVersion = parseMajorMinor(visitor.BrowserVersion)
err = datastore.SaveVisitor(visitor)
if err != nil {
return err
}
} else {
lastPageview, err := datastore.GetLastPageviewForVisitor(visitor.ID)
if err != nil && err != datastore.ErrNoResults {
return err
}
if lastPageview != nil && lastPageview.Timestamp.After(now.Add(-30*time.Minute)) {
lastPageview.Bounced = false
lastPageview.TimeOnPage = now.Unix() - lastPageview.Timestamp.Unix()
// TODO: Delay storage until in buffer?
err := datastore.UpdatePageview(lastPageview)
if err != nil {
return err
}
}
}
// get pageview details
pageview := &models.Pageview{
PageID: page.ID,
VisitorID: visitor.ID,
ReferrerUrl: q.Get("ru"),
ReferrerKeyword: q.Get("rk"),
TimeOnPage: 0,
Bounced: true, // TODO: Only mark as bounced if no other pageviews in this session
Timestamp: now,
pageview := &models.RawPageview{
SessionID: q.Get("sid"),
Pathname: q.Get("p"),
IsNewVisitor: q.Get("n") == "1",
IsUnique: q.Get("u") == "1",
IsBounce: q.Get("b") != "0",
Referrer: q.Get("r"),
Duration: 0,
Timestamp: now,
}
// only store referrer URL if not coming from own site
if strings.Contains(pageview.ReferrerUrl, page.Hostname) {
pageview.ReferrerUrl = ""
err := datastore.SaveRawPageview(pageview)
if err != nil {
return err
}
// push onto channel
pageviews <- pageview
//pageviews <- pageview
// don't you cache this
w.Header().Set("Content-Type", "image/gif")
@ -164,9 +90,3 @@ func NewCollectHandler() http.Handler {
return nil
})
}
// generateVisitorKey generates the "unique" visitor key from date, user agent + screen resolution
func enhanceVisitorKey(key string, date string, userAgent string, lang string, screenRes string) string {
byteKey := md5.Sum([]byte(date + userAgent + lang + screenRes))
return hex.EncodeToString(byteKey[:])
}

18
pkg/api/languages.go
View File

@ -1,18 +0,0 @@
package api
import (
"net/http"
"github.com/usefathom/fathom/pkg/count"
)
// URL: /api/languages
var GetLanguagesHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results, err := count.Languages(before, after, getRequestedLimit(r))
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

14
pkg/api/page_stats.go Normal file
View File

@ -0,0 +1,14 @@
package api
import (
"github.com/usefathom/fathom/pkg/models"
"net/http"
)
// URL: /api/stats/page
var GetPageStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
//before, after := getRequestedPeriods(r)
//limit := getRequestedLimit(r)
var result []*models.PageStats
return respond(w, envelope{Data: result})
})

42
pkg/api/pageviews.go
View File

@ -1,42 +0,0 @@
package api
import (
"net/http"
"github.com/usefathom/fathom/pkg/count"
"github.com/usefathom/fathom/pkg/datastore"
)
// URL: /api/pageviews
var GetPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
limit := getRequestedLimit(r)
results, err := count.Pageviews(before, after, limit)
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})
// URL: /api/pageviews/count
var GetPageviewsCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
result, err := datastore.TotalPageviews(before, after)
if err != nil {
return err
}
return respond(w, envelope{Data: result})
})
// URL: /api/pageviews/group/day
var GetPageviewsPeriodCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results, err := datastore.TotalPageviewsPerDay(before, after)
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

6
pkg/api/params.go
View File

@ -7,13 +7,15 @@ import (
"time"
)
// TODO: Move params into Params struct (with defaults)
const defaultPeriod = 7
const defaultLimit = 10
func getRequestedLimit(r *http.Request) int64 {
limit, err := strconv.ParseInt(r.URL.Query().Get("limit"), 10, 64)
if err != nil || limit == 0 {
limit = 10
limit = defaultLimit
}
return limit
@ -30,7 +32,7 @@ func getRequestedPeriods(r *http.Request) (int64, int64) {
after, err = strconv.ParseInt(r.URL.Query().Get("after"), 10, 64)
if err != nil || before == 0 {
after = time.Now().AddDate(0, 0, -7).Unix()
after = time.Now().AddDate(0, 0, -defaultPeriod).Unix()
}
return before, after

14
pkg/api/referrer_stats.go Normal file
View File

@ -0,0 +1,14 @@
package api
import (
"github.com/usefathom/fathom/pkg/models"
"net/http"
)
// URL: /api/stats/referrer
var GetReferrerStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
// before, after := getRequestedPeriods(r)
// limit := getRequestedLimit(r)
var result []*models.ReferrerStats
return respond(w, envelope{Data: result})
})

17
pkg/api/referrers.go
View File

@ -1,17 +0,0 @@
package api
import (
"net/http"
"github.com/usefathom/fathom/pkg/count"
)
// URL: /api/referrers
var GetReferrersHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results, err := count.Referrers(before, after, getRequestedLimit(r))
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

20
pkg/api/routes.go Normal file
View File

@ -0,0 +1,20 @@
package api
import (
"github.com/gorilla/mux"
"net/http"
)
func Routes(webroot string) *mux.Router {
// register routes
r := mux.NewRouter()
r.Handle("/collect", NewCollectHandler()).Methods(http.MethodGet)
r.Handle("/api/session", LoginHandler).Methods(http.MethodPost)
r.Handle("/api/session", LogoutHandler).Methods(http.MethodDelete)
r.Handle("/api/stats/page", Authorize(GetPageStatsHandler)).Methods(http.MethodGet)
r.Path("/tracker.js").Handler(http.FileServer(http.Dir(webroot + "/js/")))
r.PathPrefix("/").Handler(http.FileServer(http.Dir(webroot)))
return r
}

17
pkg/api/screens.go
View File

@ -1,17 +0,0 @@
package api
import (
"net/http"
"github.com/usefathom/fathom/pkg/count"
)
// URL: /api/screen-resolutions
var GetScreenResolutionsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results, err := count.Screens(before, after, getRequestedLimit(r))
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

44
pkg/api/site_stats.go Normal file
View File

@ -0,0 +1,44 @@
package api
import (
"github.com/usefathom/fathom/pkg/models"
"net/http"
)
// URL: /api/stats/site
var GetSiteStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
// before, after := getRequestedPeriods(r)
// limit := getRequestedLimit(r)
var results []*models.SiteStats
return respond(w, envelope{Data: results})
})
// URL: /api/stats/site/pageviews
var GetSiteStatsPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
// before, after := getRequestedPeriods(r)
// limit := getRequestedLimit(r)
var result int
return respond(w, envelope{Data: result})
})
// URL: /api/stats/site/visitors
var GetSiteStatsVisitorsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
// before, after := getRequestedPeriods(r)
var result int
return respond(w, envelope{Data: result})
})
// URL: /api/stats/site/avg-duration
var GetSiteStatsAvgDurationHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
// before, after := getRequestedPeriods(r)
var result int
return respond(w, envelope{Data: result})
})
// URL: /api/stats/site/avg-bounce
var GetSiteStatusAvgBounceHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
// before, after := getRequestedPeriods(r)
var result int
return respond(w, envelope{Data: result})
})

20
pkg/api/time.go
View File

@ -1,20 +0,0 @@
package api
import (
"net/http"
"github.com/usefathom/fathom/pkg/datastore"
)
// TODO: Come up with more consistent URL names.
// URL: /api/time-on-site/count
var GetTimeOnSiteCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
result, err := datastore.AvgTimeOnSite(before, after)
if err != nil {
return err
}
return respond(w, envelope{Data: result})
})

38
pkg/api/visitors.go
View File

@ -1,38 +0,0 @@
package api
import (
"net/http"
"github.com/usefathom/fathom/pkg/datastore"
)
// URL: /api/visitors/count
var GetVisitorsCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
result, err := datastore.TotalVisitors(before, after)
if err != nil {
return err
}
return respond(w, envelope{Data: result})
})
// URL: /api/visitors/count/realtime
var GetVisitorsRealtimeCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
result, err := datastore.RealtimeVisitors()
if err != nil {
return err
}
return respond(w, envelope{Data: result})
})
// URL: /api/visitors/count/group/:period
var GetVisitorsPeriodCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results, err := datastore.TotalVisitorsPerDay(before, after)
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

10
pkg/commands/archive.go
View File

@ -1,10 +0,0 @@
package commands
import (
"github.com/usefathom/fathom/pkg/count"
)
// Archive processes unarchived data (pageviews to aggregated count tables)
func Archive() {
count.Archive()
}

35
pkg/commands/server.go
View File

@ -2,40 +2,21 @@ package commands
import (
"fmt"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/usefathom/fathom/pkg/api"
"log"
"net/http"
"os"
"github.com/gorilla/handlers"
"github.com/usefathom/fathom/pkg/api"
)
// Server starts the HTTP server, listening on the given port
func Server(port int, webroot string) {
// register routes
r := mux.NewRouter()
r.Handle("/collect", api.NewCollectHandler()).Methods("GET")
r.Handle("/api/session", api.LoginHandler).Methods("POST")
r.Handle("/api/session", api.LogoutHandler).Methods("DELETE")
r.Handle("/api/visitors/count", api.Authorize(api.GetVisitorsCountHandler)).Methods("GET")
r.Handle("/api/visitors/count/group/{period}", api.Authorize(api.GetVisitorsPeriodCountHandler)).Methods("GET")
r.Handle("/api/visitors/count/realtime", api.Authorize(api.GetVisitorsRealtimeCountHandler)).Methods("GET")
r.Handle("/api/pageviews/count", api.Authorize(api.GetPageviewsCountHandler)).Methods("GET")
r.Handle("/api/pageviews/count/group/{period}", api.Authorize(api.GetPageviewsPeriodCountHandler)).Methods("GET")
r.Handle("/api/pageviews", api.Authorize(api.GetPageviewsHandler)).Methods("GET")
r.Handle("/api/languages", api.Authorize(api.GetLanguagesHandler)).Methods("GET")
r.Handle("/api/referrers", api.Authorize(api.GetReferrersHandler)).Methods("GET")
r.Handle("/api/screen-resolutions", api.Authorize(api.GetScreenResolutionsHandler)).Methods("GET")
//r.Handle("/api/countries", api.Authorize(api.GetCountriesHandler)).Methods("GET")
r.Handle("/api/browsers", api.Authorize(api.GetBrowsersHandler)).Methods("GET")
r.Handle("/api/bounces/count", api.Authorize(api.GetBouncesCountHandler)).Methods("GET")
r.Handle("/api/time-on-site/count", api.Authorize(api.GetTimeOnSiteCountHandler)).Methods("GET")
r.Path("/tracker.js").Handler(http.FileServer(http.Dir(webroot + "/js/")))
r.PathPrefix("/").Handler(http.FileServer(http.Dir(webroot)))
r := api.Routes(webroot)
log.Printf("Now serving %s on port %d/\n", webroot, port)
err := http.ListenAndServe(fmt.Sprintf(":%d", port), handlers.LoggingHandler(os.Stdout, r))
log.Println(err)
if err != nil {
log.Println(err)
}
}

22
pkg/count/bounces.go
View File

@ -1,22 +0,0 @@
package count
import (
"log"
"time"
"github.com/usefathom/fathom/pkg/datastore"
)
// CreateBouncesTotals aggregates pageview data for each page into daily totals
func CreateBouncesTotals(since string) {
tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
totals, err := datastore.BouncesCountPerPageAndDay(tomorrow, since)
if err != nil {
log.Fatal(err)
}
err = datastore.SavePageTotals("bounced", totals)
if err != nil {
log.Fatal(err)
}
}

39
pkg/count/browsers.go
View File

@ -1,39 +0,0 @@
package count
import (
"log"
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Browsers returns a point slice containing browser data per browser name
func Browsers(before int64, after int64, limit int64) ([]*models.Total, error) {
points, err := datastore.TotalsPerBrowser(before, after, limit)
if err != nil {
return nil, err
}
total, err := datastore.TotalBrowsers(before, after)
if err != nil {
return nil, err
}
points = calculatePercentagesOfTotal(points, total)
return points, nil
}
// CreateBrowserTotals aggregates screen data into daily totals
func CreateBrowserTotals(since string) {
tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
totals, err := datastore.BrowserCountPerDay(tomorrow, since)
if err != nil {
log.Fatal(err)
}
err = datastore.SaveTotals("browser_names", totals)
if err != nil {
log.Fatal(err)
}
}

46
pkg/count/count.go
View File

@ -1,46 +0,0 @@
package count
import (
"time"
log "github.com/sirupsen/logrus"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
func getLastArchivedDate() string {
value, _ := datastore.GetOption("last_archived")
if value == "" {
return time.Now().AddDate(-1, 0, 0).Format("2006-01-02")
}
return value
}
// Archive aggregates data into daily totals
func Archive() {
start := time.Now()
lastArchived := getLastArchivedDate()
CreatePageviewTotals(lastArchived)
CreateVisitorTotals(lastArchived)
CreateScreenTotals(lastArchived)
CreateLanguageTotals(lastArchived)
CreateBrowserTotals(lastArchived)
CreateReferrerTotals(lastArchived)
CreateBouncesTotals(lastArchived)
datastore.SetOption("last_archived", time.Now().Format("2006-01-02"))
end := time.Now()
log.Infof("finished aggregating metrics. ran for %dms.", (end.UnixNano()-start.UnixNano())/1000000)
}
func calculatePercentagesOfTotal(totals []*models.Total, total int) []*models.Total {
// calculate percentage values for each point
for _, p := range totals {
p.PercentageOfTotal = float64(p.Count) / float64(total) * 100.00
}
return totals
}

22
pkg/count/count_test.go
View File

@ -1,22 +0,0 @@
package count
import (
"testing"
"github.com/usefathom/fathom/pkg/models"
)
func TestCalculatePointPercentages(t *testing.T) {
totals := []*models.Total{
&models.Total{
Value: "Foo",
Count: 5,
},
}
totals = calculatePercentagesOfTotal(totals, 100)
if totals[0].PercentageOfTotal != 5.00 {
t.Errorf("Percentage value should be 5.00, is %.2f", totals[0].PercentageOfTotal)
}
}

39
pkg/count/languages.go
View File

@ -1,39 +0,0 @@
package count
import (
"log"
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Languages returns a point slice containing language data per language
func Languages(before int64, after int64, limit int64) ([]*models.Total, error) {
points, err := datastore.TotalsPerLanguage(before, after, limit)
if err != nil {
return nil, err
}
total, err := datastore.TotalLanguages(before, after)
if err != nil {
return nil, err
}
points = calculatePercentagesOfTotal(points, total)
return points, nil
}
// CreateLanguageTotals aggregates screen data into daily totals
func CreateLanguageTotals(since string) {
tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
totals, err := datastore.LanguageCountPerDay(tomorrow, since)
if err != nil {
log.Fatal(err)
}
err = datastore.SaveTotals("browser_languages", totals)
if err != nil {
log.Fatal(err)
}
}

39
pkg/count/pageviews.go
View File

@ -1,39 +0,0 @@
package count
import (
"log"
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Pageviews returns a point slice containing language data per language
func Pageviews(before int64, after int64, limit int64) ([]*models.Total, error) {
points, err := datastore.TotalPageviewsPerPage(before, after, limit)
if err != nil {
return nil, err
}
total, err := datastore.TotalPageviews(before, after)
if err != nil {
return nil, err
}
points = calculatePercentagesOfTotal(points, total)
return points, nil
}
// CreatePageviewTotals aggregates pageview data for each page into daily totals
func CreatePageviewTotals(since string) {
tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
totals, err := datastore.PageviewCountPerPageAndDay(tomorrow, since)
if err != nil {
log.Fatal(err)
}
err = datastore.SavePageTotals("pageviews", totals)
if err != nil {
log.Fatal(err)
}
}

39
pkg/count/referrers.go
View File

@ -1,39 +0,0 @@
package count
import (
"log"
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Referrers returns a point slice containing browser data per browser name
func Referrers(before int64, after int64, limit int64) ([]*models.Total, error) {
points, err := datastore.TotalsPerReferrer(before, after, limit)
if err != nil {
return nil, err
}
total, err := datastore.TotalReferrers(before, after)
if err != nil {
return nil, err
}
points = calculatePercentagesOfTotal(points, total)
return points, nil
}
// CreateReferrerTotals aggregates screen data into daily totals
func CreateReferrerTotals(since string) {
tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
totals, err := datastore.ReferrerCountPerDay(tomorrow, since)
if err != nil {
log.Fatal(err)
}
err = datastore.SaveTotals("referrers", totals)
if err != nil {
log.Fatal(err)
}
}

39
pkg/count/screens.go
View File

@ -1,39 +0,0 @@
package count
import (
"log"
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Screens returns a point slice containing screen data per size
func Screens(before int64, after int64, limit int64) ([]*models.Total, error) {
points, err := datastore.TotalsPerScreen(before, after, limit)
if err != nil {
return nil, err
}
total, err := datastore.TotalScreens(before, after)
if err != nil {
return nil, err
}
points = calculatePercentagesOfTotal(points, total)
return points, nil
}
// CreateScreenTotals aggregates screen data into daily totals
func CreateScreenTotals(since string) {
tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
totals, err := datastore.ScreenCountPerDay(tomorrow, since)
if err != nil {
log.Fatal(err)
}
err = datastore.SaveTotals("screens", totals)
if err != nil {
log.Fatal(err)
}
}

23
pkg/count/visitors.go
View File

@ -1,23 +0,0 @@
package count
import (
"log"
"time"
"github.com/usefathom/fathom/pkg/datastore"
)
// CreateVisitorTotals aggregates visitor data into daily totals
func CreateVisitorTotals(since string) {
tomorrow := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
totals, err := datastore.VisitorCountPerDay(tomorrow, since)
if err != nil {
log.Fatal(err)
}
err = datastore.SaveVisitorTotals(totals)
if err != nil {
log.Fatal(err)
}
}

71
pkg/counter/counter.go Normal file
View File

@ -0,0 +1,71 @@
package counter
import (
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
func Aggregate() error {
// Get unprocessed pageviews
pageviews, err := datastore.GetRawPageviews()
if err != nil && err != datastore.ErrNoResults {
return err
}
// Do we have anything to process?
if len(pageviews) == 0 {
return nil
}
// site stats
date := time.Now()
siteStats, err := getSiteStats(date)
if err != nil {
return err
}
for _, p := range pageviews {
siteStats.Pageviews += 1
if p.IsNewVisitor {
siteStats.Visitors += 1
siteStats.BouncedN += 1
if p.IsBounce {
siteStats.Bounced += 1
}
// TODO: duration
}
}
// TODO: page stats
// TODO: referrer stats
err = datastore.SaveSiteStats(siteStats)
if err != nil {
return err
}
// TODO: delete data
return nil
}
func getSiteStats(date time.Time) (*models.SiteStats, error) {
stats, err := datastore.GetSiteStats(date)
if err != nil && err != datastore.ErrNoResults {
return nil, err
}
if stats == nil {
return &models.SiteStats{
Date: date,
}, nil
}
return stats, nil
}

17
pkg/datastore/bounces.go
View File

@ -1,17 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
func BouncesCountPerPageAndDay(before string, after string) ([]*models.Total, error) {
query := dbx.Rebind(`SELECT
pv.page_id,
( COUNT(*) * 100 ) DIV ( SELECT ( COUNT(*) ) FROM pageviews WHERE page_id = pv.page_id AND bounced IS NOT NULL ) AS count,
( COUNT(DISTINCT(pv.visitor_id)) * 100 ) DIV ( SELECT ( COUNT(*) ) FROM pageviews WHERE page_id = pv.page_id AND bounced IS NOT NULL ) AS count_unique,
DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
FROM pageviews pv
WHERE pv.bounced = 1 AND pv.bounced IS NOT NULL AND pv.timestamp < ? AND pv.timestamp > ?
GROUP BY pv.page_id, date_group`)
var results []*models.Total
err := dbx.Select(&results, query, before, after)
return results, err
}

21
pkg/datastore/browsers.go
View File

@ -1,21 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
func BrowserCountPerDay(before string, after string) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
v.browser_name AS value,
COUNT(*) AS count,
COUNT(DISTINCT(pv.visitor_id)) AS count_unique,
DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
FROM pageviews pv
LEFT JOIN visitors v ON v.id = pv.visitor_id
WHERE pv.timestamp < ? AND pv.timestamp > ?
GROUP BY date_group, v.browser_name`)
err := dbx.Select(&results, query, before, after)
return results, err
}

1
pkg/datastore/datastore.go
View File

@ -44,6 +44,7 @@ func getDSN(driver string, host string, name string, user string, password strin
return dsn
}
// TODO: Move to command (but still auto-run on boot).
func runMigrations(driver string) {
migrations := migrate.FileMigrationSource{
Dir: "pkg/datastore/migrations", // TODO: Move to bindata

21
pkg/datastore/languages.go
View File

@ -1,21 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
func LanguageCountPerDay(before string, after string) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
v.browser_language AS value,
COUNT(*) AS count,
COUNT(DISTINCT(pv.visitor_id)) AS count_unique,
DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
FROM pageviews pv
LEFT JOIN visitors v ON v.id = pv.visitor_id
WHERE pv.timestamp < ? AND pv.timestamp > ?
GROUP BY date_group, v.browser_language`)
err := dbx.Select(&results, query, before, after)
return results, err
}

17
pkg/datastore/migrations/6_raw_pageviews.sql Normal file
View File

@ -0,0 +1,17 @@
-- +migrate Up
CREATE TABLE raw_pageviews(
id INT AUTO_INCREMENT PRIMARY KEY NOT NULL,
session_id VARCHAR(16) NOT NULL,
pathname VARCHAR(255) NOT NULL,
is_new_visitor TINYINT(1) NOT NULL,
is_unique TINYINT(1) NOT NULL,
is_bounce TINYINT(1) NULL,
referrer VARCHAR(255) NULL,
duration INT(4) NULL,
timestamp DATETIME NOT NULL
);
-- +migrate Down
DROP TABLE IF EXISTS raw_pageviews;

16
pkg/datastore/migrations/7_daily_page_stats.sql Normal file
View File

@ -0,0 +1,16 @@
-- +migrate Up
CREATE TABLE daily_page_stats(
pathname VARCHAR(255) NOT NULL,
views INT NOT NULL,
unique_views INT NOT NULL,
bounced INT NOT NULL,
bounced_n INT NOT NULL,
avg_duration INT NOT NULL,
avg_duration_n INT NOT NULL,
date DATE NOT NULL
);
-- +migrate Down
DROP TABLE IF EXISTS daily_page_stats;

15
pkg/datastore/migrations/8_daily_site_stats.sql Normal file
View File

@ -0,0 +1,15 @@
-- +migrate Up
CREATE TABLE daily_site_stats(
visitors INT NOT NULL,
pageviews INT NOT NULL,
bounced INT NOT NULL,
bounced_n INT NOT NULL,
avg_duration INT NOT NULL,
avg_duration_n INT NOT NULL,
date DATE NOT NULL
);
-- +migrate Down
DROP TABLE IF EXISTS daily_site_stats;

16
pkg/datastore/migrations/9_daily_referrer_stats.sql Normal file
View File

@ -0,0 +1,16 @@
-- +migrate Up
CREATE TABLE daily_referrer_stats(
url VARCHAR(255) NOT NULL,
visitors INT NOT NULL,
pageviews INT NOT NULL,
bounced INT NOT NULL,
bounced_n INT NOT NULL,
avg_duration INT NOT NULL,
avg_duration_n INT NOT NULL,
date DATE NOT NULL
);
-- +migrate Down
DROP TABLE IF EXISTS daily_referrer_stats;

1
pkg/datastore/page_stats.go Normal file
View File

@ -0,0 +1 @@
package datastore

34
pkg/datastore/pages.go
View File

@ -1,34 +0,0 @@
package datastore
import (
"database/sql"
"github.com/usefathom/fathom/pkg/models"
)
// GetPageByHostnameAndPath retrieves a page from the connected database
func GetPageByHostnameAndPath(hostname, path string) (*models.Page, error) {
p := &models.Page{}
query := dbx.Rebind(`SELECT * FROM pages WHERE hostname = ? AND path = ? LIMIT 1`)
err := dbx.Get(p, query, hostname, path)
if err != nil {
if err == sql.ErrNoRows {
return nil, ErrNoResults
}
return nil, err
}
return p, nil
}
// SavePage inserts the page model in the connected database
func SavePage(p *models.Page) error {
query := dbx.Rebind(`INSERT INTO pages(scheme, hostname, path) VALUES(?, ?, ?)`)
result, err := dbx.Exec(query, p.Scheme, p.Hostname, p.Path)
if err != nil {
return err
}
p.ID, _ = result.LastInsertId()
return nil
}

3
pkg/datastore/pageviews.go
View File

@ -1,5 +1,6 @@
package datastore
/*
import (
"database/sql"
@ -95,3 +96,5 @@ func PageviewCountPerPageAndDay(before string, after string) ([]*models.Total, e
err := dbx.Select(&results, query, before, after)
return results, err
}
*/

32
pkg/datastore/raw_pageviews.go Normal file
View File

@ -0,0 +1,32 @@
package datastore
import (
"github.com/usefathom/fathom/pkg/models"
)
const sqlInsertRawPageview = `INSERT INTO raw_pageviews(session_id, pathname, is_new_visitor, is_unique, is_bounce, referrer, duration, timestamp) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`
const sqlSelectRawPageviews = `SELECT * FROM raw_pageviews`
// SaveRawPageview inserts a single pageview model into the connected database
func SaveRawPageview(p *models.RawPageview) error {
query := dbx.Rebind(sqlInsertRawPageview)
result, err := dbx.Exec(query, p.SessionID, p.Pathname, p.IsNewVisitor, p.IsUnique, p.IsBounce, p.Referrer, p.Duration, p.Timestamp)
if err != nil {
return err
}
p.ID, _ = result.LastInsertId()
return nil
}
// SaveRawPageviews inserts multiple pageviews
func SaveRawPageviews(p []*models.RawPageview) error {
return nil // TODO: Implement this method
}
func GetRawPageviews() ([]*models.RawPageview, error) {
var results []*models.RawPageview
query := dbx.Rebind(sqlSelectRawPageviews)
err := dbx.Select(&results, query)
return results, err
}

1
pkg/datastore/referrer_stats.go Normal file
View File

@ -0,0 +1 @@
package datastore

22
pkg/datastore/referrers.go
View File

@ -1,22 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
func ReferrerCountPerDay(before string, after string) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
pv.referrer_url AS value,
COUNT(*) AS count,
COUNT(DISTINCT(pv.visitor_id)) AS count_unique,
DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
FROM pageviews pv
WHERE pv.referrer_url IS NOT NULL
AND pv.referrer_url != ''
AND pv.timestamp < ? AND pv.timestamp > ?
GROUP BY date_group, pv.referrer_url`)
err := dbx.Select(&results, query, before, after)
return results, err
}

21
pkg/datastore/screens.go
View File

@ -1,21 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
func ScreenCountPerDay(before string, after string) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
v.screen_resolution AS value,
COUNT(*) AS count,
COUNT(DISTINCT(pv.visitor_id)) AS count_unique,
DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
FROM pageviews pv
LEFT JOIN visitors v ON v.id = pv.visitor_id
WHERE pv.timestamp < ? AND pv.timestamp > ?
GROUP BY date_group, v.screen_resolution`)
err := dbx.Select(&results, query, before, after)
return results, err
}

35
pkg/datastore/site_stats.go Normal file
View File

@ -0,0 +1,35 @@
package datastore
import (
"database/sql"
"github.com/usefathom/fathom/pkg/models"
"time"
)
const sqlSelectSiteStat = `SELECT * FROM daily_site_stats WHERE date = ? LIMIT 1`
const sqlInsertSiteStats = `INSERT INTO daily_site_stats(visitors, pageviews, bounced, bounced_n, avg_duration, avg_duration_n, date) VALUES(?, ?, ?, ?, ?, ?, ?)`
/*
visitors INT NOT NULL,
pageviews INT NOT NULL,
bounced INT NOT NULL,
bounced_n INT NOT NULL,
avg_duration INT NOT NULL,
avg_duration_n INT NOT NULL,
date DATE NOT NULL
*/
func GetSiteStats(date time.Time) (*models.SiteStats, error) {
stats := &models.SiteStats{}
query := dbx.Rebind(sqlSelectSiteStat)
err := dbx.Get(stats, query, date)
if err != nil && err == sql.ErrNoRows {
return nil, ErrNoResults
}
return stats, err
}
func SaveSiteStats(s *models.SiteStats) error {
query := dbx.Rebind(sqlInsertSiteStats)
_, err := dbx.Exec(query, s.Visitors, s.Pageviews, s.Bounced, s.BouncedN, s.AvgDuration, s.AvgDurationN, s.Date)
return err
}

20
pkg/datastore/time_on_site.go
View File

@ -1,20 +0,0 @@
package datastore
// TODO: Store in visitors table? Or other table?
func AvgTimeOnSite(before int64, after int64) (int64, error) {
query := dbx.Rebind(`
SELECT ROUND(AVG(time_on_site)) FROM (
SELECT SUM(time_on_page) AS time_on_site
FROM pageviews
WHERE time_on_page > 0 AND UNIX_TIMESTAMP(timestamp) < ? AND UNIX_TIMESTAMP(timestamp) > ?
GROUP BY visitor_id
) AS time_on_site_query`)
var total int64
err := dbx.Get(&total, query, before, after)
if err != nil {
return 0, err
}
return total, nil
}

35
pkg/datastore/total.go
View File

@ -1,35 +0,0 @@
package datastore
import (
"fmt"
"github.com/usefathom/fathom/pkg/models"
)
func SaveTotals(metric string, totals []*models.Total) error {
query := fmt.Sprintf(`INSERT INTO total_%s( value, count, count_unique, date) VALUES( ?, ?, ?, ? ) ON DUPLICATE KEY UPDATE count = ?, count_unique = ?`, metric)
query = dbx.Rebind(query)
tx, err := dbx.Begin()
if err != nil {
return err
}
stmt, err := tx.Prepare(query)
if err != nil {
return err
}
defer stmt.Close()
for _, t := range totals {
result, err := stmt.Exec(t.Value, t.Count, t.CountUnique, t.Date, t.Count, t.CountUnique)
if err != nil {
return err
}
t.ID, _ = result.LastInsertId()
}
err = tx.Commit()
return err
}

33
pkg/datastore/total_bounces.go
View File

@ -1,33 +0,0 @@
package datastore
// TotalBounces returns the total number of pageviews between the given timestamps
func TotalBounces(before int64, after int64) (int64, error) {
var total int64
query := dbx.Rebind(`
SELECT COALESCE(ROUND(AVG(t.count), 0), 0)
FROM total_bounced t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
if err != nil {
return 0, err
}
return total, nil
}
// TotalUniqueBounces returns the total number of unique pageviews between the given timestamps
func TotalUniqueBounces(before int64, after int64) (int64, error) {
var total int64
query := dbx.Rebind(`
SELECT COALESCE(AVG(t.count_unique), 0)
FROM total_bounced t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
if err != nil {
return 0, err
}
return total, nil
}

47
pkg/datastore/total_browsers.go
View File

@ -1,47 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalBrowsers returns the total # of browsers between two given timestamps
func TotalBrowsers(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT
COALESCE(SUM(t.count), 0)
FROM total_browser_names t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
// TotalUniqueBrowsers returns the total # of unique browsers between two given timestamps
func TotalUniqueBrowsers(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT
COALESCE(SUM(t.count_unique), 0)
FROM total_browser_names t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
func TotalsPerBrowser(before int64, after int64, limit int64) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
t.value AS value,
COALESCE(SUM(t.count), 0) AS count,
COALESCE(SUM(t.count_unique), 0) AS count_unique
FROM total_browser_names t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY count DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
return results, err
}

49
pkg/datastore/total_languages.go
View File

@ -1,49 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalLanguages returns the total # of browser languages between two given timestamps
func TotalLanguages(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT
COALESCE(SUM(t.count), 0)
FROM total_browser_languages t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
// TotalUniqueLanguages returns the total # of unique browser languages between two given timestamps
func TotalUniqueLanguages(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT
COALESCE(SUM(t.count_unique), 0)
FROM total_browser_languages t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
func TotalsPerLanguage(before int64, after int64, limit int64) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
t.value AS value,
COALESCE(SUM(t.count), 0) AS count,
COALESCE(SUM(t.count_unique), 0) AS count_unique
FROM total_browser_languages t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY count DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
return results, err
}

106
pkg/datastore/total_pageviews.go
View File

@ -1,106 +0,0 @@
package datastore
import (
"fmt"
"github.com/usefathom/fathom/pkg/models"
)
// TotalPageviews returns the total number of pageviews between the given timestamps
func TotalPageviews(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT COALESCE(SUM(t.count), 0)
FROM total_pageviews t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
if err != nil {
return 0, err
}
return total, nil
}
// TotalUniquePageviews returns the total number of unique pageviews between the given timestamps
func TotalUniquePageviews(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT COALESCE(SUM(t.count_unique), 0)
FROM total_pageviews t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
if err != nil {
return 0, err
}
return total, nil
}
// TotalPageviewsPerDay returns a slice of data points representing the number of pageviews per day
func TotalPageviewsPerDay(before int64, after int64) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
CONCAT(p.scheme, "://", p.hostname, p.path) AS value,
COALESCE(SUM(t.count), 0) AS count,
COALESCE(SUM(t.count_unique), 0) AS count_unique,
DATE_FORMAT(t.date, '%Y-%m-%d') AS label
FROM total_pageviews t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY label, p.hostname, p.path, p.scheme`)
err := dbx.Select(&results, query, before, after)
if err != nil {
return results, err
}
return results, nil
}
// TotalPageviewsPerPage returns a set of pageview counts, grouped by page (hostname + path)
func TotalPageviewsPerPage(before int64, after int64, limit int64) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
CONCAT(p.scheme, "://", p.hostname, p.path) AS value,
COALESCE(SUM(t.count), 0) AS count,
COALESCE(SUM(t.count_unique), 0) AS count_unique
FROM total_pageviews t
LEFT JOIN pages p ON p.id = t.page_id
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY p.hostname, p.path, p.scheme
ORDER BY count DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
if err != nil {
return results, err
}
return results, nil
}
// SavePageviewTotals saves the given totals in the connected database
// Differs slightly from the metric specific totals because of the normalized pages (to save storage)
func SavePageTotals(metric string, totals []*models.Total) error {
tx, err := dbx.Begin()
if err != nil {
return nil
}
query := fmt.Sprintf(`INSERT INTO total_%s( page_id, count, count_unique, date ) VALUES( ?, ?, ?, ? ) ON DUPLICATE KEY UPDATE count = ?, count_unique = ?`, metric)
query = dbx.Rebind(query)
stmt, err := tx.Prepare(query)
if err != nil {
return err
}
for _, t := range totals {
_, err = stmt.Exec(t.PageID, t.Count, t.CountUnique, t.Date, t.Count, t.CountUnique)
}
err = tx.Commit()
return err
}

49
pkg/datastore/total_referrers.go
View File

@ -1,49 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalReferrers returns the total # of referrers between two given timestamps
func TotalReferrers(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT
COALESCE(SUM(t.count), 0)
FROM total_referrers t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
// TotalUniqueReferrers returns the total # of unique referrers between two given timestamps
func TotalUniqueReferrers(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT
COALESCE(SUM(t.count_unique), 0)
FROM total_referrers t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
func TotalsPerReferrer(before int64, after int64, limit int64) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
t.value,
COALESCE(SUM(t.count), 0) AS count,
COALESCE(SUM(t.count_unique), 0) AS count_unique
FROM total_referrers t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY count DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
return results, err
}

47
pkg/datastore/total_screens.go
View File

@ -1,47 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalScreens returns the total # of screens between two given timestamps
func TotalScreens(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT
COALESCE(SUM(t.count), 0)
FROM total_screens t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
// TotalUniqueScreens returns the total # of unique screens between two given timestamps
func TotalUniqueScreens(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT
COALESCE(SUM(t.count_unique), 0)
FROM total_screens t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
func TotalsPerScreen(before int64, after int64, limit int64) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
t.value AS value,
COALESCE(SUM(t.count), 0) AS count,
COALESCE(SUM(t.count_unique), 0) AS count_unique
FROM total_screens t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY count DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
return results, err
}

52
pkg/datastore/total_visitors.go
View File

@ -1,52 +0,0 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalVisitors returns the number of total visitors between the given timestamps
func TotalVisitors(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`
SELECT COALESCE(SUM(t.count), 0)
FROM total_visitors t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
return total, err
}
// TotalVisitorsPerDay returns a point slice containing visitor data per day
func TotalVisitorsPerDay(before int64, after int64) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`SELECT
COALESCE(SUM(t.count), 0) AS count,
COALESCE(SUM(t.count_unique), 0) AS count_unique,
DATE_FORMAT(t.date, '%Y-%m-%d') AS label
FROM total_visitors t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY label`)
err := dbx.Select(&results, query, before, after)
return results, err
}
// SaveVisitorTotals saves the given totals in the connected datastore
func SaveVisitorTotals(totals []*models.Total) error {
tx, err := dbx.Begin()
if err != nil {
return nil
}
query := dbx.Rebind(`INSERT INTO total_visitors( count, date ) VALUES( ?, ? ) ON DUPLICATE KEY UPDATE count = ?`)
stmt, err := tx.Prepare(query)
if err != nil {
return err
}
for _, t := range totals {
_, err = stmt.Exec(t.Count, t.Date, t.Count)
}
err = tx.Commit()
return err
}

62
pkg/datastore/visitors.go
View File

@ -1,62 +0,0 @@
package datastore
import (
"database/sql"
"github.com/usefathom/fathom/pkg/models"
)
// GetVisitorByKey ...
func GetVisitorByKey(key string) (*models.Visitor, error) {
v := &models.Visitor{}
query := dbx.Rebind(`SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1`)
err := dbx.Get(v, query, key)
if err != nil {
if err == sql.ErrNoRows {
return nil, ErrNoResults
}
return nil, err
}
return v, nil
}
// SaveVisitor inserts a single visitor model into the connected database
func SaveVisitor(v *models.Visitor) error {
query := dbx.Rebind(`INSERT INTO visitors(visitor_key, device_os, browser_name, browser_version, browser_language, screen_resolution, country) VALUES( ?, ?, ?, ?, ?, ?, ? )`)
result, err := dbx.Exec(query, v.Key, v.DeviceOS, v.BrowserName, v.BrowserVersion, v.BrowserLanguage, v.ScreenResolution, v.Country)
if err != nil {
return err
}
v.ID, _ = result.LastInsertId()
return nil
}
// RealtimeVisitors returns the total number of visitors in the last 3 minutes
// TODO: Query visitors table instead, using a last_seen column.
func RealtimeVisitors() (int, error) {
var result int
query := dbx.Rebind(`
SELECT COUNT(DISTINCT(pv.visitor_id))
FROM pageviews pv
WHERE pv.timestamp >= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 3 HOUR_MINUTE) AND pv.timestamp <= CURRENT_TIMESTAMP`)
err := dbx.Get(&result, query)
return result, err
}
func VisitorCountPerDay(before string, after string) ([]*models.Total, error) {
var results []*models.Total
query := dbx.Rebind(`
SELECT
COUNT(DISTINCT(pv.visitor_id)) AS count,
DATE_FORMAT(pv.timestamp, '%Y-%m-%d') AS date_group
FROM pageviews pv
WHERE pv.timestamp < ? AND pv.timestamp > ?
GROUP BY date_group`)
err := dbx.Select(&results, query, before, after)
return results, err
}

8
pkg/models/count.go
View File

@ -1,8 +0,0 @@
package models
type Count struct {
URL string `json:"url"`
Views int64 `json:"views"`
Uniques int64 `json:"uniques"`
PercentOfTotal float64 `json:"percent_of_total"`
}

9
pkg/models/page.go
View File

@ -1,9 +0,0 @@
package models
type Page struct {
ID int64 `json:"-"`
Scheme string `json:"scheme"`
Hostname string `json:"hostname"`
Path string `json:"path"`
Title string `json:"title"`
}

16
pkg/models/page_stats.go Normal file
View File

@ -0,0 +1,16 @@
package models
import (
"time"
)
type PageStats struct {
Pathname string `db:"pathname"`
Views int64 `db:"views"`
UniqueViews int64 `db:"unique_views"`
Bounced int64 `db:"bounced"`
BouncedN int64 `db:"bounced_n"`
AvgDuration int64 `db:"avg_duration"`
AvgDurationN int64 `db:"avg_duration_n"`
Date time.Time `db:"date"`
}

17
pkg/models/raw_pageview.go Normal file
View File

@ -0,0 +1,17 @@
package models
import (
"time"
)
type RawPageview struct {
ID int64 `db:"id"`
SessionID string `db:"session_id"`
Pathname string `db:"pathname"`
IsNewVisitor bool `db:"is_new_visitor"`
IsUnique bool `db:"is_unique"`
IsBounce bool `db:"is_bounce"`
Referrer string `db:"referrer"`
Duration int64 `db:"duration"`
Timestamp time.Time `db:"timestamp"`
}

16
pkg/models/referrer_stats.go Normal file
View File

@ -0,0 +1,16 @@
package models
import (
"time"
)
type ReferrerStats struct {
URL string `db:"url"`
Visitors int64 `db:"visitors"`
Pageviews int64 `db:"pageviews"`
Bounced int64 `db:"bounced"`
BouncedN int64 `db:"bounced_n"`
AvgDuration int64 `db:"avg_duration"`
AvgDurationN int64 `db:"avg_duration_n"`
Date time.Time `db:"date"`
}

15
pkg/models/site_stats.go Normal file
View File

@ -0,0 +1,15 @@
package models
import (
"time"
)
type SiteStats struct {
Visitors int64 `db:"visitors"`
Pageviews int64 `db:"pageviews"`
Bounced int64 `db:"bounced"`
BouncedN int64 `db:"bounced_n"`
AvgDuration int64 `db:"avg_duration"`
AvgDurationN int64 `db:"avg_duration_n"`
Date time.Time `db:"date"`
}

12
pkg/models/total.go
View File

@ -1,12 +0,0 @@
package models
// Total represents a daily aggregated total for a metric
type Total struct {
ID int64 `json:"-"`
PageID int64 `db:"page_id" json:"-"`
Value string `db:"value" json:"value"`
Count int64 `db:"count" json:"count"`
CountUnique int64 `db:"count_unique" json:"count_unique"`
PercentageOfTotal float64 `db:"-" json:"percentage_of_total"`
Date string `db:"date_group" json:"date,omitempty"`
}

12
pkg/models/visitor.go
View File

@ -1,12 +0,0 @@
package models
type Visitor struct {
ID int64
Key string
BrowserName string
BrowserVersion string
BrowserLanguage string
Country string
DeviceOS string
ScreenResolution string
}