This commit is contained in:
Danny van Kooten 2016-12-10 15:58:54 +01:00
parent f3e8731fae
commit 9755b72ebf
6 changed files with 105 additions and 95 deletions

View File

@ -5,11 +5,11 @@ This is a general draft document for thoughts and todo's, without any structure
### What's cooking? ### What's cooking?
- Fix seeding
- Update page title when it changes. - Update page title when it changes.
- Bulk process tracking requests (Redis or in-memory?) - Bulk process tracking requests (Redis or in-memory?)
- Allow sorting in table overviews. - Allow sorting in table overviews.
- Choose a OS license & settle on name. - Choose a OS license & settle on name.
- Envelope API responses & perhaps return total in table overview? - Envelope API responses & perhaps return total in table overview?
- Show referrals. - Geolocate IP addresses periodically.
- Geolocate unknown IP addresses periodically.
- Mask last part of IP address. - Mask last part of IP address.

View File

@ -2,14 +2,11 @@ package api
import ( import (
"net/http" "net/http"
"log"
"strings" "strings"
"github.com/mssola/user_agent" "github.com/mssola/user_agent"
"github.com/dannyvankooten/ana/models" "github.com/dannyvankooten/ana/models"
"github.com/dannyvankooten/ana/db" "github.com/dannyvankooten/ana/db"
"encoding/base64" "encoding/base64"
"crypto/md5"
"encoding/hex"
) )
func getRequestIp(r *http.Request) string { func getRequestIp(r *http.Request) string {
@ -46,7 +43,8 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
page.Save(db.Conn) page.Save(db.Conn)
} }
// find or insert visitor // find or insert visitor.
// TODO: Mask IP Address
visitor := models.Visitor{ visitor := models.Visitor{
IpAddress: getRequestIp(r), IpAddress: getRequestIp(r),
BrowserLanguage: q.Get("l"), BrowserLanguage: q.Get("l"),
@ -62,8 +60,8 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
visitor.BrowserVersion = versionParts[0] + "." + versionParts[1] visitor.BrowserVersion = versionParts[0] + "." + versionParts[1]
} }
byteKey := md5.Sum([]byte(visitor.IpAddress + visitor.DeviceOS + visitor.BrowserName + visitor.ScreenResolution)) // query by unique visitor key
visitor.Key = hex.EncodeToString(byteKey[:]) visitor.GenerateKey()
stmt, _ = db.Conn.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1") stmt, _ = db.Conn.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1")
defer stmt.Close() defer stmt.Close()
err = stmt.QueryRow(visitor.Key).Scan(&visitor.ID) err = stmt.QueryRow(visitor.Key).Scan(&visitor.ID)
@ -71,20 +69,7 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
visitor.Save(db.Conn) visitor.Save(db.Conn)
} }
// prepare statement for inserting data pageview := models.Pageview{
stmt, err = db.Conn.Prepare(`INSERT INTO pageviews(
page_id,
visitor_id,
referrer_url,
referrer_keyword
) VALUES( ?, ?, ?, ? )`)
if err != nil {
log.Fatal(err.Error())
}
defer stmt.Close()
// TODO: Mask IP Address
visit := models.Pageview{
PageID: page.ID, PageID: page.ID,
VisitorID: visitor.ID, VisitorID: visitor.ID,
ReferrerUrl: q.Get("ru"), ReferrerUrl: q.Get("ru"),
@ -92,19 +77,11 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
} }
// only store referrer URL if not coming from own site // only store referrer URL if not coming from own site
if strings.Contains(visit.ReferrerUrl, page.Hostname) { if strings.Contains(pageview.ReferrerUrl, page.Hostname) {
visit.ReferrerUrl = "" pageview.ReferrerUrl = ""
} }
_, err = stmt.Exec( pageview.Save(db.Conn)
visit.PageID,
visit.VisitorID,
visit.ReferrerUrl,
visit.ReferrerKeyword,
)
if err != nil {
log.Fatal(err)
}
// don't you cache this // don't you cache this
w.Header().Set("Content-Type", "image/gif") w.Header().Set("Content-Type", "image/gif")

View File

@ -2,11 +2,10 @@ package db
import ( import (
"github.com/dannyvankooten/ana/models" "github.com/dannyvankooten/ana/models"
"log"
"time" "time"
"math/rand" "math/rand"
// "fmt" "fmt"
// "github.com/Pallinder/go-randomdata" "github.com/Pallinder/go-randomdata"
) )
var browserNames = []string { var browserNames = []string {
@ -90,65 +89,51 @@ func seedPages() []models.Page {
} }
func Seed(n int) { func Seed(n int) {
pages := seedPages()
// pages := seedPages() // insert X random hits
for i := 0; i < n; i++ {
// prepare statement for inserting data // print a dot as progress indicator
stmt, err := Conn.Prepare(`INSERT INTO pageviews( fmt.Print(".")
page_id,
visitor_id, // create or find visitor
referrer_keyword, visitor := models.Visitor{
referrer_url, IpAddress: randomdata.IpV4Address(),
timestamp DeviceOS: "Linux",
) VALUES( ?, ?, ? ?, ? )`) BrowserName: randSliceElement(browserNames),
if err != nil { BrowserVersion: "54.0",
log.Fatal(err) BrowserLanguage: randSliceElement(browserLanguages),
ScreenResolution: randSliceElement(screenResolutions),
Country: randomdata.Country(randomdata.TwoCharCountry),
}
visitor.GenerateKey()
stmt, _ := Conn.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1")
defer stmt.Close()
err := stmt.QueryRow(visitor.Key).Scan(&visitor.ID)
if err != nil {
visitor.Save(Conn)
}
// generate random timestamp
date := randomDateBeforeNow();
timestamp := fmt.Sprintf("%s %d:%d:%d", date.Format("2006-01-02"), randInt(10, 24), randInt(10, 60), randInt(10, 60))
pv := models.Pageview{
VisitorID: visitor.ID,
ReferrerUrl: "",
ReferrerKeyword: "",
Timestamp: timestamp,
}
// insert between 1-4 pageviews for this visitor
for j := 0; j < randInt(1, 4); j++ {
page := pages[randInt(0, len(pages))]
pv.PageID = page.ID
pv.Save(Conn)
}
} }
defer stmt.Close()
// // insert X random hits
// log.Printf("Inserting %d visits", n)
// for i := 0; i < n; i++ {
//
// // print a dot as progress indicator
// fmt.Print(".")
//
// // generate random timestamp
// date := randomDateBeforeNow();
// timestamp := fmt.Sprintf("%s %d:%d:%d", date.Format("2006-01-02"), randInt(10, 24), randInt(10, 60), randInt(10, 60))
//
// visit := models.Visit{
// IpAddress: randomdata.IpV4Address(),
// BrowserName: randSliceElement(browserNames),
// BrowserVersion: "54.0.2840.100",
// BrowserLanguage: randSliceElement(browserLanguages),
// ScreenResolution: randSliceElement(screenResolutions),
// Country: randomdata.Country(randomdata.TwoCharCountry),
// ReferrerUrl: "",
// Timestamp: timestamp,
// }
//
// // insert between 1-4 pageviews for this visitor
// for j := 0; j < randInt(1, 4); j++ {
// page := pages[randInt(0, len(pages))]
// visit.PageID = page.ID
//
// _, err = stmt.Exec(
// visit.PageID,
// visit.BrowserLanguage,
// visit.BrowserName,
// visit.BrowserVersion,
// visit.Country,
// visit.IpAddress,
// visit.ReferrerUrl,
// visit.ScreenResolution,
// visit.Timestamp,
// )
// if err != nil {
// log.Fatal(err)
// }
// }
//}
} }
func randomDate() time.Time { func randomDate() time.Time {

View File

@ -10,13 +10,19 @@ const gutil = require('gulp-util')
const sass = require('gulp-sass') const sass = require('gulp-sass')
const uglify = require('gulp-uglify') const uglify = require('gulp-uglify')
const pump = require('pump') const pump = require('pump')
const debug = process.env.NODE_ENV !== 'production';
gulp.task('default', [ 'browserify', 'sass', 'tracker' ] ); let defaultTasks = [ 'browserify', 'sass', 'tracker' ] ;
if( ! debug ) {
defaultTasks.push( 'minify' );
}
gulp.task('default', defaultTasks);
gulp.task('browserify', function () { gulp.task('browserify', function () {
return browserify({ return browserify({
entries: './assets/js/script.js', entries: './assets/js/script.js',
debug: false debug: debug
}) })
.transform("babelify", {presets: ["es2015"]}) .transform("babelify", {presets: ["es2015"]})
.bundle() .bundle()
@ -31,7 +37,7 @@ gulp.task('browserify', function () {
gulp.task('minify', function(cb) { gulp.task('minify', function(cb) {
process.env.NODE_ENV = 'production'; process.env.NODE_ENV = 'production';
pump([ pump([
gulp.src('./static/js/*.js'), gulp.src('./static/js/*.js'),
uglify().on('error', gutil.log), uglify().on('error', gutil.log),

View File

@ -1,5 +1,9 @@
package models package models
import (
"database/sql"
)
type Pageview struct { type Pageview struct {
ID int64 ID int64
PageID int64 PageID int64
@ -15,3 +19,32 @@ type Pageviews struct {
Count int Count int
CountUnique int CountUnique int
} }
func(pv *Pageview) Save(conn *sql.DB) error {
// prepare statement for inserting data
stmt, err := conn.Prepare(`INSERT INTO pageviews (
page_id,
visitor_id,
referrer_url,
referrer_keyword,
timestamp
) VALUES( ?, ?, ?, ?, ? )`)
if err != nil {
return err
}
defer stmt.Close()
result, err := stmt.Exec(
pv.PageID,
pv.VisitorID,
pv.ReferrerUrl,
pv.ReferrerKeyword,
pv.Timestamp,
)
if err != nil {
return err
}
pv.ID, err = result.LastInsertId()
return err
}

View File

@ -2,6 +2,8 @@ package models
import ( import (
"database/sql" "database/sql"
"crypto/md5"
"encoding/hex"
) )
type Visitor struct { type Visitor struct {
@ -50,3 +52,10 @@ func (v *Visitor) Save(conn *sql.DB) error {
v.ID, err = result.LastInsertId() v.ID, err = result.LastInsertId()
return err return err
} }
// GenerateKey generates the "unique" visitor key
func( v *Visitor) GenerateKey() string {
byteKey := md5.Sum([]byte(v.IpAddress + v.DeviceOS + v.BrowserName + v.ScreenResolution))
v.Key = hex.EncodeToString(byteKey[:])
return v.Key
}