diff --git a/ROADMAP.md b/ROADMAP.md index f8cf8f9..ba94913 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,11 +5,11 @@ This is a general draft document for thoughts and todo's, without any structure ### What's cooking? +- Fix seeding - Update page title when it changes. - Bulk process tracking requests (Redis or in-memory?) - Allow sorting in table overviews. - Choose a OS license & settle on name. - Envelope API responses & perhaps return total in table overview? -- Show referrals. -- Geolocate unknown IP addresses periodically. +- Geolocate IP addresses periodically. - Mask last part of IP address. diff --git a/api/collect.go b/api/collect.go index f30821d..7c5e53f 100644 --- a/api/collect.go +++ b/api/collect.go @@ -2,14 +2,11 @@ package api import ( "net/http" - "log" "strings" "github.com/mssola/user_agent" "github.com/dannyvankooten/ana/models" "github.com/dannyvankooten/ana/db" "encoding/base64" - "crypto/md5" - "encoding/hex" ) func getRequestIp(r *http.Request) string { @@ -46,7 +43,8 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) { page.Save(db.Conn) } - // find or insert visitor + // find or insert visitor. + // TODO: Mask IP Address visitor := models.Visitor{ IpAddress: getRequestIp(r), BrowserLanguage: q.Get("l"), @@ -62,8 +60,8 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) { visitor.BrowserVersion = versionParts[0] + "." + versionParts[1] } - byteKey := md5.Sum([]byte(visitor.IpAddress + visitor.DeviceOS + visitor.BrowserName + visitor.ScreenResolution)) - visitor.Key = hex.EncodeToString(byteKey[:]) + // query by unique visitor key + visitor.GenerateKey() stmt, _ = db.Conn.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1") defer stmt.Close() err = stmt.QueryRow(visitor.Key).Scan(&visitor.ID) @@ -71,20 +69,7 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) { visitor.Save(db.Conn) } - // prepare statement for inserting data - stmt, err = db.Conn.Prepare(`INSERT INTO pageviews( - page_id, - visitor_id, - referrer_url, - referrer_keyword - ) VALUES( ?, ?, ?, ? )`) - if err != nil { - log.Fatal(err.Error()) - } - defer stmt.Close() - - // TODO: Mask IP Address - visit := models.Pageview{ + pageview := models.Pageview{ PageID: page.ID, VisitorID: visitor.ID, ReferrerUrl: q.Get("ru"), @@ -92,19 +77,11 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) { } // only store referrer URL if not coming from own site - if strings.Contains(visit.ReferrerUrl, page.Hostname) { - visit.ReferrerUrl = "" + if strings.Contains(pageview.ReferrerUrl, page.Hostname) { + pageview.ReferrerUrl = "" } - _, err = stmt.Exec( - visit.PageID, - visit.VisitorID, - visit.ReferrerUrl, - visit.ReferrerKeyword, - ) - if err != nil { - log.Fatal(err) - } + pageview.Save(db.Conn) // don't you cache this w.Header().Set("Content-Type", "image/gif") diff --git a/db/seed.go b/db/seed.go index 71ef364..50a1ea7 100644 --- a/db/seed.go +++ b/db/seed.go @@ -2,11 +2,10 @@ package db import ( "github.com/dannyvankooten/ana/models" - "log" "time" "math/rand" -// "fmt" -// "github.com/Pallinder/go-randomdata" + "fmt" + "github.com/Pallinder/go-randomdata" ) var browserNames = []string { @@ -90,65 +89,51 @@ func seedPages() []models.Page { } func Seed(n int) { + pages := seedPages() -// pages := seedPages() + // insert X random hits + for i := 0; i < n; i++ { - // prepare statement for inserting data - stmt, err := Conn.Prepare(`INSERT INTO pageviews( - page_id, - visitor_id, - referrer_keyword, - referrer_url, - timestamp - ) VALUES( ?, ?, ? ?, ? )`) - if err != nil { - log.Fatal(err) + // print a dot as progress indicator + fmt.Print(".") + + // create or find visitor + visitor := models.Visitor{ + IpAddress: randomdata.IpV4Address(), + DeviceOS: "Linux", + BrowserName: randSliceElement(browserNames), + BrowserVersion: "54.0", + BrowserLanguage: randSliceElement(browserLanguages), + ScreenResolution: randSliceElement(screenResolutions), + Country: randomdata.Country(randomdata.TwoCharCountry), + } + visitor.GenerateKey() + + stmt, _ := Conn.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1") + defer stmt.Close() + err := stmt.QueryRow(visitor.Key).Scan(&visitor.ID) + if err != nil { + visitor.Save(Conn) + } + + // generate random timestamp + date := randomDateBeforeNow(); + timestamp := fmt.Sprintf("%s %d:%d:%d", date.Format("2006-01-02"), randInt(10, 24), randInt(10, 60), randInt(10, 60)) + + pv := models.Pageview{ + VisitorID: visitor.ID, + ReferrerUrl: "", + ReferrerKeyword: "", + Timestamp: timestamp, + } + + // insert between 1-4 pageviews for this visitor + for j := 0; j < randInt(1, 4); j++ { + page := pages[randInt(0, len(pages))] + pv.PageID = page.ID + pv.Save(Conn) + } } - defer stmt.Close() - - // // insert X random hits - // log.Printf("Inserting %d visits", n) - // for i := 0; i < n; i++ { - // - // // print a dot as progress indicator - // fmt.Print(".") - // - // // generate random timestamp - // date := randomDateBeforeNow(); - // timestamp := fmt.Sprintf("%s %d:%d:%d", date.Format("2006-01-02"), randInt(10, 24), randInt(10, 60), randInt(10, 60)) - // - // visit := models.Visit{ - // IpAddress: randomdata.IpV4Address(), - // BrowserName: randSliceElement(browserNames), - // BrowserVersion: "54.0.2840.100", - // BrowserLanguage: randSliceElement(browserLanguages), - // ScreenResolution: randSliceElement(screenResolutions), - // Country: randomdata.Country(randomdata.TwoCharCountry), - // ReferrerUrl: "", - // Timestamp: timestamp, - // } - // - // // insert between 1-4 pageviews for this visitor - // for j := 0; j < randInt(1, 4); j++ { - // page := pages[randInt(0, len(pages))] - // visit.PageID = page.ID - // - // _, err = stmt.Exec( - // visit.PageID, - // visit.BrowserLanguage, - // visit.BrowserName, - // visit.BrowserVersion, - // visit.Country, - // visit.IpAddress, - // visit.ReferrerUrl, - // visit.ScreenResolution, - // visit.Timestamp, - // ) - // if err != nil { - // log.Fatal(err) - // } - // } - //} } func randomDate() time.Time { diff --git a/gulpfile.js b/gulpfile.js index 8eeef7e..9979ed7 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -10,13 +10,19 @@ const gutil = require('gulp-util') const sass = require('gulp-sass') const uglify = require('gulp-uglify') const pump = require('pump') +const debug = process.env.NODE_ENV !== 'production'; -gulp.task('default', [ 'browserify', 'sass', 'tracker' ] ); +let defaultTasks = [ 'browserify', 'sass', 'tracker' ] ; +if( ! debug ) { + defaultTasks.push( 'minify' ); +} + +gulp.task('default', defaultTasks); gulp.task('browserify', function () { return browserify({ entries: './assets/js/script.js', - debug: false + debug: debug }) .transform("babelify", {presets: ["es2015"]}) .bundle() @@ -31,7 +37,7 @@ gulp.task('browserify', function () { gulp.task('minify', function(cb) { process.env.NODE_ENV = 'production'; - + pump([ gulp.src('./static/js/*.js'), uglify().on('error', gutil.log), diff --git a/models/pageview.go b/models/pageview.go index cf60f95..11efc7a 100644 --- a/models/pageview.go +++ b/models/pageview.go @@ -1,5 +1,9 @@ package models +import ( + "database/sql" +) + type Pageview struct { ID int64 PageID int64 @@ -15,3 +19,32 @@ type Pageviews struct { Count int CountUnique int } + +func(pv *Pageview) Save(conn *sql.DB) error { + // prepare statement for inserting data + stmt, err := conn.Prepare(`INSERT INTO pageviews ( + page_id, + visitor_id, + referrer_url, + referrer_keyword, + timestamp + ) VALUES( ?, ?, ?, ?, ? )`) + if err != nil { + return err + } + defer stmt.Close() + + result, err := stmt.Exec( + pv.PageID, + pv.VisitorID, + pv.ReferrerUrl, + pv.ReferrerKeyword, + pv.Timestamp, + ) + if err != nil { + return err + } + + pv.ID, err = result.LastInsertId() + return err + } diff --git a/models/visitor.go b/models/visitor.go index 46210f7..4f0bc51 100644 --- a/models/visitor.go +++ b/models/visitor.go @@ -2,6 +2,8 @@ package models import ( "database/sql" + "crypto/md5" + "encoding/hex" ) type Visitor struct { @@ -50,3 +52,10 @@ func (v *Visitor) Save(conn *sql.DB) error { v.ID, err = result.LastInsertId() return err } + + // GenerateKey generates the "unique" visitor key + func( v *Visitor) GenerateKey() string { + byteKey := md5.Sum([]byte(v.IpAddress + v.DeviceOS + v.BrowserName + v.ScreenResolution)) + v.Key = hex.EncodeToString(byteKey[:]) + return v.Key + }