fix seed

ROADMAP.md

@@ -5,11 +5,11 @@ This is a general draft document for thoughts and todo's, without any structure
 
 ### What's cooking?
 
+- Fix seeding
 - Update page title when it changes.
 - Bulk process tracking requests (Redis or in-memory?)
 - Allow sorting in table overviews.
 - Choose a OS license & settle on name.
 - Envelope API responses & perhaps return total in table overview?
-- Show referrals.
+- Geolocate IP addresses periodically.
-- Geolocate unknown IP addresses periodically.
 - Mask last part of IP address.

api/collect.go

@@ -2,14 +2,11 @@ package api
 
 import (
   "net/http"
-  "log"
   "strings"
   "github.com/mssola/user_agent"
   "github.com/dannyvankooten/ana/models"
   "github.com/dannyvankooten/ana/db"
   "encoding/base64"
-  "crypto/md5"
-  "encoding/hex"
 )
 
 func getRequestIp(r *http.Request) string {
@@ -46,7 +43,8 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
     page.Save(db.Conn)
   }
 
-  // find or insert visitor
+  // find or insert visitor.
+  // TODO: Mask IP Address
   visitor := models.Visitor{
     IpAddress: getRequestIp(r),
     BrowserLanguage: q.Get("l"),
@@ -62,8 +60,8 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
       visitor.BrowserVersion = versionParts[0] + "." + versionParts[1]
   }
 
-  byteKey := md5.Sum([]byte(visitor.IpAddress + visitor.DeviceOS + visitor.BrowserName + visitor.ScreenResolution))
+  // query by unique visitor key
-  visitor.Key = hex.EncodeToString(byteKey[:])
+  visitor.GenerateKey()
   stmt, _ = db.Conn.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1")
   defer stmt.Close()
   err = stmt.QueryRow(visitor.Key).Scan(&visitor.ID)
@@ -71,20 +69,7 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
     visitor.Save(db.Conn)
   }
 
-  // prepare statement for inserting data
+  pageview := models.Pageview{
-  stmt, err = db.Conn.Prepare(`INSERT INTO pageviews(
-    page_id,
-    visitor_id,
-    referrer_url,
-    referrer_keyword
-    ) VALUES( ?, ?, ?, ? )`)
-  if err != nil {
-      log.Fatal(err.Error())
-  }
-  defer stmt.Close()
-
-  // TODO: Mask IP Address
-  visit := models.Pageview{
     PageID: page.ID,
     VisitorID: visitor.ID,
     ReferrerUrl: q.Get("ru"),
@@ -92,19 +77,11 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
   }
 
   // only store referrer URL if not coming from own site
-  if strings.Contains(visit.ReferrerUrl, page.Hostname)  {
+  if strings.Contains(pageview.ReferrerUrl, page.Hostname)  {
-    visit.ReferrerUrl = ""
+    pageview.ReferrerUrl = ""
   }
 
-  _, err = stmt.Exec(
+  pageview.Save(db.Conn)
-    visit.PageID,
-    visit.VisitorID,
-    visit.ReferrerUrl,
-    visit.ReferrerKeyword,
-  )
-  if err != nil {
-    log.Fatal(err)
-  }
 
   // don't you cache this
   w.Header().Set("Content-Type", "image/gif")

db/seed.go

@@ -2,11 +2,10 @@ package db
 
 import (
   "github.com/dannyvankooten/ana/models"
-  "log"
   "time"
   "math/rand"
-//  "fmt"
+  "fmt"
-//  "github.com/Pallinder/go-randomdata"
+  "github.com/Pallinder/go-randomdata"
 )
 
 var browserNames = []string {
@@ -90,65 +89,51 @@ func seedPages() []models.Page {
 }
 
 func Seed(n int) {
+  pages := seedPages()
 
-//  pages := seedPages()
+  // insert X random hits
+  for i := 0; i < n; i++ {
 
-  // prepare statement for inserting data
+    // print a dot as progress indicator
-  stmt, err := Conn.Prepare(`INSERT INTO pageviews(
+    fmt.Print(".")
-    page_id,
+
-    visitor_id,
+    // create or find visitor
-    referrer_keyword,
+    visitor := models.Visitor{
-    referrer_url,
+      IpAddress: randomdata.IpV4Address(),
-    timestamp
+      DeviceOS: "Linux",
-    ) VALUES( ?, ?, ? ?, ? )`)
+      BrowserName: randSliceElement(browserNames),
-  if err != nil {
+      BrowserVersion: "54.0",
-      log.Fatal(err)
+      BrowserLanguage: randSliceElement(browserLanguages),
+      ScreenResolution: randSliceElement(screenResolutions),
+      Country: randomdata.Country(randomdata.TwoCharCountry),
+    }
+    visitor.GenerateKey()
+
+    stmt, _ := Conn.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1")
+    defer stmt.Close()
+    err := stmt.QueryRow(visitor.Key).Scan(&visitor.ID)
+    if err != nil {
+      visitor.Save(Conn)
+    }
+
+    // generate random timestamp
+    date := randomDateBeforeNow();
+    timestamp := fmt.Sprintf("%s %d:%d:%d", date.Format("2006-01-02"), randInt(10, 24), randInt(10, 60), randInt(10, 60))
+
+    pv := models.Pageview{
+      VisitorID: visitor.ID,
+      ReferrerUrl: "",
+      ReferrerKeyword: "",
+      Timestamp: timestamp,
+    }
+
+    // insert between 1-4 pageviews for this visitor
+    for j := 0; j < randInt(1, 4); j++ {
+      page := pages[randInt(0, len(pages))]
+      pv.PageID = page.ID
+      pv.Save(Conn)
+    }
   }
-  defer stmt.Close()
-
-  // // insert X random hits
-  // log.Printf("Inserting %d visits", n)
-  // for i := 0; i < n; i++ {
-  //
-  //   // print a dot as progress indicator
-  //   fmt.Print(".")
-  //
-  //   // generate random timestamp
-  //   date := randomDateBeforeNow();
-  //   timestamp := fmt.Sprintf("%s %d:%d:%d", date.Format("2006-01-02"), randInt(10, 24), randInt(10, 60), randInt(10, 60))
-  //
-  //   visit := models.Visit{
-  //     IpAddress: randomdata.IpV4Address(),
-  //     BrowserName: randSliceElement(browserNames),
-  //     BrowserVersion: "54.0.2840.100",
-  //     BrowserLanguage: randSliceElement(browserLanguages),
-  //     ScreenResolution: randSliceElement(screenResolutions),
-  //     Country: randomdata.Country(randomdata.TwoCharCountry),
-  //     ReferrerUrl: "",
-  //     Timestamp: timestamp,
-  //   }
-  //
-  //   // insert between 1-4 pageviews for this visitor
-  //   for j := 0; j < randInt(1, 4); j++ {
-  //     page := pages[randInt(0, len(pages))]
-  //     visit.PageID = page.ID
-  //
-  //     _, err = stmt.Exec(
-  //       visit.PageID,
-  //       visit.BrowserLanguage,
-  //       visit.BrowserName,
-  //       visit.BrowserVersion,
-  //       visit.Country,
-  //       visit.IpAddress,
-  //       visit.ReferrerUrl,
-  //       visit.ScreenResolution,
-  //       visit.Timestamp,
-  //     )
-  //     if err != nil {
-  //       log.Fatal(err)
-  //     }
-  //   }
-  //}
 }
 
 func randomDate() time.Time {

gulpfile.js

@@ -10,13 +10,19 @@ const gutil = require('gulp-util')
 const sass = require('gulp-sass')
 const uglify = require('gulp-uglify')
 const pump = require('pump')
+const debug = process.env.NODE_ENV !== 'production';
 
-gulp.task('default', [ 'browserify', 'sass', 'tracker' ] );
+let defaultTasks = [ 'browserify', 'sass', 'tracker' ] ;
+if( ! debug ) {
+  defaultTasks.push( 'minify' );
+}
+
+gulp.task('default', defaultTasks);
 
 gulp.task('browserify', function () {
     return browserify({
             entries: './assets/js/script.js',
-            debug: false
+            debug: debug
         })
         .transform("babelify", {presets: ["es2015"]})
         .bundle()

models/pageview.go

@@ -1,5 +1,9 @@
 package models
 
+import (
+  "database/sql"
+)
+
 type Pageview struct {
   ID int64
   PageID int64
@@ -15,3 +19,32 @@ type Pageviews struct {
   Count int
   CountUnique int
 }
+
+func(pv *Pageview) Save(conn *sql.DB) error {
+   // prepare statement for inserting data
+   stmt, err := conn.Prepare(`INSERT INTO pageviews (
+     page_id,
+     visitor_id,
+     referrer_url,
+     referrer_keyword,
+     timestamp
+   ) VALUES( ?, ?, ?, ?, ? )`)
+   if err != nil {
+       return err
+   }
+   defer stmt.Close()
+
+   result, err := stmt.Exec(
+     pv.PageID,
+     pv.VisitorID,
+     pv.ReferrerUrl,
+     pv.ReferrerKeyword,
+     pv.Timestamp,
+   )
+   if err != nil {
+     return err
+   }
+
+   pv.ID, err = result.LastInsertId()
+   return err
+ }

models/visitor.go

@@ -2,6 +2,8 @@ package models
 
 import (
   "database/sql"
+  "crypto/md5"
+  "encoding/hex"
 )
 
 type Visitor struct {
@@ -50,3 +52,10 @@ func (v *Visitor) Save(conn *sql.DB) error {
     v.ID, err = result.LastInsertId()
     return err
   }
+
+  // GenerateKey generates the "unique" visitor key
+  func( v *Visitor) GenerateKey() string {
+    byteKey := md5.Sum([]byte(v.IpAddress + v.DeviceOS + v.BrowserName + v.ScreenResolution))
+    v.Key = hex.EncodeToString(byteKey[:])
+    return v.Key
+  }