Abstract calculating totals per category into count.Custom

ROADMAP.md

@@ -5,14 +5,13 @@ This is a general draft document for thoughts and todo's, without any structure
 
 ### What's cooking?
 
-- Archive `pageviews` table into daily totals for performance
+- Never query `pageviews` table directly.
-- Create archive on-the-fly if it does not exist yet
+- Create archive on-the-fly if it does not exist yet?
-- Bulk process tracking requests (Redis or in-memory?)
+- Bulk process tracking requests (Redis or in-memory)
-- Allow for multiple sites in same instance
+- Allow for multiple sites in same Ana instance
-- Update page title when it changes.
 - Custom date range picker
-- Choose a better name than "Ana"
+- Settle on a better name than "Ana"
 - Envelope API responses
-- Visual error handling on client-side.
+- Client-side error handling.
 - Geolocate IP addresses periodically.
 - Mask last part of IP address.

api/api.go

@@ -7,12 +7,6 @@ import (
   "net/http"
 )
 
-type Datapoint struct {
-  Count int
-  Label string
-  Percentage float64 `json:",omitempty"`
-}
-
 const defaultPeriod = 7
 const defaultLimit = 10
 
@@ -23,39 +17,6 @@ func checkError(err error) {
   }
 }
 
-func fillDatapoints(start int64, end int64, step time.Duration, points []Datapoint) []Datapoint {
-  // be smart about received timestamps
-  if start > end {
-    tmp := end
-    end = start
-    start = tmp
-  }
-
-  startTime := time.Unix(start, 0)
-  endTime := time.Unix(end, 0)
-  newPoints := make([]Datapoint, 0)
-
-  for startTime.Before(endTime) || startTime.Equal(endTime) {
-    point := Datapoint{
-      Count: 0,
-      Label: startTime.Format("2006-01-02"),
-    }
-
-    for j, p := range points {
-      if p.Label == point.Label || p.Label == startTime.Format("2006-01") {
-        point.Count = p.Count
-        points[j] = points[len(points)-1]
-        break
-      }
-    }
-
-    newPoints = append(newPoints, point)
-    startTime = startTime.Add(step)
-  }
-
-  return newPoints
-}
-
 func getRequestedLimit(r *http.Request) int {
   limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
   if err != nil || limit == 0 {

api/auth.go

@@ -15,7 +15,7 @@ type Login struct {
   Password string `json:"password"`
 }
 
-var store = sessions.NewFilesystemStore( "./storage/sessions/", []byte(os.Getenv("ANA_SECRET_KEY")))
+var store = sessions.NewFilesystemStore("./storage/sessions/", []byte(os.Getenv("ANA_SECRET_KEY")))
 
 // URL: POST /api/session
 var LoginHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

api/browsers.go

@@ -2,7 +2,6 @@ package api
 
 import (
   "net/http"
-  "github.com/dannyvankooten/ana/db"
   "github.com/dannyvankooten/ana/count"
   "encoding/json"
 )
@@ -15,7 +14,7 @@ var GetBrowsersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Re
   total := count.Visitors(before, after)
 
   // get rows
-  stmt, err := db.Conn.Prepare(`
+  results := count.Custom(`
     SELECT
     v.browser_name,
     COUNT(DISTINCT(pv.visitor_id)) AS count
@@ -24,22 +23,7 @@ var GetBrowsersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Re
     WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ? AND v.browser_name IS NOT NULL
     GROUP BY v.browser_name
     ORDER BY count DESC
-    LIMIT ?`)
+    LIMIT ?`, before, after, getRequestedLimit(r), total)
-  checkError(err)
-  defer stmt.Close()
-  rows, err := stmt.Query(before, after, defaultLimit)
-  checkError(err)
-  defer rows.Close()
-
-  results := make([]Datapoint, 0)
-  for rows.Next() {
-    var d Datapoint
-    err = rows.Scan(&d.Label, &d.Count);
-    checkError(err)
-
-    d.Percentage = float64(d.Count) / total * 100
-    results = append(results, d)
-  }
 
   w.Header().Set("Content-Type", "application/json")
   json.NewEncoder(w).Encode(results)

api/countries.go

@@ -2,7 +2,6 @@ package api
 
 import (
   "net/http"
-  "github.com/dannyvankooten/ana/db"
   "github.com/dannyvankooten/ana/count"
   "encoding/json"
 )
@@ -15,7 +14,7 @@ var GetCountriesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
   total := count.Visitors(before, after)
 
   // get rows
-  stmt, err := db.Conn.Prepare(`
+  results := count.Custom(`
     SELECT
     v.country,
     COUNT(DISTINCT(pv.visitor_id)) AS count
@@ -24,22 +23,7 @@ var GetCountriesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
     WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ? AND v.country IS NOT NULL
     GROUP BY v.country
     ORDER BY count DESC
-    LIMIT ?`)
+    LIMIT ?`, before, after, getRequestedLimit(r), total)
-  checkError(err)
-  defer stmt.Close()
-  rows, err := stmt.Query(before, after, getRequestedLimit(r))
-  checkError(err)
-  defer rows.Close()
-
-  results := make([]Datapoint, 0)
-  for rows.Next() {
-    var d Datapoint
-    err = rows.Scan(&d.Label, &d.Count);
-    checkError(err)
-
-    d.Percentage = float64(d.Count) / total * 100
-    results = append(results, d)
-  }
 
   w.Header().Set("Content-Type", "application/json")
   json.NewEncoder(w).Encode(results)

api/languages.go

@@ -2,7 +2,6 @@ package api
 
 import (
   "net/http"
-  "github.com/dannyvankooten/ana/db"
   "github.com/dannyvankooten/ana/count"
   "encoding/json"
 )
@@ -14,7 +13,7 @@ var GetLanguagesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
   // get total
   total := count.Visitors(before, after)
 
-  stmt, err := db.Conn.Prepare(`
+  results := count.Custom(`
     SELECT
     v.browser_language,
     COUNT(v.id) AS count
@@ -23,24 +22,8 @@ var GetLanguagesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
     WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ?
     GROUP BY v.browser_language
     ORDER BY count DESC
-    LIMIT ?`)
+    LIMIT ?`, before, after, getRequestedLimit(r), total,
-  checkError(err)
+  )
-  defer stmt.Close()
-
-  rows, err := stmt.Query(before, after, defaultLimit)
-  checkError(err)
-  defer rows.Close()
-
-  results := make([]Datapoint, 0)
-
-  for rows.Next() {
-    var d Datapoint
-    err = rows.Scan(&d.Label, &d.Count);
-    checkError(err)
-
-    d.Percentage = float64(d.Count) / total * 100
-    results = append(results, d)
-  }
 
   w.Header().Set("Content-Type", "application/json")
   json.NewEncoder(w).Encode(results)

api/pageviews.go

@@ -2,12 +2,10 @@ package api
 
 import (
   "net/http"
+  "encoding/json"
   "github.com/dannyvankooten/ana/models"
   "github.com/dannyvankooten/ana/db"
   "github.com/dannyvankooten/ana/count"
-  "encoding/json"
-  "github.com/gorilla/mux"
-  "time"
 )
 
 // URL: /api/pageviews
@@ -57,40 +55,8 @@ var GetPageviewsCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *h
 
 // URL: /api/pageviews/group/day
 var GetPageviewsPeriodCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-  vars := mux.Vars(r)
-  period := vars["period"]
-  formats := map[string]string {
-    "day": "%Y-%m-%d",
-    "month": "%Y-%m",
-  }
   before, after := getRequestedPeriods(r)
-  stmt, err := db.Conn.Prepare(`SELECT
+  results := count.PageviewsPerDay(before, after)
-      SUM(a.count) AS count,
-      DATE_FORMAT(a.date, ?) AS date_group
-    FROM archive a
-    WHERE a.metric = 'pageviews' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ?
-    GROUP BY date_group`)
-  checkError(err)
-  defer stmt.Close()
-
-  rows, err := stmt.Query(formats[period], before, after)
-  checkError(err)
-  defer rows.Close()
-
-  results := make([]Datapoint, 0)
-  for rows.Next() {
-    v := Datapoint{}
-    err = rows.Scan(&v.Count, &v.Label);
-    checkError(err)
-    results = append(results, v)
-  }
-
-  d := time.Hour * 24;
-  if period == "month" {
-    d = d * 30
-  }
-  results = fillDatapoints(before, after, d, results)
-
   w.Header().Set("Content-Type", "application/json")
   json.NewEncoder(w).Encode(results)
 })

api/referrers.go

@@ -2,10 +2,8 @@ package api
 
 import (
   "net/http"
-  "github.com/dannyvankooten/ana/db"
   "github.com/dannyvankooten/ana/count"
   "encoding/json"
-  "strings"
 )
 
 // URL: /api/referrers
@@ -16,7 +14,7 @@ var GetReferrersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
   total := count.Visitors(before, after)
 
   // get rows
-  stmt, err := db.Conn.Prepare(`
+  results := count.Custom(`
     SELECT
     pv.referrer_url,
     COUNT(DISTINCT(pv.visitor_id)) AS count
@@ -26,25 +24,7 @@ var GetReferrersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
     AND pv.referrer_url != ""
     GROUP BY pv.referrer_url
     ORDER BY count DESC
-    LIMIT ?`)
+    LIMIT ?`, before, after, getRequestedLimit(r), total)
-  checkError(err)
-  defer stmt.Close()
-  rows, err := stmt.Query(before, after, getRequestedLimit(r))
-  checkError(err)
-  defer rows.Close()
-
-  results := make([]Datapoint, 0)
-  for rows.Next() {
-    var d Datapoint
-    err = rows.Scan(&d.Label, &d.Count);
-    d.Label = strings.Replace(d.Label, "http://", "", 1)
-    d.Label = strings.Replace(d.Label, "https://", "", 1)
-    d.Label = strings.TrimRight(d.Label, "/")
-    checkError(err)
-
-    d.Percentage = float64(d.Count) / total * 100
-    results = append(results, d)
-  }
 
   w.Header().Set("Content-Type", "application/json")
   json.NewEncoder(w).Encode(results)

api/screen-resolutions.go

@@ -2,7 +2,6 @@ package api
 
 import (
   "net/http"
-  "github.com/dannyvankooten/ana/db"
   "github.com/dannyvankooten/ana/count"
   "encoding/json"
 )
@@ -15,7 +14,7 @@ var GetScreenResolutionsHandler = http.HandlerFunc(func(w http.ResponseWriter, r
   total := count.Visitors(before, after)
 
   // get rows
-  stmt, err := db.Conn.Prepare(`
+  results := count.Custom(`
     SELECT
     v.screen_resolution,
     COUNT(DISTINCT(pv.visitor_id)) AS count
@@ -24,22 +23,7 @@ var GetScreenResolutionsHandler = http.HandlerFunc(func(w http.ResponseWriter, r
     WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ?
     GROUP BY v.screen_resolution
     ORDER BY count DESC
-    LIMIT ?`)
+    LIMIT ?`, before, after, getRequestedLimit(r), total)
-  checkError(err)
-  defer stmt.Close()
-  rows, err := stmt.Query(before, after, getRequestedLimit(r))
-  checkError(err)
-  defer rows.Close()
-
-  results := make([]Datapoint, 0)
-  for rows.Next() {
-    var d Datapoint
-    err = rows.Scan(&d.Label, &d.Count);
-    checkError(err)
-
-    d.Percentage = float64(d.Count) / total * 100
-    results = append(results, d)
-  }
 
   w.Header().Set("Content-Type", "application/json")
   json.NewEncoder(w).Encode(results)

api/visitors.go

@@ -5,8 +5,6 @@ import (
   "github.com/dannyvankooten/ana/db"
   "github.com/dannyvankooten/ana/count"
   "encoding/json"
-  "github.com/gorilla/mux"
-  "time"
 )
 
 // URL: /api/visitors/count
@@ -30,40 +28,8 @@ var GetVisitorsRealtimeCountHandler = http.HandlerFunc(func(w http.ResponseWrite
 
 // URL: /api/visitors/count/group/:period
 var GetVisitorsPeriodCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-  vars := mux.Vars(r)
-  period := vars["period"]
-  formats := map[string]string {
-    "day": "%Y-%m-%d",
-    "month": "%Y-%m",
-  }
-
-  stmt, err := db.Conn.Prepare(`SELECT
-      SUM(a.count) AS count,
-      DATE_FORMAT(a.date, ?) AS date_group
-    FROM archive a
-    WHERE a.metric = 'visitors' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ?
-    GROUP BY date_group`)
-  checkError(err)
-  defer stmt.Close()
-
   before, after := getRequestedPeriods(r)
-  rows, err := stmt.Query(formats[period], before, after)
+  results := count.VisitorsPerDay(before, after)
-  checkError(err)
-
-  results := make([]Datapoint, 0)
-  defer rows.Close()
-  for rows.Next() {
-    v := Datapoint{}
-    err = rows.Scan(&v.Count, &v.Label);
-    checkError(err)
-    results = append(results, v)
-  }
-
-  d := time.Hour * 24;
-  if period == "month" {
-    d = d * 30
-  }
-  results = fillDatapoints(before, after, d, results)
   w.Header().Set("Content-Type", "application/json")
   json.NewEncoder(w).Encode(results)
 })

assets/js/components/Graph.js

@@ -22,12 +22,12 @@ function Chart(element, showPrimary, showSecondary) {
 
   var pageviewTip = d3.tip()
       .attr('class', 'd3-tip')
-      .html((d) => '<span>' + numbers.formatWithComma(d.Count) + '</span>' + ' pageviews')
+      .html((d) => '<span>' + numbers.formatWithComma(d.Value) + '</span>' + ' pageviews')
       .offset([-12, 0]);
 
   var visitorTip = d3.tip()
       .attr('class', 'd3-tip')
-      .html((d) => '<span>' + numbers.formatWithComma(d.Count) + '</span>' + ' visitors' )
+      .html((d) => '<span>' + numbers.formatWithComma(d.Value) + '</span>' + ' visitors' )
       .offset([-12, 0]);
 
   var graph = d3.select('#graph');
@@ -52,7 +52,7 @@ function Chart(element, showPrimary, showSecondary) {
   }
 
   function draw() {
-    var max = d3.max(showPrimary ? primaryData : secondaryData, (d) => d.Count);
+    var max = d3.max(showPrimary ? primaryData : secondaryData, (d) => d.Value);
     var ticks = primaryData.length;
     var xTick = Math.round(ticks / 7);
 
@@ -86,8 +86,8 @@ function Chart(element, showPrimary, showSecondary) {
 
       bars.append('rect')
         .attr('width', x.bandwidth())
-        .attr('height', (d) => (h - y(d.Count)) )
+        .attr('height', (d) => (h - y(d.Value)) )
-        .attr('y', (d) => y(d.Count))
+        .attr('y', (d) => y(d.Value))
         .on('mouseover', pageviewTip.show)
         .on('mouseout', pageviewTip.hide);
     }
@@ -102,8 +102,8 @@ function Chart(element, showPrimary, showSecondary) {
 
       visitorBars.append('rect')
         .attr('width', x.bandwidth() * 0.66 )
-        .attr('height', (d) => (h - y(d.Count)) )
+        .attr('height', (d) => (h - y(d.Value)) )
-        .attr('y', (d) => y(d.Count))
+        .attr('y', (d) => y(d.Value))
         .on('mouseover', visitorTip.show)
         .on('mouseout', visitorTip.hide);
     }

assets/js/components/Table.js

@@ -52,8 +52,9 @@ class Table extends Component {
     const after = before - ( period * dayInSeconds );
 
     Client.request(`${this.props.endpoint}?before=${before}&after=${after}&limit=${limit}`)
-      .then((d) => { this.setState({ loading: false, records: d })})
+      .then((d) => {
-      .catch((e) => { console.log(e) })
+        this.setState({ loading: false, records: d }
+      )}).catch((e) => { console.log(e) })
   }
 
   render() {
@@ -61,8 +62,8 @@ class Table extends Component {
       <tr>
         <td class="muted">{i+1}</td>
         {this.labelCell(p)}
-        <td>{numbers.formatWithComma(p.Count)}</td>
+        <td>{numbers.formatWithComma(p.Value)}</td>
-        <td>{Math.round(p.Percentage)}%</td>
+        <td>{Math.round(p.PercentageValue)}%</td>
       </tr>
     ));
 

assets/js/pages/dashboard.js

@@ -64,7 +64,7 @@ class Dashboard extends Component {
             <Table period={this.state.period} endpoint="screen-resolutions" title="Screen Resolutions" headers={["#", "Resolution", "Count", "%"]} />
           </div>
           <div class="col-2">
-            <Table period={this.state.period} endpoint="referrers" title="Referrers" headers={["#", "URL", "Count", "%"]} labelCell={(p) => ( <td><a href={"//" + p.Label}>{p.Label.substring(0, 15)}</a></td>)} />
+            <Table period={this.state.period} endpoint="referrers" title="Referrers" headers={["#", "URL", "Count", "%"]} labelCell={(p) => ( <td><a href={"//" + p.Label}>{p.Label.substring(0, 15).replace('https://', '').replace('http://', '')}</a></td>)} />
           </div>
           <div class="col-2">
             <Table period={this.state.period} endpoint="browsers" title="Browsers" headers={["#", "Browser", "Count", "%"]} onAuthError={this.props.onLogout} />

count/count.go

@@ -4,6 +4,7 @@ import(
   "database/sql"
   "log"
   "github.com/dannyvankooten/ana/db"
+  "time"
 )
 
 type Archive struct {
@@ -14,6 +15,12 @@ type Archive struct {
   Date string
 }
 
+type Point struct {
+  Label string
+  Value int
+  PercentageValue float64
+}
+
 func (a *Archive) Save(Conn *sql.DB) error {
   stmt, err := db.Conn.Prepare(`INSERT INTO archive(
     metric,
@@ -113,3 +120,65 @@ func checkError(err error) {
     log.Fatal(err)
   }
 }
+
+func Custom(sql string, before int64, after int64, limit int, total float64) []Point {
+  stmt, err := db.Conn.Prepare(sql)
+  checkError(err)
+  defer stmt.Close()
+
+  rows, err := stmt.Query(before, after, limit)
+  checkError(err)
+  defer rows.Close()
+
+  results := newPointSlice(rows, total)
+  return results
+}
+
+func newPointSlice(rows *sql.Rows, total float64) []Point {
+  results := make([]Point, 0)
+  for rows.Next() {
+    var d Point
+    err := rows.Scan(&d.Label, &d.Value);
+    checkError(err)
+
+    d.PercentageValue = float64(d.Value) / total * 100
+    results = append(results, d)
+  }
+
+  return results
+}
+
+
+func fill(start int64, end int64, points []Point) []Point {
+  // be smart about received timestamps
+  if start > end {
+    tmp := end
+    end = start
+    start = tmp
+  }
+
+  startTime := time.Unix(start, 0)
+  endTime := time.Unix(end, 0)
+  newPoints := make([]Point, 0)
+  step := time.Hour * 24
+
+  for startTime.Before(endTime) || startTime.Equal(endTime) {
+    point := Point{
+      Value: 0,
+      Label: startTime.Format("2006-01-02"),
+    }
+
+    for j, p := range points {
+      if p.Label == point.Label || p.Label == startTime.Format("2006-01") {
+        point.Value = p.Value
+        points[j] = points[len(points)-1]
+        break
+      }
+    }
+
+    newPoints = append(newPoints, point)
+    startTime = startTime.Add(step)
+  }
+
+  return newPoints
+}

count/pageviews.go

@@ -17,3 +17,30 @@ func Pageviews(before int64, after int64) float64 {
   stmt.QueryRow(before, after).Scan(&total)
   return total
 }
+
+func PageviewsPerDay(before int64, after int64) []Point {
+  stmt, err := db.Conn.Prepare(`SELECT
+      SUM(a.count) AS count,
+      DATE_FORMAT(a.date, '%Y-%m-%d') AS date_group
+    FROM archive a
+    WHERE a.metric = 'pageviews' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ?
+    GROUP BY date_group`)
+  checkError(err)
+  defer stmt.Close()
+
+  rows, err := stmt.Query(before, after)
+  checkError(err)
+  defer rows.Close()
+
+  results := make([]Point, 0)
+  defer rows.Close()
+  for rows.Next() {
+    p := Point{}
+    err = rows.Scan(&p.Value, &p.Label);
+    checkError(err)
+    results = append(results, p)
+  }
+
+  results = fill(after, before, results)
+  return results
+}

count/visitors.go

@@ -17,3 +17,30 @@ func Visitors(before int64, after int64) float64 {
   stmt.QueryRow(before, after).Scan(&total)
   return total
 }
+
+func VisitorsPerDay(before int64, after int64) ([]Point) {
+  stmt, err := db.Conn.Prepare(`SELECT
+      SUM(a.count) AS count,
+      DATE_FORMAT(a.date, '%Y-%m-%d') AS date_group
+    FROM archive a
+    WHERE a.metric = 'visitors' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ?
+    GROUP BY date_group`)
+  checkError(err)
+  defer stmt.Close()
+
+  rows, err := stmt.Query(before, after)
+  checkError(err)
+
+  results := make([]Point, 0)
+  defer rows.Close()
+  for rows.Next() {
+    p := Point{}
+    err = rows.Scan(&p.Value, &p.Label);
+    checkError(err)
+    results = append(results, p)
+  }
+
+  results = fill(after, before, results)
+
+  return results
+}