move pageview queries to datastore pkg & rebind query for other db engines than mysql

pkg/api/http.go

@@ -27,6 +27,7 @@ func HandleError(w http.ResponseWriter, r *http.Request, err error) {
 		"error":   err,
 	}).Error("error handling request")
 
+	w.WriteHeader(http.StatusInternalServerError)
 	w.Header().Set("Content-Type", "application/json")
 	w.Write([]byte("false"))
 }

pkg/api/pageviews.go

@@ -3,56 +3,13 @@ package api
 import (
 	"net/http"
 
-	"github.com/usefathom/fathom/pkg/count"
 	"github.com/usefathom/fathom/pkg/datastore"
 )
 
-type pageviews struct {
-	Hostname    string
-	Path        string
-	Count       int
-	CountUnique int
-}
-
 // URL: /api/pageviews
 var GetPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
 	before, after := getRequestedPeriods(r)
-
-	stmt, err := datastore.DB.Prepare(`
-		SELECT
-			p.hostname,
-			p.path,
-			SUM(t.count) AS count,
-			SUM(t.count_unique) AS count_unique
-		FROM total_pageviews t
-		LEFT JOIN pages p ON p.id = t.page_id
-		WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-		GROUP BY p.path, p.hostname
-		ORDER BY count DESC
-		LIMIT ?`)
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-
-	rows, err := stmt.Query(before, after, defaultLimit)
-	if err != nil {
-		return err
-	}
-	defer rows.Close()
-
-	results := make([]pageviews, 0)
-	for rows.Next() {
-		var p pageviews
-		err = rows.Scan(&p.Hostname, &p.Path, &p.Count, &p.CountUnique)
-		if err != nil {
-			return err
-		}
-
-		results = append(results, p)
-	}
-
-	err = rows.Err()
+	results, err := datastore.TotalPageviewsPerPage(before, after, defaultLimit)
 	if err != nil {
 		return err
 	}
@@ -63,13 +20,21 @@ var GetPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Reques
 // URL: /api/pageviews/count
 var GetPageviewsCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
 	before, after := getRequestedPeriods(r)
-	result := count.Pageviews(before, after)
+	result, err := datastore.TotalPageviews(before, after)
+	if err != nil {
+		return err
+	}
+
 	return respond(w, envelope{Data: result})
 })
 
 // URL: /api/pageviews/group/day
 var GetPageviewsPeriodCountHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
 	before, after := getRequestedPeriods(r)
-	results := count.PageviewsPerDay(before, after)
+	results, err := datastore.TotalPageviewsPerDay(before, after)
+	if err != nil {
+		return err
+	}
+
 	return respond(w, envelope{Data: results})
 })

pkg/count/count.go

@@ -41,8 +41,7 @@ func Archive() {
 	CreateBrowserTotals(lastArchived)
 	CreateReferrerTotals(lastArchived)
 
-	err := datastore.SetOption("last_archived", time.Now().Format("2006-01-02"))
-	checkError(err)
+	datastore.SetOption("last_archived", time.Now().Format("2006-01-02"))
 }
 
 // Save the Total in the given database connection + table
@@ -100,40 +99,6 @@ func calculatePointPercentages(points []Point, total int) []Point {
 	return points
 }
 
-func fill(start int64, end int64, points []Point) []Point {
-	// be smart about received timestamps
-	if start > end {
-		tmp := end
-		end = start
-		start = tmp
-	}
-
-	startTime := time.Unix(start, 0)
-	endTime := time.Unix(end, 0)
-	var newPoints []Point
-	step := time.Hour * 24
-
-	for startTime.Before(endTime) || startTime.Equal(endTime) {
-		point := Point{
-			Value: 0,
-			Label: startTime.Format("2006-01-02"),
-		}
-
-		for j, p := range points {
-			if p.Label == point.Label || p.Label == startTime.Format("2006-01") {
-				point.Value = p.Value
-				points[j] = points[len(points)-1]
-				break
-			}
-		}
-
-		newPoints = append(newPoints, point)
-		startTime = startTime.Add(step)
-	}
-
-	return newPoints
-}
-
 func queryTotalRows(sql string, lastArchived string) *sql.Rows {
 	stmt, err := datastore.DB.Prepare(sql)
 	checkError(err)

pkg/count/pageviews.go

@@ -2,51 +2,6 @@ package count
 
 import "github.com/usefathom/fathom/pkg/datastore"
 
-// Pageviews returns the total number of pageviews between the given timestamps
-func Pageviews(before int64, after int64) int {
-	var total int
-
-	// get total
-	stmt, err := datastore.DB.Prepare(`
-    SELECT
-    	IFNULL( SUM(t.count), 0 )
-    FROM total_pageviews t
-    WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
-	checkError(err)
-	defer stmt.Close()
-
-	stmt.QueryRow(before, after).Scan(&total)
-	return total
-}
-
-// PageviewsPerDay returns a slice of data points representing the number of pageviews per day
-func PageviewsPerDay(before int64, after int64) []Point {
-	stmt, err := datastore.DB.Prepare(`SELECT
-      SUM(t.count) AS count,
-      DATE_FORMAT(t.date, '%Y-%m-%d') AS date_group
-    FROM total_pageviews t
-    WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
-    GROUP BY date_group`)
-	checkError(err)
-	defer stmt.Close()
-
-	rows, err := stmt.Query(before, after)
-	checkError(err)
-	defer rows.Close()
-
-	var results []Point
-	defer rows.Close()
-	for rows.Next() {
-		p := Point{}
-		err = rows.Scan(&p.Value, &p.Label)
-		checkError(err)
-		results = append(results, p)
-	}
-
-	results = fill(after, before, results)
-	return results
-}
-
 // CreatePageviewTotals aggregates pageview data for each page into daily totals
 func CreatePageviewTotals(since string) {
 	stmt, err := datastore.DB.Prepare(`SELECT

pkg/count/visitors.go

@@ -52,8 +52,6 @@ func VisitorsPerDay(before int64, after int64) []Point {
 		results = append(results, p)
 	}
 
-	results = fill(after, before, results)
-
 	return results
 }
 

pkg/datastore/total_pageviews.go

@@ -0,0 +1,61 @@
+package datastore
+
+import (
+	"github.com/usefathom/fathom/pkg/models"
+)
+
+// TotalPageviews returns the total number of pageviews between the given timestamps
+func TotalPageviews(before int64, after int64) (int, error) {
+	var total int
+
+	query := dbx.Rebind(`SELECT IFNULL( SUM(t.count), 0 ) FROM total_pageviews t WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
+	err := dbx.Get(&total, query, before, after)
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}
+
+// TotalPageviewsPerDay returns a slice of data points representing the number of pageviews per day
+func TotalPageviewsPerDay(before int64, after int64) ([]*models.Point, error) {
+	var results []*models.Point
+
+	query := dbx.Rebind(`
+		SELECT
+	      SUM(t.count) AS value,
+	      DATE_FORMAT(t.date, '%Y-%m-%d') AS label
+	    FROM total_pageviews t
+	    WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
+	    GROUP BY label`)
+
+	err := dbx.Select(&results, query, before, after)
+	if err != nil {
+		return results, err
+	}
+
+	return results, nil
+}
+
+// TotalPageviewsPerPage returns a set of pageview counts, grouped by page (hostname + path)
+func TotalPageviewsPerPage(before int64, after int64, limit int64) ([]*models.PageviewCount, error) {
+	var results []*models.PageviewCount
+	query := dbx.Rebind(`
+		SELECT
+			p.hostname,
+			p.path,
+			SUM(t.count) AS count,
+			SUM(t.count_unique) AS countunique
+		FROM total_pageviews t
+		LEFT JOIN pages p ON p.id = t.page_id
+		WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
+		GROUP BY p.path, p.hostname
+		ORDER BY count DESC
+		LIMIT ?`)
+	err := dbx.Select(&results, query, before, after, limit)
+	if err != nil {
+		return results, err
+	}
+
+	return results, nil
+}

pkg/models/pageview.go

@@ -8,3 +8,10 @@ type Pageview struct {
 	ReferrerUrl     string
 	Timestamp       string
 }
+
+type PageviewCount struct {
+	Hostname    string
+	Path        string
+	Count       int
+	CountUnique int
+}

pkg/models/point.go

@@ -0,0 +1,8 @@
+package models
+
+// Point represents a data point, will always have a Label and Value
+type Point struct {
+	Label           string
+	Value           int
+	PercentageValue float64
+}