From 4468b706013071ed09768cc61e32ea183037b080 Mon Sep 17 00:00:00 2001 From: Danny van Kooten Date: Sun, 11 Dec 2016 12:52:10 +0100 Subject: [PATCH] Abstract calculating totals per category into count.Custom --- ROADMAP.md | 13 +++---- api/api.go | 39 ------------------- api/auth.go | 2 +- api/browsers.go | 20 +--------- api/countries.go | 20 +--------- api/languages.go | 23 ++---------- api/pageviews.go | 38 +------------------ api/referrers.go | 24 +----------- api/screen-resolutions.go | 20 +--------- api/visitors.go | 36 +----------------- assets/js/components/Graph.js | 14 +++---- assets/js/components/Table.js | 9 +++-- assets/js/pages/dashboard.js | 2 +- count/{archive.go => count.go} | 69 ++++++++++++++++++++++++++++++++++ count/pageviews.go | 27 +++++++++++++ count/visitors.go | 27 +++++++++++++ 16 files changed, 157 insertions(+), 226 deletions(-) rename count/{archive.go => count.go} (59%) diff --git a/ROADMAP.md b/ROADMAP.md index 8f60425..65a8e36 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,14 +5,13 @@ This is a general draft document for thoughts and todo's, without any structure ### What's cooking? -- Archive `pageviews` table into daily totals for performance -- Create archive on-the-fly if it does not exist yet -- Bulk process tracking requests (Redis or in-memory?) -- Allow for multiple sites in same instance -- Update page title when it changes. +- Never query `pageviews` table directly. +- Create archive on-the-fly if it does not exist yet? +- Bulk process tracking requests (Redis or in-memory) +- Allow for multiple sites in same Ana instance - Custom date range picker -- Choose a better name than "Ana" +- Settle on a better name than "Ana" - Envelope API responses -- Visual error handling on client-side. +- Client-side error handling. - Geolocate IP addresses periodically. - Mask last part of IP address. diff --git a/api/api.go b/api/api.go index f5cc829..c285d49 100644 --- a/api/api.go +++ b/api/api.go @@ -7,12 +7,6 @@ import ( "net/http" ) -type Datapoint struct { - Count int - Label string - Percentage float64 `json:",omitempty"` -} - const defaultPeriod = 7 const defaultLimit = 10 @@ -23,39 +17,6 @@ func checkError(err error) { } } -func fillDatapoints(start int64, end int64, step time.Duration, points []Datapoint) []Datapoint { - // be smart about received timestamps - if start > end { - tmp := end - end = start - start = tmp - } - - startTime := time.Unix(start, 0) - endTime := time.Unix(end, 0) - newPoints := make([]Datapoint, 0) - - for startTime.Before(endTime) || startTime.Equal(endTime) { - point := Datapoint{ - Count: 0, - Label: startTime.Format("2006-01-02"), - } - - for j, p := range points { - if p.Label == point.Label || p.Label == startTime.Format("2006-01") { - point.Count = p.Count - points[j] = points[len(points)-1] - break - } - } - - newPoints = append(newPoints, point) - startTime = startTime.Add(step) - } - - return newPoints -} - func getRequestedLimit(r *http.Request) int { limit, err := strconv.Atoi(r.URL.Query().Get("limit")) if err != nil || limit == 0 { diff --git a/api/auth.go b/api/auth.go index 879650e..b2792f1 100644 --- a/api/auth.go +++ b/api/auth.go @@ -15,7 +15,7 @@ type Login struct { Password string `json:"password"` } -var store = sessions.NewFilesystemStore( "./storage/sessions/", []byte(os.Getenv("ANA_SECRET_KEY"))) +var store = sessions.NewFilesystemStore("./storage/sessions/", []byte(os.Getenv("ANA_SECRET_KEY"))) // URL: POST /api/session var LoginHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/api/browsers.go b/api/browsers.go index d8fabf2..7e0326c 100644 --- a/api/browsers.go +++ b/api/browsers.go @@ -2,7 +2,6 @@ package api import ( "net/http" - "github.com/dannyvankooten/ana/db" "github.com/dannyvankooten/ana/count" "encoding/json" ) @@ -15,7 +14,7 @@ var GetBrowsersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Re total := count.Visitors(before, after) // get rows - stmt, err := db.Conn.Prepare(` + results := count.Custom(` SELECT v.browser_name, COUNT(DISTINCT(pv.visitor_id)) AS count @@ -24,22 +23,7 @@ var GetBrowsersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Re WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ? AND v.browser_name IS NOT NULL GROUP BY v.browser_name ORDER BY count DESC - LIMIT ?`) - checkError(err) - defer stmt.Close() - rows, err := stmt.Query(before, after, defaultLimit) - checkError(err) - defer rows.Close() - - results := make([]Datapoint, 0) - for rows.Next() { - var d Datapoint - err = rows.Scan(&d.Label, &d.Count); - checkError(err) - - d.Percentage = float64(d.Count) / total * 100 - results = append(results, d) - } + LIMIT ?`, before, after, getRequestedLimit(r), total) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(results) diff --git a/api/countries.go b/api/countries.go index ed5a657..2ab00a7 100644 --- a/api/countries.go +++ b/api/countries.go @@ -2,7 +2,6 @@ package api import ( "net/http" - "github.com/dannyvankooten/ana/db" "github.com/dannyvankooten/ana/count" "encoding/json" ) @@ -15,7 +14,7 @@ var GetCountriesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R total := count.Visitors(before, after) // get rows - stmt, err := db.Conn.Prepare(` + results := count.Custom(` SELECT v.country, COUNT(DISTINCT(pv.visitor_id)) AS count @@ -24,22 +23,7 @@ var GetCountriesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ? AND v.country IS NOT NULL GROUP BY v.country ORDER BY count DESC - LIMIT ?`) - checkError(err) - defer stmt.Close() - rows, err := stmt.Query(before, after, getRequestedLimit(r)) - checkError(err) - defer rows.Close() - - results := make([]Datapoint, 0) - for rows.Next() { - var d Datapoint - err = rows.Scan(&d.Label, &d.Count); - checkError(err) - - d.Percentage = float64(d.Count) / total * 100 - results = append(results, d) - } + LIMIT ?`, before, after, getRequestedLimit(r), total) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(results) diff --git a/api/languages.go b/api/languages.go index 1f6fd8c..c2fdbc3 100644 --- a/api/languages.go +++ b/api/languages.go @@ -2,7 +2,6 @@ package api import ( "net/http" - "github.com/dannyvankooten/ana/db" "github.com/dannyvankooten/ana/count" "encoding/json" ) @@ -14,7 +13,7 @@ var GetLanguagesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R // get total total := count.Visitors(before, after) - stmt, err := db.Conn.Prepare(` + results := count.Custom(` SELECT v.browser_language, COUNT(v.id) AS count @@ -23,24 +22,8 @@ var GetLanguagesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ? GROUP BY v.browser_language ORDER BY count DESC - LIMIT ?`) - checkError(err) - defer stmt.Close() - - rows, err := stmt.Query(before, after, defaultLimit) - checkError(err) - defer rows.Close() - - results := make([]Datapoint, 0) - - for rows.Next() { - var d Datapoint - err = rows.Scan(&d.Label, &d.Count); - checkError(err) - - d.Percentage = float64(d.Count) / total * 100 - results = append(results, d) - } + LIMIT ?`, before, after, getRequestedLimit(r), total, + ) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(results) diff --git a/api/pageviews.go b/api/pageviews.go index b4cd5d6..4dc3d1f 100644 --- a/api/pageviews.go +++ b/api/pageviews.go @@ -2,12 +2,10 @@ package api import ( "net/http" + "encoding/json" "github.com/dannyvankooten/ana/models" "github.com/dannyvankooten/ana/db" "github.com/dannyvankooten/ana/count" - "encoding/json" - "github.com/gorilla/mux" - "time" ) // URL: /api/pageviews @@ -57,40 +55,8 @@ var GetPageviewsCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *h // URL: /api/pageviews/group/day var GetPageviewsPeriodCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - period := vars["period"] - formats := map[string]string { - "day": "%Y-%m-%d", - "month": "%Y-%m", - } before, after := getRequestedPeriods(r) - stmt, err := db.Conn.Prepare(`SELECT - SUM(a.count) AS count, - DATE_FORMAT(a.date, ?) AS date_group - FROM archive a - WHERE a.metric = 'pageviews' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ? - GROUP BY date_group`) - checkError(err) - defer stmt.Close() - - rows, err := stmt.Query(formats[period], before, after) - checkError(err) - defer rows.Close() - - results := make([]Datapoint, 0) - for rows.Next() { - v := Datapoint{} - err = rows.Scan(&v.Count, &v.Label); - checkError(err) - results = append(results, v) - } - - d := time.Hour * 24; - if period == "month" { - d = d * 30 - } - results = fillDatapoints(before, after, d, results) - + results := count.PageviewsPerDay(before, after) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(results) }) diff --git a/api/referrers.go b/api/referrers.go index ebde874..0fc1117 100644 --- a/api/referrers.go +++ b/api/referrers.go @@ -2,10 +2,8 @@ package api import ( "net/http" - "github.com/dannyvankooten/ana/db" "github.com/dannyvankooten/ana/count" "encoding/json" - "strings" ) // URL: /api/referrers @@ -16,7 +14,7 @@ var GetReferrersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R total := count.Visitors(before, after) // get rows - stmt, err := db.Conn.Prepare(` + results := count.Custom(` SELECT pv.referrer_url, COUNT(DISTINCT(pv.visitor_id)) AS count @@ -26,25 +24,7 @@ var GetReferrersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R AND pv.referrer_url != "" GROUP BY pv.referrer_url ORDER BY count DESC - LIMIT ?`) - checkError(err) - defer stmt.Close() - rows, err := stmt.Query(before, after, getRequestedLimit(r)) - checkError(err) - defer rows.Close() - - results := make([]Datapoint, 0) - for rows.Next() { - var d Datapoint - err = rows.Scan(&d.Label, &d.Count); - d.Label = strings.Replace(d.Label, "http://", "", 1) - d.Label = strings.Replace(d.Label, "https://", "", 1) - d.Label = strings.TrimRight(d.Label, "/") - checkError(err) - - d.Percentage = float64(d.Count) / total * 100 - results = append(results, d) - } + LIMIT ?`, before, after, getRequestedLimit(r), total) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(results) diff --git a/api/screen-resolutions.go b/api/screen-resolutions.go index afef3c1..b427897 100644 --- a/api/screen-resolutions.go +++ b/api/screen-resolutions.go @@ -2,7 +2,6 @@ package api import ( "net/http" - "github.com/dannyvankooten/ana/db" "github.com/dannyvankooten/ana/count" "encoding/json" ) @@ -15,7 +14,7 @@ var GetScreenResolutionsHandler = http.HandlerFunc(func(w http.ResponseWriter, r total := count.Visitors(before, after) // get rows - stmt, err := db.Conn.Prepare(` + results := count.Custom(` SELECT v.screen_resolution, COUNT(DISTINCT(pv.visitor_id)) AS count @@ -24,22 +23,7 @@ var GetScreenResolutionsHandler = http.HandlerFunc(func(w http.ResponseWriter, r WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ? GROUP BY v.screen_resolution ORDER BY count DESC - LIMIT ?`) - checkError(err) - defer stmt.Close() - rows, err := stmt.Query(before, after, getRequestedLimit(r)) - checkError(err) - defer rows.Close() - - results := make([]Datapoint, 0) - for rows.Next() { - var d Datapoint - err = rows.Scan(&d.Label, &d.Count); - checkError(err) - - d.Percentage = float64(d.Count) / total * 100 - results = append(results, d) - } + LIMIT ?`, before, after, getRequestedLimit(r), total) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(results) diff --git a/api/visitors.go b/api/visitors.go index 95f5e04..7081ee1 100644 --- a/api/visitors.go +++ b/api/visitors.go @@ -5,8 +5,6 @@ import ( "github.com/dannyvankooten/ana/db" "github.com/dannyvankooten/ana/count" "encoding/json" - "github.com/gorilla/mux" - "time" ) // URL: /api/visitors/count @@ -30,40 +28,8 @@ var GetVisitorsRealtimeCountHandler = http.HandlerFunc(func(w http.ResponseWrite // URL: /api/visitors/count/group/:period var GetVisitorsPeriodCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - period := vars["period"] - formats := map[string]string { - "day": "%Y-%m-%d", - "month": "%Y-%m", - } - - stmt, err := db.Conn.Prepare(`SELECT - SUM(a.count) AS count, - DATE_FORMAT(a.date, ?) AS date_group - FROM archive a - WHERE a.metric = 'visitors' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ? - GROUP BY date_group`) - checkError(err) - defer stmt.Close() - before, after := getRequestedPeriods(r) - rows, err := stmt.Query(formats[period], before, after) - checkError(err) - - results := make([]Datapoint, 0) - defer rows.Close() - for rows.Next() { - v := Datapoint{} - err = rows.Scan(&v.Count, &v.Label); - checkError(err) - results = append(results, v) - } - - d := time.Hour * 24; - if period == "month" { - d = d * 30 - } - results = fillDatapoints(before, after, d, results) + results := count.VisitorsPerDay(before, after) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(results) }) diff --git a/assets/js/components/Graph.js b/assets/js/components/Graph.js index c13edfb..8a4bc43 100644 --- a/assets/js/components/Graph.js +++ b/assets/js/components/Graph.js @@ -22,12 +22,12 @@ function Chart(element, showPrimary, showSecondary) { var pageviewTip = d3.tip() .attr('class', 'd3-tip') - .html((d) => '' + numbers.formatWithComma(d.Count) + '' + ' pageviews') + .html((d) => '' + numbers.formatWithComma(d.Value) + '' + ' pageviews') .offset([-12, 0]); var visitorTip = d3.tip() .attr('class', 'd3-tip') - .html((d) => '' + numbers.formatWithComma(d.Count) + '' + ' visitors' ) + .html((d) => '' + numbers.formatWithComma(d.Value) + '' + ' visitors' ) .offset([-12, 0]); var graph = d3.select('#graph'); @@ -52,7 +52,7 @@ function Chart(element, showPrimary, showSecondary) { } function draw() { - var max = d3.max(showPrimary ? primaryData : secondaryData, (d) => d.Count); + var max = d3.max(showPrimary ? primaryData : secondaryData, (d) => d.Value); var ticks = primaryData.length; var xTick = Math.round(ticks / 7); @@ -86,8 +86,8 @@ function Chart(element, showPrimary, showSecondary) { bars.append('rect') .attr('width', x.bandwidth()) - .attr('height', (d) => (h - y(d.Count)) ) - .attr('y', (d) => y(d.Count)) + .attr('height', (d) => (h - y(d.Value)) ) + .attr('y', (d) => y(d.Value)) .on('mouseover', pageviewTip.show) .on('mouseout', pageviewTip.hide); } @@ -102,8 +102,8 @@ function Chart(element, showPrimary, showSecondary) { visitorBars.append('rect') .attr('width', x.bandwidth() * 0.66 ) - .attr('height', (d) => (h - y(d.Count)) ) - .attr('y', (d) => y(d.Count)) + .attr('height', (d) => (h - y(d.Value)) ) + .attr('y', (d) => y(d.Value)) .on('mouseover', visitorTip.show) .on('mouseout', visitorTip.hide); } diff --git a/assets/js/components/Table.js b/assets/js/components/Table.js index a484f08..2b1d0ff 100644 --- a/assets/js/components/Table.js +++ b/assets/js/components/Table.js @@ -52,8 +52,9 @@ class Table extends Component { const after = before - ( period * dayInSeconds ); Client.request(`${this.props.endpoint}?before=${before}&after=${after}&limit=${limit}`) - .then((d) => { this.setState({ loading: false, records: d })}) - .catch((e) => { console.log(e) }) + .then((d) => { + this.setState({ loading: false, records: d } + )}).catch((e) => { console.log(e) }) } render() { @@ -61,8 +62,8 @@ class Table extends Component { {i+1} {this.labelCell(p)} - {numbers.formatWithComma(p.Count)} - {Math.round(p.Percentage)}% + {numbers.formatWithComma(p.Value)} + {Math.round(p.PercentageValue)}% )); diff --git a/assets/js/pages/dashboard.js b/assets/js/pages/dashboard.js index 3e9416b..c61d637 100644 --- a/assets/js/pages/dashboard.js +++ b/assets/js/pages/dashboard.js @@ -64,7 +64,7 @@ class Dashboard extends Component {
-
( )} /> +
{p.Label.substring(0, 15)}
( )} />
{p.Label.substring(0, 15).replace('https://', '').replace('http://', '')}
diff --git a/count/archive.go b/count/count.go similarity index 59% rename from count/archive.go rename to count/count.go index 8eb6077..5b77362 100644 --- a/count/archive.go +++ b/count/count.go @@ -4,6 +4,7 @@ import( "database/sql" "log" "github.com/dannyvankooten/ana/db" + "time" ) type Archive struct { @@ -14,6 +15,12 @@ type Archive struct { Date string } +type Point struct { + Label string + Value int + PercentageValue float64 +} + func (a *Archive) Save(Conn *sql.DB) error { stmt, err := db.Conn.Prepare(`INSERT INTO archive( metric, @@ -113,3 +120,65 @@ func checkError(err error) { log.Fatal(err) } } + +func Custom(sql string, before int64, after int64, limit int, total float64) []Point { + stmt, err := db.Conn.Prepare(sql) + checkError(err) + defer stmt.Close() + + rows, err := stmt.Query(before, after, limit) + checkError(err) + defer rows.Close() + + results := newPointSlice(rows, total) + return results +} + +func newPointSlice(rows *sql.Rows, total float64) []Point { + results := make([]Point, 0) + for rows.Next() { + var d Point + err := rows.Scan(&d.Label, &d.Value); + checkError(err) + + d.PercentageValue = float64(d.Value) / total * 100 + results = append(results, d) + } + + return results +} + + +func fill(start int64, end int64, points []Point) []Point { + // be smart about received timestamps + if start > end { + tmp := end + end = start + start = tmp + } + + startTime := time.Unix(start, 0) + endTime := time.Unix(end, 0) + newPoints := make([]Point, 0) + step := time.Hour * 24 + + for startTime.Before(endTime) || startTime.Equal(endTime) { + point := Point{ + Value: 0, + Label: startTime.Format("2006-01-02"), + } + + for j, p := range points { + if p.Label == point.Label || p.Label == startTime.Format("2006-01") { + point.Value = p.Value + points[j] = points[len(points)-1] + break + } + } + + newPoints = append(newPoints, point) + startTime = startTime.Add(step) + } + + return newPoints +} diff --git a/count/pageviews.go b/count/pageviews.go index e29a4fb..09f7369 100644 --- a/count/pageviews.go +++ b/count/pageviews.go @@ -17,3 +17,30 @@ func Pageviews(before int64, after int64) float64 { stmt.QueryRow(before, after).Scan(&total) return total } + +func PageviewsPerDay(before int64, after int64) []Point { + stmt, err := db.Conn.Prepare(`SELECT + SUM(a.count) AS count, + DATE_FORMAT(a.date, '%Y-%m-%d') AS date_group + FROM archive a + WHERE a.metric = 'pageviews' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ? + GROUP BY date_group`) + checkError(err) + defer stmt.Close() + + rows, err := stmt.Query(before, after) + checkError(err) + defer rows.Close() + + results := make([]Point, 0) + defer rows.Close() + for rows.Next() { + p := Point{} + err = rows.Scan(&p.Value, &p.Label); + checkError(err) + results = append(results, p) + } + + results = fill(after, before, results) + return results +} diff --git a/count/visitors.go b/count/visitors.go index 2e6ecd2..b7b7c18 100644 --- a/count/visitors.go +++ b/count/visitors.go @@ -17,3 +17,30 @@ func Visitors(before int64, after int64) float64 { stmt.QueryRow(before, after).Scan(&total) return total } + +func VisitorsPerDay(before int64, after int64) ([]Point) { + stmt, err := db.Conn.Prepare(`SELECT + SUM(a.count) AS count, + DATE_FORMAT(a.date, '%Y-%m-%d') AS date_group + FROM archive a + WHERE a.metric = 'visitors' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ? + GROUP BY date_group`) + checkError(err) + defer stmt.Close() + + rows, err := stmt.Query(before, after) + checkError(err) + + results := make([]Point, 0) + defer rows.Close() + for rows.Next() { + p := Point{} + err = rows.Scan(&p.Value, &p.Label); + checkError(err) + results = append(results, p) + } + + results = fill(after, before, results) + + return results +}