status-im/fathom
2
0
Fork 0
mirror of https://github.com/status-im/fathom.git synced 2025-02-28 19:10:36 +00:00
Code Issues Projects Releases Wiki Activity

Abstract calculating totals per category into count.Custom

Browse Source
This commit is contained in:
Danny van Kooten 2016-12-11 12:52:10 +01:00
parent 00f8d2cbba
commit 4468b70601
16 changed files with 157 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ROADMAP.md
View File

@ -5,14 +5,13 @@ This is a general draft document for thoughts and todo's, without any structure
### What's cooking?
- Archive `pageviews` table into daily totals for performance
- Create archive on-the-fly if it does not exist yet
- Bulk process tracking requests (Redis or in-memory?)
- Allow for multiple sites in same instance
- Update page title when it changes.
- Never query `pageviews` table directly.
- Create archive on-the-fly if it does not exist yet?
- Bulk process tracking requests (Redis or in-memory)
- Allow for multiple sites in same Ana instance
- Custom date range picker
- Choose a better name than "Ana"
- Settle on a better name than "Ana"
- Envelope API responses
- Visual error handling on client-side.
- Client-side error handling.
- Geolocate IP addresses periodically.
- Mask last part of IP address.

39
api/api.go
View File

@ -7,12 +7,6 @@ import (
"net/http"
)
type Datapoint struct {
Count int
Label string
Percentage float64 `json:",omitempty"`
}
const defaultPeriod = 7
const defaultLimit = 10
@ -23,39 +17,6 @@ func checkError(err error) {
}
}
func fillDatapoints(start int64, end int64, step time.Duration, points []Datapoint) []Datapoint {
// be smart about received timestamps
if start > end {
tmp := end
end = start
start = tmp
}
startTime := time.Unix(start, 0)
endTime := time.Unix(end, 0)
newPoints := make([]Datapoint, 0)
for startTime.Before(endTime) || startTime.Equal(endTime) {
point := Datapoint{
Count: 0,
Label: startTime.Format("2006-01-02"),
}
for j, p := range points {
if p.Label == point.Label || p.Label == startTime.Format("2006-01") {
point.Count = p.Count
points[j] = points[len(points)-1]
break
}
}
newPoints = append(newPoints, point)
startTime = startTime.Add(step)
}
return newPoints
}
func getRequestedLimit(r *http.Request) int {
limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
if err != nil || limit == 0 {

2
api/auth.go
View File

@ -15,7 +15,7 @@ type Login struct {
Password string `json:"password"`
}
var store = sessions.NewFilesystemStore( "./storage/sessions/", []byte(os.Getenv("ANA_SECRET_KEY")))
var store = sessions.NewFilesystemStore("./storage/sessions/", []byte(os.Getenv("ANA_SECRET_KEY")))
// URL: POST /api/session
var LoginHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

20
api/browsers.go
View File

@ -2,7 +2,6 @@ package api
import (
"net/http"
"github.com/dannyvankooten/ana/db"
"github.com/dannyvankooten/ana/count"
"encoding/json"
)
@ -15,7 +14,7 @@ var GetBrowsersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Re
total := count.Visitors(before, after)
// get rows
stmt, err := db.Conn.Prepare(`
results := count.Custom(`
SELECT
v.browser_name,
COUNT(DISTINCT(pv.visitor_id)) AS count
@ -24,22 +23,7 @@ var GetBrowsersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Re
WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ? AND v.browser_name IS NOT NULL
GROUP BY v.browser_name
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(before, after, defaultLimit)
checkError(err)
defer rows.Close()
results := make([]Datapoint, 0)
for rows.Next() {
var d Datapoint
err = rows.Scan(&d.Label, &d.Count);
checkError(err)
d.Percentage = float64(d.Count) / total * 100
results = append(results, d)
}
LIMIT ?`, before, after, getRequestedLimit(r), total)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)

20
api/countries.go
View File

@ -2,7 +2,6 @@ package api
import (
"net/http"
"github.com/dannyvankooten/ana/db"
"github.com/dannyvankooten/ana/count"
"encoding/json"
)
@ -15,7 +14,7 @@ var GetCountriesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
total := count.Visitors(before, after)
// get rows
stmt, err := db.Conn.Prepare(`
results := count.Custom(`
SELECT
v.country,
COUNT(DISTINCT(pv.visitor_id)) AS count
@ -24,22 +23,7 @@ var GetCountriesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ? AND v.country IS NOT NULL
GROUP BY v.country
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(before, after, getRequestedLimit(r))
checkError(err)
defer rows.Close()
results := make([]Datapoint, 0)
for rows.Next() {
var d Datapoint
err = rows.Scan(&d.Label, &d.Count);
checkError(err)
d.Percentage = float64(d.Count) / total * 100
results = append(results, d)
}
LIMIT ?`, before, after, getRequestedLimit(r), total)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)

23
api/languages.go
View File

@ -2,7 +2,6 @@ package api
import (
"net/http"
"github.com/dannyvankooten/ana/db"
"github.com/dannyvankooten/ana/count"
"encoding/json"
)
@ -14,7 +13,7 @@ var GetLanguagesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
// get total
total := count.Visitors(before, after)
stmt, err := db.Conn.Prepare(`
results := count.Custom(`
SELECT
v.browser_language,
COUNT(v.id) AS count
@ -23,24 +22,8 @@ var GetLanguagesHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ?
GROUP BY v.browser_language
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(before, after, defaultLimit)
checkError(err)
defer rows.Close()
results := make([]Datapoint, 0)
for rows.Next() {
var d Datapoint
err = rows.Scan(&d.Label, &d.Count);
checkError(err)
d.Percentage = float64(d.Count) / total * 100
results = append(results, d)
}
LIMIT ?`, before, after, getRequestedLimit(r), total,
)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)

38
api/pageviews.go
View File

@ -2,12 +2,10 @@ package api
import (
"net/http"
"encoding/json"
"github.com/dannyvankooten/ana/models"
"github.com/dannyvankooten/ana/db"
"github.com/dannyvankooten/ana/count"
"encoding/json"
"github.com/gorilla/mux"
"time"
)
// URL: /api/pageviews
@ -57,40 +55,8 @@ var GetPageviewsCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *h
// URL: /api/pageviews/group/day
var GetPageviewsPeriodCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
period := vars["period"]
formats := map[string]string {
"day": "%Y-%m-%d",
"month": "%Y-%m",
}
before, after := getRequestedPeriods(r)
stmt, err := db.Conn.Prepare(`SELECT
SUM(a.count) AS count,
DATE_FORMAT(a.date, ?) AS date_group
FROM archive a
WHERE a.metric = 'pageviews' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ?
GROUP BY date_group`)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(formats[period], before, after)
checkError(err)
defer rows.Close()
results := make([]Datapoint, 0)
for rows.Next() {
v := Datapoint{}
err = rows.Scan(&v.Count, &v.Label);
checkError(err)
results = append(results, v)
}
d := time.Hour * 24;
if period == "month" {
d = d * 30
}
results = fillDatapoints(before, after, d, results)
results := count.PageviewsPerDay(before, after)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)
})

24
api/referrers.go
View File

@ -2,10 +2,8 @@ package api
import (
"net/http"
"github.com/dannyvankooten/ana/db"
"github.com/dannyvankooten/ana/count"
"encoding/json"
"strings"
)
// URL: /api/referrers
@ -16,7 +14,7 @@ var GetReferrersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
total := count.Visitors(before, after)
// get rows
stmt, err := db.Conn.Prepare(`
results := count.Custom(`
SELECT
pv.referrer_url,
COUNT(DISTINCT(pv.visitor_id)) AS count
@ -26,25 +24,7 @@ var GetReferrersHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.R
AND pv.referrer_url != ""
GROUP BY pv.referrer_url
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(before, after, getRequestedLimit(r))
checkError(err)
defer rows.Close()
results := make([]Datapoint, 0)
for rows.Next() {
var d Datapoint
err = rows.Scan(&d.Label, &d.Count);
d.Label = strings.Replace(d.Label, "http://", "", 1)
d.Label = strings.Replace(d.Label, "https://", "", 1)
d.Label = strings.TrimRight(d.Label, "/")
checkError(err)
d.Percentage = float64(d.Count) / total * 100
results = append(results, d)
}
LIMIT ?`, before, after, getRequestedLimit(r), total)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)

20
api/screen-resolutions.go
View File

@ -2,7 +2,6 @@ package api
import (
"net/http"
"github.com/dannyvankooten/ana/db"
"github.com/dannyvankooten/ana/count"
"encoding/json"
)
@ -15,7 +14,7 @@ var GetScreenResolutionsHandler = http.HandlerFunc(func(w http.ResponseWriter, r
total := count.Visitors(before, after)
// get rows
stmt, err := db.Conn.Prepare(`
results := count.Custom(`
SELECT
v.screen_resolution,
COUNT(DISTINCT(pv.visitor_id)) AS count
@ -24,22 +23,7 @@ var GetScreenResolutionsHandler = http.HandlerFunc(func(w http.ResponseWriter, r
WHERE UNIX_TIMESTAMP(pv.timestamp) <= ? AND UNIX_TIMESTAMP(pv.timestamp) >= ?
GROUP BY v.screen_resolution
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(before, after, getRequestedLimit(r))
checkError(err)
defer rows.Close()
results := make([]Datapoint, 0)
for rows.Next() {
var d Datapoint
err = rows.Scan(&d.Label, &d.Count);
checkError(err)
d.Percentage = float64(d.Count) / total * 100
results = append(results, d)
}
LIMIT ?`, before, after, getRequestedLimit(r), total)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)

36
api/visitors.go
View File

@ -5,8 +5,6 @@ import (
"github.com/dannyvankooten/ana/db"
"github.com/dannyvankooten/ana/count"
"encoding/json"
"github.com/gorilla/mux"
"time"
)
// URL: /api/visitors/count
@ -30,40 +28,8 @@ var GetVisitorsRealtimeCountHandler = http.HandlerFunc(func(w http.ResponseWrite
// URL: /api/visitors/count/group/:period
var GetVisitorsPeriodCountHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
period := vars["period"]
formats := map[string]string {
"day": "%Y-%m-%d",
"month": "%Y-%m",
}
stmt, err := db.Conn.Prepare(`SELECT
SUM(a.count) AS count,
DATE_FORMAT(a.date, ?) AS date_group
FROM archive a
WHERE a.metric = 'visitors' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ?
GROUP BY date_group`)
checkError(err)
defer stmt.Close()
before, after := getRequestedPeriods(r)
rows, err := stmt.Query(formats[period], before, after)
checkError(err)
results := make([]Datapoint, 0)
defer rows.Close()
for rows.Next() {
v := Datapoint{}
err = rows.Scan(&v.Count, &v.Label);
checkError(err)
results = append(results, v)
}
d := time.Hour * 24;
if period == "month" {
d = d * 30
}
results = fillDatapoints(before, after, d, results)
results := count.VisitorsPerDay(before, after)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)
})

14
assets/js/components/Graph.js
View File

@ -22,12 +22,12 @@ function Chart(element, showPrimary, showSecondary) {
var pageviewTip = d3.tip()
.attr('class', 'd3-tip')
.html((d) => '<span>' + numbers.formatWithComma(d.Count) + '</span>' + ' pageviews')
.html((d) => '<span>' + numbers.formatWithComma(d.Value) + '</span>' + ' pageviews')
.offset([-12, 0]);
var visitorTip = d3.tip()
.attr('class', 'd3-tip')
.html((d) => '<span>' + numbers.formatWithComma(d.Count) + '</span>' + ' visitors' )
.html((d) => '<span>' + numbers.formatWithComma(d.Value) + '</span>' + ' visitors' )
.offset([-12, 0]);
var graph = d3.select('#graph');
@ -52,7 +52,7 @@ function Chart(element, showPrimary, showSecondary) {
}
function draw() {
var max = d3.max(showPrimary ? primaryData : secondaryData, (d) => d.Count);
var max = d3.max(showPrimary ? primaryData : secondaryData, (d) => d.Value);
var ticks = primaryData.length;
var xTick = Math.round(ticks / 7);
@ -86,8 +86,8 @@ function Chart(element, showPrimary, showSecondary) {
bars.append('rect')
.attr('width', x.bandwidth())
.attr('height', (d) => (h - y(d.Count)) )
.attr('y', (d) => y(d.Count))
.attr('height', (d) => (h - y(d.Value)) )
.attr('y', (d) => y(d.Value))
.on('mouseover', pageviewTip.show)
.on('mouseout', pageviewTip.hide);
}
@ -102,8 +102,8 @@ function Chart(element, showPrimary, showSecondary) {
visitorBars.append('rect')
.attr('width', x.bandwidth() * 0.66 )
.attr('height', (d) => (h - y(d.Count)) )
.attr('y', (d) => y(d.Count))
.attr('height', (d) => (h - y(d.Value)) )
.attr('y', (d) => y(d.Value))
.on('mouseover', visitorTip.show)
.on('mouseout', visitorTip.hide);
}

9
assets/js/components/Table.js
View File

@ -52,8 +52,9 @@ class Table extends Component {
const after = before - ( period * dayInSeconds );
Client.request(`${this.props.endpoint}?before=${before}&after=${after}&limit=${limit}`)
.then((d) => { this.setState({ loading: false, records: d })})
.catch((e) => { console.log(e) })
.then((d) => {
this.setState({ loading: false, records: d }
)}).catch((e) => { console.log(e) })
}
render() {
@ -61,8 +62,8 @@ class Table extends Component {
<tr>
<td class="muted">{i+1}</td>
{this.labelCell(p)}
<td>{numbers.formatWithComma(p.Count)}</td>
<td>{Math.round(p.Percentage)}%</td>
<td>{numbers.formatWithComma(p.Value)}</td>
<td>{Math.round(p.PercentageValue)}%</td>
</tr>
));

2
assets/js/pages/dashboard.js
View File

@ -64,7 +64,7 @@ class Dashboard extends Component {
<Table period={this.state.period} endpoint="screen-resolutions" title="Screen Resolutions" headers={["#", "Resolution", "Count", "%"]} />
</div>
<div class="col-2">
<Table period={this.state.period} endpoint="referrers" title="Referrers" headers={["#", "URL", "Count", "%"]} labelCell={(p) => ( <td><a href={"//" + p.Label}>{p.Label.substring(0, 15)}</a></td>)} />
<Table period={this.state.period} endpoint="referrers" title="Referrers" headers={["#", "URL", "Count", "%"]} labelCell={(p) => ( <td><a href={"//" + p.Label}>{p.Label.substring(0, 15).replace('https://', '').replace('http://', '')}</a></td>)} />
</div>
<div class="col-2">
<Table period={this.state.period} endpoint="browsers" title="Browsers" headers={["#", "Browser", "Count", "%"]} onAuthError={this.props.onLogout} />

69
count/archive.go → count/count.go
View File

@ -4,6 +4,7 @@ import(
"database/sql"
"log"
"github.com/dannyvankooten/ana/db"
"time"
)
type Archive struct {
@ -14,6 +15,12 @@ type Archive struct {
Date string
}
type Point struct {
Label string
Value int
PercentageValue float64
}
func (a *Archive) Save(Conn *sql.DB) error {
stmt, err := db.Conn.Prepare(`INSERT INTO archive(
metric,
@ -113,3 +120,65 @@ func checkError(err error) {
log.Fatal(err)
}
}
func Custom(sql string, before int64, after int64, limit int, total float64) []Point {
stmt, err := db.Conn.Prepare(sql)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(before, after, limit)
checkError(err)
defer rows.Close()
results := newPointSlice(rows, total)
return results
}
func newPointSlice(rows *sql.Rows, total float64) []Point {
results := make([]Point, 0)
for rows.Next() {
var d Point
err := rows.Scan(&d.Label, &d.Value);
checkError(err)
d.PercentageValue = float64(d.Value) / total * 100
results = append(results, d)
}
return results
}
func fill(start int64, end int64, points []Point) []Point {
// be smart about received timestamps
if start > end {
tmp := end
end = start
start = tmp
}
startTime := time.Unix(start, 0)
endTime := time.Unix(end, 0)
newPoints := make([]Point, 0)
step := time.Hour * 24
for startTime.Before(endTime) || startTime.Equal(endTime) {
point := Point{
Value: 0,
Label: startTime.Format("2006-01-02"),
}
for j, p := range points {
if p.Label == point.Label || p.Label == startTime.Format("2006-01") {
point.Value = p.Value
points[j] = points[len(points)-1]
break
}
}
newPoints = append(newPoints, point)
startTime = startTime.Add(step)
}
return newPoints
}

27
count/pageviews.go
View File

@ -17,3 +17,30 @@ func Pageviews(before int64, after int64) float64 {
stmt.QueryRow(before, after).Scan(&total)
return total
}
func PageviewsPerDay(before int64, after int64) []Point {
stmt, err := db.Conn.Prepare(`SELECT
SUM(a.count) AS count,
DATE_FORMAT(a.date, '%Y-%m-%d') AS date_group
FROM archive a
WHERE a.metric = 'pageviews' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ?
GROUP BY date_group`)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(before, after)
checkError(err)
defer rows.Close()
results := make([]Point, 0)
defer rows.Close()
for rows.Next() {
p := Point{}
err = rows.Scan(&p.Value, &p.Label);
checkError(err)
results = append(results, p)
}
results = fill(after, before, results)
return results
}

27
count/visitors.go
View File

@ -17,3 +17,30 @@ func Visitors(before int64, after int64) float64 {
stmt.QueryRow(before, after).Scan(&total)
return total
}
func VisitorsPerDay(before int64, after int64) ([]Point) {
stmt, err := db.Conn.Prepare(`SELECT
SUM(a.count) AS count,
DATE_FORMAT(a.date, '%Y-%m-%d') AS date_group
FROM archive a
WHERE a.metric = 'visitors' AND UNIX_TIMESTAMP(a.date) <= ? AND UNIX_TIMESTAMP(a.date) >= ?
GROUP BY date_group`)
checkError(err)
defer stmt.Close()
rows, err := stmt.Query(before, after)
checkError(err)
results := make([]Point, 0)
defer rows.Close()
for rows.Next() {
p := Point{}
err = rows.Scan(&p.Value, &p.Label);
checkError(err)
results = append(results, p)
}
results = fill(after, before, results)
return results
}