status-im/fathom
2
0
Fork 0
mirror of https://github.com/status-im/fathom.git synced 2025-03-01 03:20:27 +00:00
Code Issues Projects Releases Wiki Activity

move remaining queries to datastore pkg & bubble up errors to http handlers

Browse Source
This commit is contained in:
Danny van Kooten 2018-04-27 17:28:03 +02:00
parent b0d16b648b
commit a20e2b1227
16 changed files with 159 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/api/browsers.go
View File

@ -8,6 +8,9 @@ import (
// URL: /api/browsers
var GetBrowsersHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results := count.Browsers(before, after, getRequestedLimit(r))
results, err := count.Browsers(before, after, getRequestedLimit(r))
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

6
pkg/api/languages.go
View File

@ -9,6 +9,10 @@ import (
// URL: /api/languages
var GetLanguagesHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results := count.Languages(before, after, getRequestedLimit(r))
results, err := count.Languages(before, after, getRequestedLimit(r))
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

4
pkg/api/params.go
View File

@ -10,8 +10,8 @@ import (
const defaultPeriod = 7
const defaultLimit = 10
func getRequestedLimit(r *http.Request) int {
limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
func getRequestedLimit(r *http.Request) int64 {
limit, err := strconv.ParseInt(r.URL.Query().Get("limit"), 10, 64)
if err != nil || limit == 0 {
limit = 10
}

5
pkg/api/referrers.go
View File

@ -9,6 +9,9 @@ import (
// URL: /api/referrers
var GetReferrersHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results := count.Referrers(before, after, getRequestedLimit(r))
results, err := count.Referrers(before, after, getRequestedLimit(r))
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

5
pkg/api/screens.go
View File

@ -9,6 +9,9 @@ import (
// URL: /api/screen-resolutions
var GetScreenResolutionsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
before, after := getRequestedPeriods(r)
results := count.Screens(before, after, getRequestedLimit(r))
results, err := count.Screens(before, after, getRequestedLimit(r))
if err != nil {
return err
}
return respond(w, envelope{Data: results})
})

29
pkg/count/browsers.go
View File

@ -5,32 +5,23 @@ import (
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Browsers returns a point slice containing browser data per browser name
func Browsers(before int64, after int64, limit int) []Point {
stmt, err := datastore.DB.Prepare(`
SELECT
t.value,
SUM(t.count_unique) AS count
FROM total_browser_names t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
func Browsers(before int64, after int64, limit int64) ([]*models.Point, error) {
points, err := datastore.TotalsPerBrowser(before, after, limit)
if err != nil {
return nil, err
}
rows, err := stmt.Query(before, after, limit)
checkError(err)
points := newPointSlice(rows)
total, err := datastore.TotalUniqueBrowsers(before, after)
checkError(err)
if err != nil {
return nil, err
}
points = calculatePointPercentages(points, total)
return points
return points, nil
}
// CreateBrowserTotals aggregates screen data into daily totals

35
pkg/count/count.go
View File

@ -1,21 +1,14 @@
package count
import (
"database/sql"
"time"
log "github.com/sirupsen/logrus"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Point represents a data point, will always have a Label and Value
type Point struct {
Label string
Value int
PercentageValue float64
}
func getLastArchivedDate() string {
value, _ := datastore.GetOption("last_archived")
if value == "" {
@ -42,30 +35,10 @@ func Archive() {
log.Infof("finished aggregating metrics. ran for %dms.", (end.UnixNano()-start.UnixNano())/1000000)
}
func checkError(err error) {
if err != nil {
log.Fatal(err)
}
}
func newPointSlice(rows *sql.Rows) []Point {
results := make([]Point, 0)
// append point slices
for rows.Next() {
var d Point
err := rows.Scan(&d.Label, &d.Value)
checkError(err)
results = append(results, d)
}
return results
}
func calculatePointPercentages(points []Point, total int) []Point {
func calculatePointPercentages(points []*models.Point, total int) []*models.Point {
// calculate percentage values for each point
for i, d := range points {
points[i].PercentageValue = float64(d.Value) / float64(total) * 100
for _, p := range points {
p.PercentageValue = float64(p.Value) / float64(total) * 100.00
}
return points

27
pkg/count/count_test.go
View File

@ -3,32 +3,13 @@ package count
import (
"testing"
"time"
"github.com/usefathom/fathom/pkg/models"
)
func TestFill(t *testing.T) {
start := time.Date(2016, time.January, 1, 0, 0, 0, 0, time.Local)
end := time.Date(2016, time.January, 5, 0, 0, 0, 0, time.Local)
points := []Point{
Point{
Label: start.Format("2006-01-02"),
Value: 1,
},
Point{
Label: end.Format("2006-01-02"),
Value: 1,
},
}
filled := fill(start.Unix(), end.Unix(), points)
if len(filled) != 5 {
t.Error("Length of filled points slice does not match expected length")
}
}
func TestCalculatePointPercentages(t *testing.T) {
points := []Point{
Point{
points := []*models.Point{
&models.Point{
Label: "Foo",
Value: 5,
},

29
pkg/count/languages.go
View File

@ -5,32 +5,23 @@ import (
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Languages returns a point slice containing language data per language
func Languages(before int64, after int64, limit int) []Point {
stmt, err := datastore.DB.Prepare(`
SELECT
t.value,
SUM(t.count_unique) AS count
FROM total_browser_languages t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
func Languages(before int64, after int64, limit int64) ([]*models.Point, error) {
points, err := datastore.TotalsPerLanguage(before, after, limit)
if err != nil {
return nil, err
}
rows, err := stmt.Query(before, after, limit)
checkError(err)
points := newPointSlice(rows)
total, err := datastore.TotalUniqueLanguages(before, after)
checkError(err)
if err != nil {
return nil, err
}
points = calculatePointPercentages(points, total)
return points
return points, nil
}
// CreateLanguageTotals aggregates screen data into daily totals

29
pkg/count/referrers.go
View File

@ -5,32 +5,23 @@ import (
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Referrers returns a point slice containing browser data per browser name
func Referrers(before int64, after int64, limit int) []Point {
stmt, err := datastore.DB.Prepare(`
SELECT
t.value,
SUM(t.count) AS count
FROM total_referrers t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
func Referrers(before int64, after int64, limit int64) ([]*models.Point, error) {
points, err := datastore.TotalsPerReferrer(before, after, limit)
if err != nil {
return nil, err
}
rows, err := stmt.Query(before, after, limit)
checkError(err)
points := newPointSlice(rows)
total, err := datastore.TotalReferrers(before, after)
checkError(err)
if err != nil {
return nil, err
}
points = calculatePointPercentages(points, total)
return points
return points, nil
}
// CreateReferrerTotals aggregates screen data into daily totals

29
pkg/count/screens.go
View File

@ -5,32 +5,23 @@ import (
"time"
"github.com/usefathom/fathom/pkg/datastore"
"github.com/usefathom/fathom/pkg/models"
)
// Screens returns a point slice containing screen data per size
func Screens(before int64, after int64, limit int) []Point {
stmt, err := datastore.DB.Prepare(`
SELECT
t.value,
SUM(t.count_unique) AS count
FROM total_screens t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY count DESC
LIMIT ?`)
checkError(err)
defer stmt.Close()
func Screens(before int64, after int64, limit int64) ([]*models.Point, error) {
points, err := datastore.TotalsPerScreen(before, after, limit)
if err != nil {
return nil, err
}
rows, err := stmt.Query(before, after, limit)
checkError(err)
points := newPointSlice(rows)
total, err := datastore.TotalUniqueScreens(before, after)
checkError(err)
if err != nil {
return nil, err
}
points = calculatePointPercentages(points, total)
return points
return points, nil
}
// CreateScreenTotals aggregates screen data into daily totals

19
pkg/datastore/total_browsers.go
View File

@ -1,5 +1,7 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalUniqueBrowsers returns the total # of unique browsers between two given timestamps
func TotalUniqueBrowsers(before int64, after int64) (int, error) {
var total int
@ -12,3 +14,20 @@ func TotalUniqueBrowsers(before int64, after int64) (int, error) {
err := dbx.Get(&total, query, before, after)
return total, err
}
func TotalsPerBrowser(before int64, after int64, limit int64) ([]*models.Point, error) {
var results []*models.Point
query := dbx.Rebind(`
SELECT
t.value AS label,
SUM(t.count_unique) AS value
FROM total_browser_names t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY label
ORDER BY value DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
return results, err
}

19
pkg/datastore/total_languages.go
View File

@ -1,5 +1,7 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalUniqueLanguages returns the total # of unique browser languages between two given timestamps
func TotalUniqueLanguages(before int64, after int64) (int, error) {
var total int
@ -13,3 +15,20 @@ func TotalUniqueLanguages(before int64, after int64) (int, error) {
err := dbx.Get(&total, query, before, after)
return total, err
}
func TotalsPerLanguage(before int64, after int64, limit int64) ([]*models.Point, error) {
var results []*models.Point
query := dbx.Rebind(`
SELECT
t.value AS label,
SUM(t.count_unique) AS value
FROM total_browser_languages t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY label
ORDER BY value DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
return results, err
}

13
pkg/datastore/total_pageviews.go
View File

@ -8,7 +8,9 @@ import (
func TotalPageviews(before int64, after int64) (int, error) {
var total int
query := dbx.Rebind(`SELECT IFNULL( SUM(t.count), 0 ) FROM total_pageviews t WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
query := dbx.Rebind(`
SELECT IFNULL( SUM(t.count), 0 )
FROM total_pageviews t WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?`)
err := dbx.Get(&total, query, before, after)
if err != nil {
return 0, err
@ -73,14 +75,7 @@ func SavePageviewTotals(totals []*models.Total) error {
}
for _, t := range totals {
_, err = stmt.Exec(
t.PageID,
t.Count,
t.CountUnique,
t.Date,
t.Count,
t.CountUnique,
)
_, err = stmt.Exec(t.PageID, t.Count, t.CountUnique, t.Date, t.Count, t.CountUnique)
}
err = tx.Commit()

19
pkg/datastore/total_referrers.go
View File

@ -1,5 +1,7 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalReferrers returns the total # of referrers between two given timestamps
func TotalReferrers(before int64, after int64) (int, error) {
var total int
@ -13,3 +15,20 @@ func TotalReferrers(before int64, after int64) (int, error) {
err := dbx.Get(&total, query, before, after)
return total, err
}
func TotalsPerReferrer(before int64, after int64, limit int64) ([]*models.Point, error) {
var results []*models.Point
query := dbx.Rebind(`
SELECT
t.value AS label,
SUM(t.count) AS value
FROM total_referrers t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY label
ORDER BY value DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
return results, err
}

19
pkg/datastore/total_screens.go
View File

@ -1,5 +1,7 @@
package datastore
import "github.com/usefathom/fathom/pkg/models"
// TotalUniqueScreens returns the total # of screens between two given timestamps
func TotalUniqueScreens(before int64, after int64) (int, error) {
var total int
@ -12,3 +14,20 @@ func TotalUniqueScreens(before int64, after int64) (int, error) {
err := dbx.Get(&total, query, before, after)
return total, err
}
func TotalsPerScreen(before int64, after int64, limit int64) ([]*models.Point, error) {
var results []*models.Point
query := dbx.Rebind(`
SELECT
t.value AS label,
SUM(t.count_unique) AS value
FROM total_screens t
WHERE UNIX_TIMESTAMP(t.date) <= ? AND UNIX_TIMESTAMP(t.date) >= ?
GROUP BY t.value
ORDER BY value DESC
LIMIT ?`)
err := dbx.Select(&results, query, before, after, limit)
return results, err
}