status-im/fathom
2
0
Fork 0
mirror of https://github.com/status-im/fathom.git synced 2025-03-01 03:20:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'hourly-storage'

Browse Source
This commit is contained in:
Danny van Kooten 2018-11-14 10:04:16 +01:00
commit 5610a0a4f2
29 changed files with 347 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
assets/src/css/styles.css
View File

@ -123,9 +123,6 @@ div.delete a { color: red; }
.box-pages { grid-column: 2; grid-row: 2 ; }
.box-referrers { grid-column: 3; grid-row: 2; }
/* since we hide chart for views with less than a day worth of data, move tables to row 1 */
.ltday .box-pages, .ltday .box-referrers{ grid-row: 1; }
.half { display: grid; grid-template-columns: 1fr 1fr; grid-gap: 12px; align-items: center; }
.half div { text-align: right; }
.half div.submit { text-align: left; }
@ -137,8 +134,6 @@ div.delete a { color: red; }
}
.login-page.flex-rapper { width: 100%; height: 100%; display: flex; align-items: center; justify-content: center; }
.login-rapper { text-align: left; width: 320px; }
.login-page label { position: relative; }

166
assets/src/js/components/Chart.js vendored
View File

@ -8,34 +8,25 @@ import * as d3 from 'd3';
import 'd3-transition';
d3.tip = require('d3-tip');
const formatDay = d3.timeFormat("%e"),
formatMonth = d3.timeFormat("%b"),
formatMonthDay = d3.timeFormat("%b %e"),
formatYear = d3.timeFormat("%Y");
const
formatHour = d3.timeFormat("%H"),
formatDay = d3.timeFormat("%e"),
formatMonth = d3.timeFormat("%b"),
formatMonthDay = d3.timeFormat("%b %e"),
formatYear = d3.timeFormat("%Y");
const t = d3.transition().duration(600).ease(d3.easeQuadOut);
// tooltip
const tip = d3.tip().attr('class', 'd3-tip').html((d) => (`
<div class="tip-heading">${d.Date.toLocaleDateString()}</div>
<div class="tip-content">
<div class="tip-pageviews">
<div class="tip-number">${d.Pageviews}</div>
<div class="tip-metric">Pageviews</div>
</div>
<div class="tip-visitors">
<div class="tip-number">${d.Visitors}</div>
<div class="tip-metric">Visitors</div>
</div>
</div>
`));
function padZero(s) {
return s < 10 ? "0" + s : s;
}
function timeFormatPicker(n) {
return function(d, i) {
if( n === 24 ) {
return formatHour(d);
}
if(d.getDate() === 1) {
return d.getMonth() === 0 ? formatYear(d) : formatMonth(d)
}
@ -50,46 +41,6 @@ function timeFormatPicker(n) {
}
}
function prepareData(startUnix, endUnix, data) {
// add timezone offset back in to get local start date
const timezoneOffset = (new Date()).getTimezoneOffset() * 60;
let startDate = new Date((startUnix + timezoneOffset) * 1000);
let endDate = new Date((endUnix + timezoneOffset) * 1000);
let datamap = [];
let newData = [];
// create keyed array for quick date access
let length = data.length;
let d, dateParts, date, key;
for(var i=0;i<length;i++) {
d = data[i];
// replace date with actual date object & store in datamap
dateParts = d.Date.split('T')[0].split('-');
date = new Date(dateParts[0], dateParts[1]-1, dateParts[2], 0, 0, 0)
key = date.getFullYear() + "-" + padZero(date.getMonth() + 1) + "-" + padZero(date.getDate());
d.Date = date;
datamap[key] = d;
}
// make sure we have values for each date
let currentDate = startDate;
while(currentDate < endDate) {
key = currentDate.getFullYear() + "-" + padZero(currentDate.getMonth() + 1) + "-" + padZero(currentDate.getDate());
data = datamap[key] ? datamap[key] : {
"Pageviews": 0,
"Visitors": 0,
"Date": new Date(currentDate),
};
newData.push(data);
currentDate.setDate(currentDate.getDate() + 1);
}
return newData;
}
class Chart extends Component {
constructor(props) {
super(props)
@ -97,6 +48,8 @@ class Chart extends Component {
this.state = {
loading: false,
data: [],
diffInDays: 1,
hoursPerTick: 24,
}
}
@ -105,6 +58,13 @@ class Chart extends Component {
return;
}
let daysDiff = Math.round((newProps.before-newProps.after)/24/60/60);
let stepHours = daysDiff > 1 ? 24 : 1;
this.setState({
diffInDays: daysDiff,
hoursPerTick: stepHours,
})
this.fetchData(newProps)
}
@ -112,6 +72,60 @@ class Chart extends Component {
return o.siteId != n.siteId || o.before != n.before || o.after != n.after;
}
@bind
prepareData(data) {
let startDate = new Date(this.props.after * 1000);
let endDate = new Date(this.props.before * 1000);
let newData = [];
// instantiate JS Date objects
data = data.map((d) => {
d.Date = new Date(d.Date);
return d
})
// make sure we have values for each date (so 0 value for gaps)
let currentDate = startDate, nextDate, tick, offset = 0;
while(currentDate < endDate) {
tick = {
"Pageviews": 0,
"Visitors": 0,
"Date": new Date(currentDate),
};
nextDate = new Date(currentDate)
nextDate.setHours(nextDate.getHours() + this.state.hoursPerTick);
// grab data that falls between currentDate & nextDate
for(let i=data.length-offset-1; i>=0; i--) {
// Because 9AM should be included in 9AM-10AM range, check for equality here
if( data[i].Date >= nextDate) {
break;
}
// increment offset so subsequent dates can skip first X items in array
offset += 1;
// continue to next item in array if we're still below our target date
if( data[i].Date < currentDate) {
continue;
}
// add to tick data
tick.Pageviews += data[i].Pageviews;
tick.Visitors += data[i].Visitors;
}
newData.push(tick);
currentDate = nextDate;
}
return newData;
}
@bind
prepareChart() {
let padding = { top: 12, right: 12, bottom: 24, left: 40 };
@ -130,19 +144,31 @@ class Chart extends Component {
this.x = d3.scaleBand().range([0, this.innerWidth]).padding(0.1)
this.y = d3.scaleLinear().range([this.innerHeight, 0])
this.ctx.call(tip)
// tooltip
this.tip = d3.tip().attr('class', 'd3-tip').html((d) => {
let title = d.Date.toLocaleDateString();
if(this.state.diffInDays <= 1) {
title += ` ${d.Date.getHours()}:00 - ${d.Date.getHours() + 1}:00`
}
return (`<div class="tip-heading">${title}</div>
<div class="tip-content">
<div class="tip-pageviews">
<div class="tip-number">${d.Pageviews}</div>
<div class="tip-metric">Pageviews</div>
</div>
<div class="tip-visitors">
<div class="tip-number">${d.Visitors}</div>
<div class="tip-metric">Visitors</div>
</div>
</div>`)});
this.ctx.call(this.tip)
}
@bind
redrawChart() {
let data = this.state.data;
// hide chart & bail if we're trying to show less than 1 day worth of data
this.base.parentNode.style.display = data.length <= 1 ? 'none' : '';
if(data.length <= 1) {
return;
}
if( ! this.ctx ) {
this.prepareChart()
}
@ -156,8 +182,8 @@ class Chart extends Component {
let yAxis = d3.axisLeft().scale(y).ticks(3).tickSize(-innerWidth)
let xAxis = d3.axisBottom().scale(x).tickFormat(timeFormatPicker(data.length))
// hide all "day" ticks if we're watching more than 100 days of data
if(data.length > 100) {
// hide all "day" ticks if we're watching more than 31 days of data
if(data.length > 31) {
xAxis.tickValues(data.filter(d => d.Date.getDate() === 1).map(d => d.Date))
}
@ -214,7 +240,7 @@ class Chart extends Component {
.attr('y', (d) => y(d.Visitors))
// add event listeners for tooltips
days.on('mouseover', tip.show).on('mouseout', tip.hide)
days.on('mouseover', this.tip.show).on('mouseout', this.tip.hide)
}
@bind
@ -228,7 +254,7 @@ class Chart extends Component {
return;
}
let chartData = prepareData(props.after, props.before, d);
let chartData = this.prepareData(d);
this.setState({
loading: false,
data: chartData,

14
assets/src/js/components/DatePicker.js
View File

@ -10,16 +10,6 @@ const padZero = function(n){return n<10? '0'+n:''+n;}
function getNow() {
let now = new Date()
let tzOffset = now.getTimezoneOffset() * 60 * 1000;
// if we're ahead of UTC, stick to UTC's "now"
// this is ugly but a sad necessity for now because we store and aggregate statistics using UTC dates (without time data)
// For those ahead of UTC, "today" will always be empty if they're checking early on in their day
// see https://github.com/usefathom/fathom/issues/134
if (tzOffset < 0) {
now.setTime(now.getTime() + tzOffset )
}
return now
}
@ -116,8 +106,8 @@ class DatePicker extends Component {
// create unix timestamps from local date objects
let before, after;
before = Math.round((+endDate) / 1000) - endDate.getTimezoneOffset() * 60;
after = Math.round((+startDate) / 1000) - startDate.getTimezoneOffset() * 60;
before = Math.round((+endDate) / 1000);
after = Math.round((+startDate) / 1000);
this.setState({
period: period || '',

6
pkg/aggregator/store.go
View File

@ -10,7 +10,7 @@ import (
)
func (agg *Aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*models.SiteStats, error) {
cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02"))
cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02T15"))
if stats, ok := r.Sites[cacheKey]; ok {
return stats, nil
@ -35,7 +35,7 @@ func (agg *Aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*mod
}
func (agg *Aggregator) getPageStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.PageStats, error) {
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02T15"), hostname, pathname)
if stats, ok := r.Pages[cacheKey]; ok {
return stats, nil
}
@ -71,7 +71,7 @@ func (agg *Aggregator) getPageStats(r *results, siteID int64, t time.Time, hostn
}
func (agg *Aggregator) getReferrerStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02T15"), hostname, pathname)
if stats, ok := r.Referrers[cacheKey]; ok {
return stats, nil
}

4
pkg/api/routes.go
View File

@ -23,10 +23,6 @@ func (api *API) Routes() *mux.Router {
r.Handle("/api/sites/{id:[0-9]+}/stats/site", api.Authorize(HandlerFunc(api.GetSiteStatsHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/groupby/day", api.Authorize(HandlerFunc(api.GetSiteStatsPerDayHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/pageviews", api.Authorize(HandlerFunc(api.GetSiteStatsPageviewsHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/visitors", api.Authorize(HandlerFunc(api.GetSiteStatsVisitorsHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/duration", api.Authorize(HandlerFunc(api.GetSiteStatsDurationHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/bounces", api.Authorize(HandlerFunc(api.GetSiteStatsBouncesHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/site/realtime", api.Authorize(HandlerFunc(api.GetSiteStatsRealtimeHandler))).Methods(http.MethodGet)
r.Handle("/api/sites/{id:[0-9]+}/stats/pages", api.Authorize(HandlerFunc(api.GetPageStatsHandler))).Methods(http.MethodGet)

40
pkg/api/site_stats.go
View File

@ -14,46 +14,6 @@ func (api *API) GetSiteStatsHandler(w http.ResponseWriter, r *http.Request) erro
return respond(w, http.StatusOK, envelope{Data: result})
}
// URL: /api/stats/site/pageviews
func (api *API) GetSiteStatsPageviewsHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
result, err := api.database.GetTotalSiteViews(params.SiteID, params.StartDate, params.EndDate)
if err != nil {
return err
}
return respond(w, http.StatusOK, envelope{Data: result})
}
// URL: /api/stats/site/visitors
func (api *API) GetSiteStatsVisitorsHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
result, err := api.database.GetTotalSiteVisitors(params.SiteID, params.StartDate, params.EndDate)
if err != nil {
return err
}
return respond(w, http.StatusOK, envelope{Data: result})
}
// URL: /api/stats/site/duration
func (api *API) GetSiteStatsDurationHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
result, err := api.database.GetAverageSiteDuration(params.SiteID, params.StartDate, params.EndDate)
if err != nil {
return err
}
return respond(w, http.StatusOK, envelope{Data: result})
}
// URL: /api/stats/site/bounces
func (api *API) GetSiteStatsBouncesHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)
result, err := api.database.GetAverageSiteBounceRate(params.SiteID, params.StartDate, params.EndDate)
if err != nil {
return err
}
return respond(w, http.StatusOK, envelope{Data: result})
}
// URL: /api/stats/site/realtime
func (api *API) GetSiteStatsRealtimeHandler(w http.ResponseWriter, r *http.Request) error {
params := GetRequestParams(r)

3
pkg/cli/cli.go
View File

@ -2,6 +2,7 @@ package cli
import (
"os"
"time"
log "github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -20,7 +21,7 @@ var app *App
func Run(v string) error {
// force all times in UTC, regardless of server timezone
os.Setenv("TZ", "")
time.Local = time.UTC
// setup CLI app
app = &App{cli.NewApp(), nil, nil}

5
pkg/datastore/datastore.go
View File

@ -30,11 +30,6 @@ type Datastore interface {
GetSiteStatsPerDay(int64, time.Time, time.Time) ([]*models.SiteStats, error)
SaveSiteStats(*models.SiteStats) error
GetAggregatedSiteStats(int64, time.Time, time.Time) (*models.SiteStats, error)
GetTotalSiteViews(int64, time.Time, time.Time) (int64, error)
GetTotalSiteVisitors(int64, time.Time, time.Time) (int64, error)
GetTotalSiteSessions(int64, time.Time, time.Time) (int64, error)
GetAverageSiteDuration(int64, time.Time, time.Time) (float64, error)
GetAverageSiteBounceRate(int64, time.Time, time.Time) (float64, error)
GetRealtimeVisitorCount(int64) (int64, error)
// pageviews

3
pkg/datastore/sqlstore/config.go
View File

@ -55,14 +55,13 @@ func (c *Config) DSN() string {
mc.DBName = c.Name
mc.Params = map[string]string{
"parseTime": "true",
"loc": "Local",
}
if c.SSLMode != "" {
mc.Params["tls"] = c.SSLMode
}
dsn = mc.FormatDSN()
case SQLITE:
dsn = c.Name + "?_loc=auto&_busy_timeout=5000"
dsn = c.Name + "?_busy_timeout=5000"
}
return dsn

19
pkg/datastore/sqlstore/migrations/mysql/21_alter_page_stats_table.sql Normal file
View File

@ -0,0 +1,19 @@
-- +migrate Up
CREATE TABLE page_stats(
site_id INTEGER NOT NULL DEFAULT 1,
hostname_id INTEGER NOT NULL,
pathname_id INTEGER NOT NULL,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
entries INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts DATETIME NOT NULL
) CHARACTER SET=utf8;
INSERT INTO page_stats
SELECT site_id, hostname_id, pathname_id, pageviews, visitors, entries, bounce_rate, known_durations, avg_duration, CONCAT(date, ' 00:00:00')
FROM daily_page_stats s ;
DROP TABLE daily_page_stats;
-- +migrate Down

17
pkg/datastore/sqlstore/migrations/mysql/22_alter_site_stats_table.sql Normal file
View File

@ -0,0 +1,17 @@
-- +migrate Up
CREATE TABLE site_stats(
site_id INTEGER NOT NULL DEFAULT 1,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
sessions INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts DATETIME NOT NULL
) CHARACTER SET=utf8;
INSERT INTO site_stats
SELECT site_id, pageviews, visitors, sessions, bounce_rate, known_durations, avg_duration, CONCAT(date, ' 00:00:00')
FROM daily_site_stats s ;
DROP TABLE daily_site_stats;
-- +migrate Down

19
pkg/datastore/sqlstore/migrations/mysql/23_alter_referrer_stats_table.sql Normal file
View File

@ -0,0 +1,19 @@
-- +migrate Up
CREATE TABLE referrer_stats(
site_id INTEGER NOT NULL DEFAULT 1,
hostname_id INTEGER NOT NULL,
pathname_id INTEGER NOT NULL,
groupname VARCHAR(255) NULL,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts DATETIME NOT NULL
) CHARACTER SET=utf8;
INSERT INTO referrer_stats
SELECT site_id, hostname_id, pathname_id, groupname, pageviews, visitors, bounce_rate, known_durations, avg_duration, CONCAT(date, ' 00:00:00')
FROM daily_referrer_stats s;
DROP TABLE daily_referrer_stats;
-- +migrate Down

9
pkg/datastore/sqlstore/migrations/mysql/24_recreate_stat_table_indices.sql Normal file
View File

@ -0,0 +1,9 @@
-- +migrate Up
CREATE UNIQUE INDEX unique_page_stats ON page_stats(site_id, hostname_id, pathname_id, ts);
CREATE UNIQUE INDEX unique_referrer_stats ON referrer_stats(site_id, hostname_id, pathname_id, ts);
CREATE UNIQUE INDEX unique_site_stats ON site_stats(site_id, ts);
-- +migrate Down
DROP INDEX unique_page_stats ON page_stats;
DROP INDEX unique_referrer_stats ON referrer_stats;
DROP INDEX unique_site_stats ON site_stats;

20
pkg/datastore/sqlstore/migrations/postgres/22_alter_page_stats_table.sql Normal file
View File

@ -0,0 +1,20 @@
-- +migrate Up
CREATE TABLE page_stats(
site_id INTEGER NOT NULL DEFAULT 1,
hostname_id INTEGER NOT NULL,
pathname_id INTEGER NOT NULL,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
entries INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts TIMESTAMP WITHOUT TIME ZONE NOT NULL
);
INSERT INTO page_stats
SELECT site_id, hostname_id, pathname_id, pageviews, visitors, entries, bounce_rate, known_durations, avg_duration, (date || ' 00:00:00')::timestamp
FROM daily_page_stats s;
DROP TABLE daily_page_stats;
-- +migrate Down

19
pkg/datastore/sqlstore/migrations/postgres/23_alter_referrer_stats_table.sql Normal file
View File

@ -0,0 +1,19 @@
-- +migrate Up
CREATE TABLE referrer_stats(
site_id INTEGER NOT NULL DEFAULT 1,
hostname_id INTEGER NOT NULL,
pathname_id INTEGER NOT NULL,
groupname VARCHAR(255) NULL,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts TIMESTAMP WITHOUT TIME ZONE NOT NULL
);
INSERT INTO referrer_stats
SELECT site_id, hostname_id, pathname_id, groupname, pageviews, visitors, bounce_rate, known_durations, avg_duration, (date || ' 00:00:00')::timestamp
FROM daily_referrer_stats s;
DROP TABLE daily_referrer_stats;
-- +migrate Down

17
pkg/datastore/sqlstore/migrations/postgres/24_alter_site_stats_table.sql Normal file
View File

@ -0,0 +1,17 @@
-- +migrate Up
CREATE TABLE site_stats(
site_id INTEGER NOT NULL DEFAULT 1,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
sessions INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts TIMESTAMP WITHOUT TIME ZONE NOT NULL
);
INSERT INTO site_stats
SELECT site_id, pageviews, visitors, sessions, bounce_rate, known_durations, avg_duration, (date || ' 00:00:00')::timestamp
FROM daily_site_stats s;
DROP TABLE daily_site_stats;
-- +migrate Down

11
pkg/datastore/sqlstore/migrations/postgres/25_recreate_stat_table_indices.sql Normal file
View File

@ -0,0 +1,11 @@
-- +migrate Up
DROP INDEX IF EXISTS unique_daily_page_stats;
DROP INDEX IF EXISTS unique_daily_referrer_stats;
CREATE UNIQUE INDEX unique_page_stats ON page_stats(site_id, hostname_id, pathname_id, ts);
CREATE UNIQUE INDEX unique_referrer_stats ON referrer_stats(site_id, hostname_id, pathname_id, ts);
CREATE UNIQUE INDEX unique_site_stats ON site_stats(site_id, ts);
-- +migrate Down
DROP INDEX IF EXISTS unique_page_stats;
DROP INDEX IF EXISTS unique_referrer_stats;
DROP INDEX IF EXISTS unique_site_stats;

7
pkg/datastore/sqlstore/migrations/postgres/26_alter_pageviews_table.sql Normal file
View File

@ -0,0 +1,7 @@
-- +migrate Up
ALTER TABLE pageviews ALTER COLUMN timestamp TYPE TIMESTAMP WITHOUT TIME ZONE;
-- +migrate Down
ALTER TABLE pageviews ALTER COLUMN timestamp TYPE TIMESTAMP WITH TIME ZONE;

19
pkg/datastore/sqlstore/migrations/sqlite3/21_alter_page_stats_table.sql Normal file
View File

@ -0,0 +1,19 @@
-- +migrate Up
CREATE TABLE page_stats(
site_id INTEGER NOT NULL DEFAULT 1,
hostname_id INTEGER NOT NULL,
pathname_id INTEGER NOT NULL,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
entries INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts DATETIME NOT NULL
);
INSERT INTO page_stats
SELECT site_id, hostname_id, pathname_id, pageviews, visitors, entries, bounce_rate, known_durations, avg_duration, date || ' 00:00:00'
FROM daily_page_stats s ;
DROP TABLE daily_page_stats;
-- +migrate Down

17
pkg/datastore/sqlstore/migrations/sqlite3/22_alter_site_stats_table.sql Normal file
View File

@ -0,0 +1,17 @@
-- +migrate Up
CREATE TABLE site_stats(
site_id INTEGER NOT NULL DEFAULT 1,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
sessions INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts DATETIME NOT NULL
);
INSERT INTO site_stats
SELECT site_id, pageviews, visitors, sessions, bounce_rate, known_durations, avg_duration, date || ' 00:00:00'
FROM daily_site_stats s ;
DROP TABLE daily_site_stats;
-- +migrate Down

19
pkg/datastore/sqlstore/migrations/sqlite3/23_alter_referrer_stats_table.sql Normal file
View File

@ -0,0 +1,19 @@
-- +migrate Up
CREATE TABLE referrer_stats(
site_id INTEGER NOT NULL DEFAULT 1,
hostname_id INTEGER NOT NULL,
pathname_id INTEGER NOT NULL,
groupname VARCHAR(255) NULL,
pageviews INTEGER NOT NULL,
visitors INTEGER NOT NULL,
bounce_rate FLOAT NOT NULL,
known_durations INTEGER NOT NULL DEFAULT 0,
avg_duration FLOAT NOT NULL,
ts DATETIME NOT NULL
);
INSERT INTO referrer_stats
SELECT site_id, hostname_id, pathname_id, groupname, pageviews, visitors, bounce_rate, known_durations, avg_duration, date || ' 00:00:00'
FROM daily_referrer_stats s;
DROP TABLE daily_referrer_stats;
-- +migrate Down

11
pkg/datastore/sqlstore/migrations/sqlite3/24_recreate_stat_table_indices.sql Normal file
View File

@ -0,0 +1,11 @@
-- +migrate Up
DROP INDEX IF EXISTS unique_daily_page_stats;
DROP INDEX IF EXISTS unique_daily_referrer_stats;
CREATE UNIQUE INDEX unique_page_stats ON page_stats(site_id, hostname_id, pathname_id, ts);
CREATE UNIQUE INDEX unique_referrer_stats ON referrer_stats(site_id, hostname_id, pathname_id, ts);
CREATE UNIQUE INDEX unique_site_stats ON site_stats(site_id, ts);
-- +migrate Down
DROP INDEX IF EXISTS unique_page_stats;
DROP INDEX IF EXISTS unique_referrer_stats;
DROP INDEX IF EXISTS unique_site_stats;

22
pkg/datastore/sqlstore/page_stats.go
View File

@ -9,8 +9,8 @@ import (
func (db *sqlstore) GetPageStats(siteID int64, date time.Time, hostnameID int64, pathnameID int64) (*models.PageStats, error) {
stats := &models.PageStats{New: false}
query := db.Rebind(`SELECT * FROM daily_page_stats WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND date = ? LIMIT 1`)
err := db.Get(stats, query, siteID, hostnameID, pathnameID, date.Format("2006-01-02"))
query := db.Rebind(`SELECT * FROM page_stats WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND ts = ? LIMIT 1`)
err := db.Get(stats, query, siteID, hostnameID, pathnameID, date.Format(DATE_FORMAT))
if err == sql.ErrNoRows {
return nil, ErrNoResults
}
@ -27,14 +27,14 @@ func (db *sqlstore) SavePageStats(s *models.PageStats) error {
}
func (db *sqlstore) insertPageStats(s *models.PageStats) error {
query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, site_id, hostname_id, pathname_id, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format("2006-01-02"))
query := db.Rebind(`INSERT INTO page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, site_id, hostname_id, pathname_id, ts) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format(DATE_FORMAT))
return err
}
func (db *sqlstore) updatePageStats(s *models.PageStats) error {
query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ?, avg_duration = ?, known_durations = ? WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND date = ?`)
_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format("2006-01-02"))
query := db.Rebind(`UPDATE page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ?, avg_duration = ?, known_durations = ? WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND ts = ?`)
_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format(DATE_FORMAT))
return err
}
@ -48,19 +48,19 @@ func (db *sqlstore) GetAggregatedPageStats(siteID int64, startDate time.Time, en
SUM(entries) AS entries,
COALESCE(SUM(entries*bounce_rate) / NULLIF(SUM(entries), 0), 0.00) AS bounce_rate,
COALESCE(SUM(pageviews*avg_duration) / SUM(pageviews), 0.00) AS avg_duration
FROM daily_page_stats s
FROM page_stats s
LEFT JOIN hostnames h ON h.id = s.hostname_id
LEFT JOIN pathnames p ON p.id = s.pathname_id
WHERE site_id = ? AND date >= ? AND date <= ?
WHERE site_id = ? AND ts >= ? AND ts <= ?
GROUP BY hostname, pathname
ORDER BY pageviews DESC LIMIT ?`)
err := db.Select(&result, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit)
err := db.Select(&result, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT), limit)
return result, err
}
func (db *sqlstore) GetAggregatedPageStatsPageviews(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
var result int64
query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM daily_page_stats WHERE site_id = ? AND date >= ? AND date <= ?`)
err := db.Get(&result, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM page_stats WHERE site_id = ? AND ts >= ? AND ts <= ?`)
err := db.Get(&result, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT))
return result, err
}

22
pkg/datastore/sqlstore/referrer_stats.go
View File

@ -9,8 +9,8 @@ import (
func (db *sqlstore) GetReferrerStats(siteID int64, date time.Time, hostnameID int64, pathnameID int64) (*models.ReferrerStats, error) {
stats := &models.ReferrerStats{New: false}
query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE site_id = ? AND date = ? AND hostname_id = ? AND pathname_id = ? LIMIT 1`)
err := db.Get(stats, query, siteID, date.Format("2006-01-02"), hostnameID, pathnameID)
query := db.Rebind(`SELECT * FROM referrer_stats WHERE site_id = ? AND ts = ? AND hostname_id = ? AND pathname_id = ? LIMIT 1`)
err := db.Get(stats, query, siteID, date.Format(DATE_FORMAT), hostnameID, pathnameID)
if err == sql.ErrNoRows {
return nil, ErrNoResults
}
@ -27,14 +27,14 @@ func (db *sqlstore) SaveReferrerStats(s *models.ReferrerStats) error {
}
func (db *sqlstore) insertReferrerStats(s *models.ReferrerStats) error {
query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, site_id, hostname_id, pathname_id, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format("2006-01-02"))
query := db.Rebind(`INSERT INTO referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, site_id, hostname_id, pathname_id, ts) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format(DATE_FORMAT))
return err
}
func (db *sqlstore) updateReferrerStats(s *models.ReferrerStats) error {
query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ?, avg_duration = ?, known_durations = ?, groupname = ? WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND date = ?`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format("2006-01-02"))
query := db.Rebind(`UPDATE referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ?, avg_duration = ?, known_durations = ?, groupname = ? WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND ts = ?`)
_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format(DATE_FORMAT))
return err
}
@ -49,10 +49,10 @@ func (db *sqlstore) GetAggregatedReferrerStats(siteID int64, startDate time.Time
SUM(pageviews) AS pageviews,
COALESCE(SUM(pageviews*NULLIF(bounce_rate, 0)) / SUM(pageviews), 0.00) AS bounce_rate,
COALESCE(SUM(pageviews*avg_duration) / SUM(pageviews), 0.00) AS avg_duration
FROM daily_referrer_stats s
FROM referrer_stats s
LEFT JOIN hostnames h ON h.id = s.hostname_id
LEFT JOIN pathnames p ON p.id = s.pathname_id
WHERE site_id = ? AND date >= ? AND date <= ? `
WHERE site_id = ? AND ts >= ? AND ts <= ? `
if db.Config.Driver == "sqlite3" {
sql = sql + `GROUP BY COALESCE(NULLIF(groupname, ''), hostname || pathname ) `
@ -63,13 +63,13 @@ func (db *sqlstore) GetAggregatedReferrerStats(siteID int64, startDate time.Time
query := db.Rebind(sql)
err := db.Select(&result, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit)
err := db.Select(&result, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT), limit)
return result, mapError(err)
}
func (db *sqlstore) GetAggregatedReferrerStatsPageviews(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
var result int64
query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM daily_referrer_stats WHERE site_id = ? AND date >= ? AND date <= ?`)
err := db.Get(&result, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM referrer_stats WHERE site_id = ? AND ts >= ? AND ts <= ?`)
err := db.Get(&result, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT))
return result, mapError(err)
}

64
pkg/datastore/sqlstore/site_stats.go
View File

@ -10,9 +10,9 @@ import (
func (db *sqlstore) GetSiteStats(siteID int64, date time.Time) (*models.SiteStats, error) {
stats := &models.SiteStats{New: false}
query := db.Rebind(`SELECT * FROM daily_site_stats WHERE site_id = ? AND date = ? LIMIT 1`)
query := db.Rebind(`SELECT * FROM site_stats WHERE site_id = ? AND ts = ? LIMIT 1`)
err := db.Get(stats, query, siteID, date.Format("2006-01-02"))
err := db.Get(stats, query, siteID, date.Format(DATE_FORMAT))
if err == sql.ErrNoRows {
return nil, ErrNoResults
}
@ -29,22 +29,21 @@ func (db *sqlstore) SaveSiteStats(s *models.SiteStats) error {
}
func (db *sqlstore) insertSiteStats(s *models.SiteStats) error {
query := db.Rebind(`INSERT INTO daily_site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.SiteID, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
query := db.Rebind(`INSERT INTO site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, ts) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, s.SiteID, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format(DATE_FORMAT))
return err
}
func (db *sqlstore) updateSiteStats(s *models.SiteStats) error {
query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ?, avg_duration = ?, known_durations = ? WHERE site_id = ? AND date = ?`)
_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Date.Format("2006-01-02"))
query := db.Rebind(`UPDATE site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ?, avg_duration = ?, known_durations = ? WHERE site_id = ? AND ts = ?`)
_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Date.Format(DATE_FORMAT))
return err
}
func (db *sqlstore) GetSiteStatsPerDay(siteID int64, startDate time.Time, endDate time.Time) ([]*models.SiteStats, error) {
results := []*models.SiteStats{}
sql := `SELECT * FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
query := db.Rebind(sql)
err := db.Select(&results, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
query := db.Rebind(`SELECT * FROM site_stats WHERE site_id = ? AND ts >= ? AND ts <= ? ORDER BY ts DESC`)
err := db.Select(&results, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT))
return results, err
}
@ -56,51 +55,12 @@ func (db *sqlstore) GetAggregatedSiteStats(siteID int64, startDate time.Time, en
COALESCE(SUM(sessions), 0) AS sessions,
COALESCE(SUM(pageviews*avg_duration) / NULLIF(SUM(pageviews), 0), 0.00) AS avg_duration,
COALESCE(SUM(sessions*bounce_rate) / NULLIF(SUM(sessions), 0), 0.00) AS bounce_rate
FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ? LIMIT 1`)
err := db.Get(stats, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
FROM site_stats
WHERE site_id = ? AND ts >= ? AND ts <= ? LIMIT 1`)
err := db.Get(stats, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT))
return stats, mapError(err)
}
func (db *sqlstore) GetTotalSiteViews(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
sql := `SELECT COALESCE(SUM(pageviews), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
query := db.Rebind(sql)
var total int64
err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
return total, mapError(err)
}
func (db *sqlstore) GetTotalSiteVisitors(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
sql := `SELECT COALESCE(SUM(visitors), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
query := db.Rebind(sql)
var total int64
err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
return total, mapError(err)
}
func (db *sqlstore) GetTotalSiteSessions(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
sql := `SELECT COALESCE(SUM(sessions), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
query := db.Rebind(sql)
var total int64
err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
return total, mapError(err)
}
func (db *sqlstore) GetAverageSiteDuration(siteID int64, startDate time.Time, endDate time.Time) (float64, error) {
sql := `SELECT COALESCE(SUM(pageviews*avg_duration)/SUM(pageviews), 0.00) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
query := db.Rebind(sql)
var total float64
err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
return total, mapError(err)
}
func (db *sqlstore) GetAverageSiteBounceRate(siteID int64, startDate time.Time, endDate time.Time) (float64, error) {
sql := `SELECT COALESCE(SUM(sessions*bounce_rate)/SUM(sessions), 4) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
query := db.Rebind(sql)
var total float64
err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
return total, mapError(err)
}
func (db *sqlstore) GetRealtimeVisitorCount(siteID int64) (int64, error) {
var siteTrackingID string
if err := db.Get(&siteTrackingID, db.Rebind(`SELECT tracking_id FROM sites WHERE id = ? LIMIT 1`), siteID); err != nil && err != sql.ErrNoRows {
@ -113,7 +73,7 @@ func (db *sqlstore) GetRealtimeVisitorCount(siteID int64) (int64, error) {
// for backwards compatibility with tracking snippets without an explicit site tracking ID (< 1.1.0)
if siteID == 1 {
sql = `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR site_tracking_id = '' ) AND is_finished = FALSE AND timestamp > ?`
sql = `SELECT COUNT(*) FROM pageviews p WHERE ( site_tracking_id = ? OR site_tracking_id = '' ) AND is_finished = FALSE AND timestamp > ?`
} else {
sql = `SELECT COUNT(*) FROM pageviews p WHERE site_tracking_id = ? AND is_finished = FALSE AND timestamp > ?`
}

2
pkg/datastore/sqlstore/sqlstore.go
View File

@ -19,6 +19,8 @@ const (
MYSQL = "mysql"
POSTGRES = "postgres"
SQLITE = "sqlite3"
DATE_FORMAT = "2006-01-02 15:00:00"
)
type sqlstore struct {

2
pkg/models/page_stats.go
View File

@ -17,7 +17,7 @@ type PageStats struct {
BounceRate float64 `db:"bounce_rate"`
AvgDuration float64 `db:"avg_duration"`
KnownDurations int64 `db:"known_durations"`
Date time.Time `db:"date" json:",omitempty"`
Date time.Time `db:"ts" json:",omitempty"`
}
func (s *PageStats) HandlePageview(p *Pageview) {

2
pkg/models/referrer_stats.go
View File

@ -17,7 +17,7 @@ type ReferrerStats struct {
BounceRate float64 `db:"bounce_rate"`
AvgDuration float64 `db:"avg_duration"`
KnownDurations int64 `db:"known_durations"`
Date time.Time `db:"date" json:",omitempty"`
Date time.Time `db:"ts" json:",omitempty"`
}
func (s *ReferrerStats) HandlePageview(p *Pageview) {

2
pkg/models/site_stats.go
View File

@ -14,7 +14,7 @@ type SiteStats struct {
BounceRate float64 `db:"bounce_rate"`
AvgDuration float64 `db:"avg_duration"`
KnownDurations int64 `db:"known_durations" json:",omitempty"`
Date time.Time `db:"date" json:",omitempty"`
Date time.Time `db:"ts" json:",omitempty"`
}
func (s *SiteStats) FormattedDuration() string {