Merge branch 'hourly-storage'

assets/src/css/styles.css

@@ -123,9 +123,6 @@ div.delete a { color: red; }
 	.box-pages { grid-column: 2; grid-row: 2 ; }
 	.box-referrers { grid-column: 3; grid-row: 2; }
 
-	/* since we hide chart for views with less than a day worth of data, move tables to row 1 */
-	.ltday .box-pages, .ltday .box-referrers{ grid-row: 1; }
-
 	.half { display: grid; grid-template-columns: 1fr 1fr; grid-gap: 12px; align-items: center; }
 	.half div { text-align: right; }
 	.half div.submit { text-align: left; }
@@ -137,8 +134,6 @@ div.delete a { color: red; }
 }
 
 
-
-
 .login-page.flex-rapper { width: 100%; height: 100%; display: flex; align-items: center; justify-content: center; }
 .login-rapper { text-align: left; width: 320px; }
 .login-page label { position: relative; }

assets/src/js/components/Chart.js

@@ -8,34 +8,25 @@ import * as d3 from 'd3';
 import 'd3-transition';
 d3.tip = require('d3-tip');
 
-const formatDay = d3.timeFormat("%e"),
+const 
+  formatHour = d3.timeFormat("%H"),
+  formatDay = d3.timeFormat("%e"),
   formatMonth = d3.timeFormat("%b"),
   formatMonthDay = d3.timeFormat("%b %e"),
   formatYear = d3.timeFormat("%Y");
 
 const t = d3.transition().duration(600).ease(d3.easeQuadOut);
 
-// tooltip
-const tip = d3.tip().attr('class', 'd3-tip').html((d) => (`
-  <div class="tip-heading">${d.Date.toLocaleDateString()}</div>
-  <div class="tip-content">
-    <div class="tip-pageviews">
-      <div class="tip-number">${d.Pageviews}</div>
-      <div class="tip-metric">Pageviews</div>
-    </div>
-    <div class="tip-visitors">
-      <div class="tip-number">${d.Visitors}</div>
-      <div class="tip-metric">Visitors</div>
-    </div>
-  </div>
-`));
-
 function padZero(s) {
   return s < 10 ? "0" + s : s;
 }
 
 function timeFormatPicker(n) {
   return function(d, i) {
+    if( n === 24 ) {
+      return formatHour(d);
+    }
+    
     if(d.getDate() === 1) {
       return d.getMonth() === 0 ? formatYear(d) : formatMonth(d) 
     } 
@@ -50,46 +41,6 @@ function timeFormatPicker(n) {
   }
 }
 
-function prepareData(startUnix, endUnix, data) {
-  // add timezone offset back in to get local start date
-  const timezoneOffset = (new Date()).getTimezoneOffset() * 60;
-  let startDate = new Date((startUnix + timezoneOffset) * 1000);
-  let endDate = new Date((endUnix + timezoneOffset) * 1000);
-  let datamap = [];
-  let newData = [];
-
-   // create keyed array for quick date access
-  let length = data.length;
-  let d, dateParts, date, key;
-  for(var i=0;i<length;i++) {
-    d = data[i];
-    // replace date with actual date object & store in datamap
-    dateParts = d.Date.split('T')[0].split('-');
-    date = new Date(dateParts[0], dateParts[1]-1, dateParts[2], 0, 0, 0)
-    key = date.getFullYear() + "-" + padZero(date.getMonth() + 1) + "-" + padZero(date.getDate());
-    d.Date = date;
-    datamap[key] = d;
-  }
-
-  // make sure we have values for each date
-  let currentDate = startDate;
-  while(currentDate < endDate) {
-    key = currentDate.getFullYear() + "-" + padZero(currentDate.getMonth() + 1) + "-" + padZero(currentDate.getDate());
-    data = datamap[key] ? datamap[key] : {
-        "Pageviews": 0,
-        "Visitors": 0,
-        "Date": new Date(currentDate),
-     };
-
-    newData.push(data);  
-    currentDate.setDate(currentDate.getDate() + 1);
-  }
-
- return newData;
-}
-
-
-
 class Chart extends Component {
   constructor(props) {
     super(props)
@@ -97,6 +48,8 @@ class Chart extends Component {
     this.state = {
       loading: false,
       data: [],
+      diffInDays: 1,
+      hoursPerTick: 24,
     }
   }
 
@@ -105,6 +58,13 @@ class Chart extends Component {
       return;
     }
     
+    let daysDiff = Math.round((newProps.before-newProps.after)/24/60/60);
+    let stepHours = daysDiff > 1 ? 24 : 1;
+    this.setState({
+      diffInDays: daysDiff,
+      hoursPerTick: stepHours,
+    })
+
     this.fetchData(newProps)
   }
 
@@ -112,6 +72,60 @@ class Chart extends Component {
     return o.siteId != n.siteId || o.before != n.before || o.after != n.after;
   }
 
+  @bind
+  prepareData(data) {
+    let startDate = new Date(this.props.after * 1000);
+    let endDate = new Date(this.props.before * 1000);
+    let newData = [];
+
+    // instantiate JS Date objects
+    data = data.map((d) => {
+      d.Date = new Date(d.Date);
+      return d
+    })
+  
+    // make sure we have values for each date (so 0 value for gaps)
+    let currentDate = startDate, nextDate, tick, offset = 0;
+    while(currentDate < endDate) {
+      tick = {
+          "Pageviews": 0,
+          "Visitors": 0,
+          "Date": new Date(currentDate),
+      };
+
+      nextDate = new Date(currentDate)
+      nextDate.setHours(nextDate.getHours() + this.state.hoursPerTick);
+
+      // grab data that falls between currentDate & nextDate
+      for(let i=data.length-offset-1; i>=0; i--) {
+
+        // Because 9AM should be included in 9AM-10AM range, check for equality here
+        if( data[i].Date >= nextDate) {
+          break;
+        }
+
+         // increment offset so subsequent dates can skip first X items in array
+         offset += 1;
+
+        // continue to next item in array if we're still below our target date
+        if( data[i].Date < currentDate) {
+          continue;
+        }
+
+        // add to tick data
+        tick.Pageviews += data[i].Pageviews;
+        tick.Visitors += data[i].Visitors;
+      }
+
+      newData.push(tick);  
+      currentDate = nextDate;
+    }
+
+  return newData;
+  }
+
+
+  
   @bind
   prepareChart() {
     let padding = { top: 12, right: 12, bottom: 24, left: 40 };
@@ -130,19 +144,31 @@ class Chart extends Component {
 
     this.x = d3.scaleBand().range([0, this.innerWidth]).padding(0.1)
     this.y = d3.scaleLinear().range([this.innerHeight, 0])
-    this.ctx.call(tip)
+
+      // tooltip
+    this.tip = d3.tip().attr('class', 'd3-tip').html((d) => {
+      let title =  d.Date.toLocaleDateString();
+      if(this.state.diffInDays <= 1) {
+        title += ` ${d.Date.getHours()}:00 - ${d.Date.getHours() + 1}:00`
+      }
+      return (`<div class="tip-heading">${title}</div>
+      <div class="tip-content">
+        <div class="tip-pageviews">
+          <div class="tip-number">${d.Pageviews}</div>
+          <div class="tip-metric">Pageviews</div>
+        </div>
+        <div class="tip-visitors">
+          <div class="tip-number">${d.Visitors}</div>
+          <div class="tip-metric">Visitors</div>
+        </div>
+      </div>`)});
+    this.ctx.call(this.tip)
   }
 
   @bind
   redrawChart() {
     let data = this.state.data;
 
-    // hide chart & bail if we're trying to show less than 1 day worth of data
-    this.base.parentNode.style.display = data.length <= 1 ? 'none' : '';
-    if(data.length <= 1) {
-      return;
-    }
-
     if( ! this.ctx ) {
       this.prepareChart()
     }
@@ -156,8 +182,8 @@ class Chart extends Component {
     let yAxis = d3.axisLeft().scale(y).ticks(3).tickSize(-innerWidth)
     let xAxis = d3.axisBottom().scale(x).tickFormat(timeFormatPicker(data.length))
 
-     // hide all "day" ticks if we're watching more than 100 days of data
+     // hide all "day" ticks if we're watching more than 31 days of data
-    if(data.length > 100) {
+    if(data.length > 31) {
       xAxis.tickValues(data.filter(d => d.Date.getDate() === 1).map(d => d.Date))
     }
 
@@ -214,7 +240,7 @@ class Chart extends Component {
       .attr('y', (d) => y(d.Visitors))   
       
     // add event listeners for tooltips
-    days.on('mouseover', tip.show).on('mouseout', tip.hide)   
+    days.on('mouseover', this.tip.show).on('mouseout', this.tip.hide)   
   }
 
   @bind
@@ -228,7 +254,7 @@ class Chart extends Component {
           return;
         }
 
-        let chartData = prepareData(props.after, props.before, d);
+        let chartData = this.prepareData(d);
         this.setState({ 
           loading: false,
           data: chartData,

assets/src/js/components/DatePicker.js

@@ -10,16 +10,6 @@ const padZero = function(n){return n<10? '0'+n:''+n;}
 
 function getNow() {
   let now = new Date()
-  let tzOffset = now.getTimezoneOffset()  * 60 * 1000;
-
-  // if we're ahead of UTC, stick to UTC's "now"
-  // this is ugly but a sad necessity for now because we store and aggregate statistics using UTC dates (without time data)
-  // For those ahead of UTC, "today" will always be empty if they're checking early on in their day
-  // see https://github.com/usefathom/fathom/issues/134
-  if (tzOffset < 0) {
-    now.setTime(now.getTime() + tzOffset )
-  }
-
   return now
 }
 
@@ -116,8 +106,8 @@ class DatePicker extends Component {
 
     // create unix timestamps from local date objects
     let before, after;
-    before = Math.round((+endDate) / 1000) - endDate.getTimezoneOffset() * 60;
+    before = Math.round((+endDate) / 1000);
-    after = Math.round((+startDate) / 1000) - startDate.getTimezoneOffset() * 60;
+    after = Math.round((+startDate) / 1000);
 
     this.setState({
       period: period || '',

pkg/aggregator/store.go

@@ -10,7 +10,7 @@ import (
 )
 
 func (agg *Aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*models.SiteStats, error) {
-	cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02"))
+	cacheKey := fmt.Sprintf("%d-%s", siteID, t.Format("2006-01-02T15"))
 	if stats, ok := r.Sites[cacheKey]; ok {
 		return stats, nil
 
@@ -35,7 +35,7 @@ func (agg *Aggregator) getSiteStats(r *results, siteID int64, t time.Time) (*mod
 }
 
 func (agg *Aggregator) getPageStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.PageStats, error) {
-	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
+	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02T15"), hostname, pathname)
 	if stats, ok := r.Pages[cacheKey]; ok {
 		return stats, nil
 	}
@@ -71,7 +71,7 @@ func (agg *Aggregator) getPageStats(r *results, siteID int64, t time.Time, hostn
 }
 
 func (agg *Aggregator) getReferrerStats(r *results, siteID int64, t time.Time, hostname string, pathname string) (*models.ReferrerStats, error) {
-	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02"), hostname, pathname)
+	cacheKey := fmt.Sprintf("%d-%s-%s-%s", siteID, t.Format("2006-01-02T15"), hostname, pathname)
 	if stats, ok := r.Referrers[cacheKey]; ok {
 		return stats, nil
 	}

pkg/api/routes.go

@@ -23,10 +23,6 @@ func (api *API) Routes() *mux.Router {
 
 	r.Handle("/api/sites/{id:[0-9]+}/stats/site", api.Authorize(HandlerFunc(api.GetSiteStatsHandler))).Methods(http.MethodGet)
 	r.Handle("/api/sites/{id:[0-9]+}/stats/site/groupby/day", api.Authorize(HandlerFunc(api.GetSiteStatsPerDayHandler))).Methods(http.MethodGet)
-	r.Handle("/api/sites/{id:[0-9]+}/stats/site/pageviews", api.Authorize(HandlerFunc(api.GetSiteStatsPageviewsHandler))).Methods(http.MethodGet)
-	r.Handle("/api/sites/{id:[0-9]+}/stats/site/visitors", api.Authorize(HandlerFunc(api.GetSiteStatsVisitorsHandler))).Methods(http.MethodGet)
-	r.Handle("/api/sites/{id:[0-9]+}/stats/site/duration", api.Authorize(HandlerFunc(api.GetSiteStatsDurationHandler))).Methods(http.MethodGet)
-	r.Handle("/api/sites/{id:[0-9]+}/stats/site/bounces", api.Authorize(HandlerFunc(api.GetSiteStatsBouncesHandler))).Methods(http.MethodGet)
 	r.Handle("/api/sites/{id:[0-9]+}/stats/site/realtime", api.Authorize(HandlerFunc(api.GetSiteStatsRealtimeHandler))).Methods(http.MethodGet)
 
 	r.Handle("/api/sites/{id:[0-9]+}/stats/pages", api.Authorize(HandlerFunc(api.GetPageStatsHandler))).Methods(http.MethodGet)

pkg/api/site_stats.go

@@ -14,46 +14,6 @@ func (api *API) GetSiteStatsHandler(w http.ResponseWriter, r *http.Request) erro
 	return respond(w, http.StatusOK, envelope{Data: result})
 }
 
-// URL: /api/stats/site/pageviews
-func (api *API) GetSiteStatsPageviewsHandler(w http.ResponseWriter, r *http.Request) error {
-	params := GetRequestParams(r)
-	result, err := api.database.GetTotalSiteViews(params.SiteID, params.StartDate, params.EndDate)
-	if err != nil {
-		return err
-	}
-	return respond(w, http.StatusOK, envelope{Data: result})
-}
-
-// URL: /api/stats/site/visitors
-func (api *API) GetSiteStatsVisitorsHandler(w http.ResponseWriter, r *http.Request) error {
-	params := GetRequestParams(r)
-	result, err := api.database.GetTotalSiteVisitors(params.SiteID, params.StartDate, params.EndDate)
-	if err != nil {
-		return err
-	}
-	return respond(w, http.StatusOK, envelope{Data: result})
-}
-
-// URL: /api/stats/site/duration
-func (api *API) GetSiteStatsDurationHandler(w http.ResponseWriter, r *http.Request) error {
-	params := GetRequestParams(r)
-	result, err := api.database.GetAverageSiteDuration(params.SiteID, params.StartDate, params.EndDate)
-	if err != nil {
-		return err
-	}
-	return respond(w, http.StatusOK, envelope{Data: result})
-}
-
-// URL: /api/stats/site/bounces
-func (api *API) GetSiteStatsBouncesHandler(w http.ResponseWriter, r *http.Request) error {
-	params := GetRequestParams(r)
-	result, err := api.database.GetAverageSiteBounceRate(params.SiteID, params.StartDate, params.EndDate)
-	if err != nil {
-		return err
-	}
-	return respond(w, http.StatusOK, envelope{Data: result})
-}
-
 // URL: /api/stats/site/realtime
 func (api *API) GetSiteStatsRealtimeHandler(w http.ResponseWriter, r *http.Request) error {
 	params := GetRequestParams(r)

pkg/cli/cli.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"os"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -20,7 +21,7 @@ var app *App
 
 func Run(v string) error {
 	// force all times in UTC, regardless of server timezone
-	os.Setenv("TZ", "")
+	time.Local = time.UTC
 
 	// setup CLI app
 	app = &App{cli.NewApp(), nil, nil}

pkg/datastore/datastore.go

@@ -30,11 +30,6 @@ type Datastore interface {
 	GetSiteStatsPerDay(int64, time.Time, time.Time) ([]*models.SiteStats, error)
 	SaveSiteStats(*models.SiteStats) error
 	GetAggregatedSiteStats(int64, time.Time, time.Time) (*models.SiteStats, error)
-	GetTotalSiteViews(int64, time.Time, time.Time) (int64, error)
-	GetTotalSiteVisitors(int64, time.Time, time.Time) (int64, error)
-	GetTotalSiteSessions(int64, time.Time, time.Time) (int64, error)
-	GetAverageSiteDuration(int64, time.Time, time.Time) (float64, error)
-	GetAverageSiteBounceRate(int64, time.Time, time.Time) (float64, error)
 	GetRealtimeVisitorCount(int64) (int64, error)
 
 	// pageviews

pkg/datastore/sqlstore/config.go

@@ -55,14 +55,13 @@ func (c *Config) DSN() string {
 		mc.DBName = c.Name
 		mc.Params = map[string]string{
 			"parseTime": "true",
-			"loc":       "Local",
 		}
 		if c.SSLMode != "" {
 			mc.Params["tls"] = c.SSLMode
 		}
 		dsn = mc.FormatDSN()
 	case SQLITE:
-		dsn = c.Name + "?_loc=auto&_busy_timeout=5000"
+		dsn = c.Name + "?_busy_timeout=5000"
 	}
 
 	return dsn

pkg/datastore/sqlstore/migrations/mysql/21_alter_page_stats_table.sql

@@ -0,0 +1,19 @@
+-- +migrate Up
+CREATE TABLE page_stats(
+   site_id INTEGER NOT NULL DEFAULT 1,
+   hostname_id INTEGER NOT NULL,
+   pathname_id INTEGER NOT NULL,
+   pageviews INTEGER NOT NULL,
+   visitors INTEGER NOT NULL,
+   entries INTEGER NOT NULL,
+   bounce_rate FLOAT NOT NULL,
+   known_durations INTEGER NOT NULL DEFAULT 0,
+   avg_duration FLOAT NOT NULL,
+   ts DATETIME NOT NULL
+) CHARACTER SET=utf8;
+INSERT INTO page_stats 
+    SELECT site_id, hostname_id, pathname_id, pageviews, visitors, entries, bounce_rate, known_durations, avg_duration, CONCAT(date, ' 00:00:00')
+    FROM daily_page_stats s ;
+DROP TABLE daily_page_stats;
+
+-- +migrate Down

pkg/datastore/sqlstore/migrations/mysql/22_alter_site_stats_table.sql

@@ -0,0 +1,17 @@
+-- +migrate Up
+CREATE TABLE site_stats(
+   site_id INTEGER NOT NULL DEFAULT 1,
+   pageviews INTEGER NOT NULL,
+   visitors INTEGER NOT NULL,
+   sessions INTEGER NOT NULL,
+   bounce_rate FLOAT NOT NULL,
+   known_durations INTEGER NOT NULL DEFAULT 0,
+   avg_duration FLOAT NOT NULL,
+   ts DATETIME NOT NULL
+) CHARACTER SET=utf8;
+INSERT INTO site_stats 
+    SELECT site_id, pageviews, visitors, sessions, bounce_rate, known_durations, avg_duration,  CONCAT(date, ' 00:00:00')
+    FROM daily_site_stats s ;
+DROP TABLE daily_site_stats;
+
+-- +migrate Down

pkg/datastore/sqlstore/migrations/mysql/23_alter_referrer_stats_table.sql

@@ -0,0 +1,19 @@
+-- +migrate Up
+CREATE TABLE referrer_stats(
+    site_id INTEGER NOT NULL DEFAULT 1, 
+    hostname_id INTEGER NOT NULL,
+    pathname_id INTEGER NOT NULL,
+    groupname VARCHAR(255) NULL, 
+    pageviews INTEGER NOT NULL, 
+    visitors INTEGER NOT NULL, 
+    bounce_rate FLOAT NOT NULL, 
+    known_durations INTEGER NOT NULL DEFAULT 0, 
+    avg_duration FLOAT NOT NULL, 
+    ts DATETIME NOT NULL 
+) CHARACTER SET=utf8;
+INSERT INTO referrer_stats 
+    SELECT site_id, hostname_id, pathname_id, groupname, pageviews, visitors, bounce_rate, known_durations, avg_duration, CONCAT(date, ' 00:00:00')
+    FROM daily_referrer_stats s;
+DROP TABLE daily_referrer_stats;
+
+-- +migrate Down

pkg/datastore/sqlstore/migrations/mysql/24_recreate_stat_table_indices.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+CREATE UNIQUE INDEX unique_page_stats ON page_stats(site_id, hostname_id, pathname_id, ts);
+CREATE UNIQUE INDEX unique_referrer_stats ON referrer_stats(site_id, hostname_id, pathname_id, ts);
+CREATE UNIQUE INDEX unique_site_stats ON site_stats(site_id, ts);
+
+-- +migrate Down
+DROP INDEX unique_page_stats ON page_stats;
+DROP INDEX unique_referrer_stats ON referrer_stats;
+DROP INDEX unique_site_stats ON site_stats;

pkg/datastore/sqlstore/migrations/postgres/22_alter_page_stats_table.sql

@@ -0,0 +1,20 @@
+-- +migrate Up
+CREATE TABLE page_stats(
+   site_id INTEGER NOT NULL DEFAULT 1,
+   hostname_id INTEGER NOT NULL,
+   pathname_id INTEGER NOT NULL,
+   pageviews INTEGER NOT NULL,
+   visitors INTEGER NOT NULL,
+   entries INTEGER NOT NULL,
+   bounce_rate FLOAT NOT NULL,
+   known_durations INTEGER NOT NULL DEFAULT 0,
+   avg_duration FLOAT NOT NULL,
+   ts TIMESTAMP WITHOUT TIME ZONE NOT NULL
+);
+INSERT INTO page_stats 
+    SELECT site_id, hostname_id, pathname_id, pageviews, visitors, entries, bounce_rate, known_durations, avg_duration, (date || ' 00:00:00')::timestamp
+    FROM daily_page_stats s;
+DROP TABLE daily_page_stats;
+
+-- +migrate Down
+

pkg/datastore/sqlstore/migrations/postgres/23_alter_referrer_stats_table.sql

@@ -0,0 +1,19 @@
+-- +migrate Up
+CREATE TABLE referrer_stats(
+    site_id INTEGER NOT NULL DEFAULT 1, 
+    hostname_id INTEGER NOT NULL,
+    pathname_id INTEGER NOT NULL,
+    groupname VARCHAR(255) NULL, 
+    pageviews INTEGER NOT NULL, 
+    visitors INTEGER NOT NULL, 
+    bounce_rate FLOAT NOT NULL, 
+    known_durations INTEGER NOT NULL DEFAULT 0, 
+    avg_duration FLOAT NOT NULL, 
+    ts TIMESTAMP WITHOUT TIME ZONE NOT NULL 
+);
+INSERT INTO referrer_stats 
+    SELECT site_id, hostname_id, pathname_id, groupname, pageviews, visitors, bounce_rate, known_durations, avg_duration, (date || ' 00:00:00')::timestamp
+    FROM daily_referrer_stats s;
+DROP TABLE daily_referrer_stats;
+
+-- +migrate Down

pkg/datastore/sqlstore/migrations/postgres/24_alter_site_stats_table.sql

@@ -0,0 +1,17 @@
+-- +migrate Up
+CREATE TABLE site_stats(
+   site_id INTEGER NOT NULL DEFAULT 1,
+   pageviews INTEGER NOT NULL,
+   visitors INTEGER NOT NULL,
+   sessions INTEGER NOT NULL,
+   bounce_rate FLOAT NOT NULL,
+   known_durations INTEGER NOT NULL DEFAULT 0,
+   avg_duration FLOAT NOT NULL,
+   ts TIMESTAMP WITHOUT TIME ZONE NOT NULL
+);
+INSERT INTO site_stats 
+    SELECT site_id, pageviews, visitors, sessions, bounce_rate, known_durations, avg_duration, (date || ' 00:00:00')::timestamp
+    FROM daily_site_stats s;
+DROP TABLE daily_site_stats;
+
+-- +migrate Down

pkg/datastore/sqlstore/migrations/postgres/25_recreate_stat_table_indices.sql

@@ -0,0 +1,11 @@
+-- +migrate Up
+DROP INDEX IF EXISTS unique_daily_page_stats;
+DROP INDEX IF EXISTS unique_daily_referrer_stats;
+CREATE UNIQUE INDEX unique_page_stats ON page_stats(site_id, hostname_id, pathname_id, ts);
+CREATE UNIQUE INDEX unique_referrer_stats ON referrer_stats(site_id, hostname_id, pathname_id, ts);
+CREATE UNIQUE INDEX unique_site_stats ON site_stats(site_id, ts);
+
+-- +migrate Down
+DROP INDEX IF EXISTS unique_page_stats;
+DROP INDEX IF EXISTS unique_referrer_stats;
+DROP INDEX IF EXISTS unique_site_stats;

pkg/datastore/sqlstore/migrations/postgres/26_alter_pageviews_table.sql

@@ -0,0 +1,7 @@
+-- +migrate Up
+
+ALTER TABLE pageviews ALTER COLUMN timestamp TYPE TIMESTAMP WITHOUT TIME ZONE;
+
+-- +migrate Down
+
+ALTER TABLE pageviews ALTER COLUMN timestamp TYPE TIMESTAMP WITH TIME ZONE;

pkg/datastore/sqlstore/migrations/sqlite3/21_alter_page_stats_table.sql

@@ -0,0 +1,19 @@
+-- +migrate Up
+CREATE TABLE page_stats(
+   site_id INTEGER NOT NULL DEFAULT 1,
+   hostname_id INTEGER NOT NULL,
+   pathname_id INTEGER NOT NULL,
+   pageviews INTEGER NOT NULL,
+   visitors INTEGER NOT NULL,
+   entries INTEGER NOT NULL,
+   bounce_rate FLOAT NOT NULL,
+   known_durations INTEGER NOT NULL DEFAULT 0,
+   avg_duration FLOAT NOT NULL,
+   ts DATETIME NOT NULL
+);
+INSERT INTO page_stats 
+    SELECT site_id, hostname_id, pathname_id, pageviews, visitors, entries, bounce_rate, known_durations, avg_duration, date || ' 00:00:00'
+    FROM daily_page_stats s ;
+DROP TABLE daily_page_stats;
+
+-- +migrate Down

pkg/datastore/sqlstore/migrations/sqlite3/22_alter_site_stats_table.sql

@@ -0,0 +1,17 @@
+-- +migrate Up
+CREATE TABLE site_stats(
+   site_id INTEGER NOT NULL DEFAULT 1,
+   pageviews INTEGER NOT NULL,
+   visitors INTEGER NOT NULL,
+   sessions INTEGER NOT NULL,
+   bounce_rate FLOAT NOT NULL,
+   known_durations INTEGER NOT NULL DEFAULT 0,
+   avg_duration FLOAT NOT NULL,
+   ts DATETIME NOT NULL
+);
+INSERT INTO site_stats 
+    SELECT site_id, pageviews, visitors, sessions, bounce_rate, known_durations, avg_duration, date || ' 00:00:00'
+    FROM daily_site_stats s ;
+DROP TABLE daily_site_stats;
+
+-- +migrate Down

pkg/datastore/sqlstore/migrations/sqlite3/23_alter_referrer_stats_table.sql

@@ -0,0 +1,19 @@
+-- +migrate Up
+CREATE TABLE referrer_stats(
+    site_id INTEGER NOT NULL DEFAULT 1, 
+    hostname_id INTEGER NOT NULL,
+    pathname_id INTEGER NOT NULL,
+    groupname VARCHAR(255) NULL, 
+    pageviews INTEGER NOT NULL, 
+    visitors INTEGER NOT NULL, 
+    bounce_rate FLOAT NOT NULL, 
+    known_durations INTEGER NOT NULL DEFAULT 0, 
+    avg_duration FLOAT NOT NULL, 
+    ts DATETIME NOT NULL 
+);
+INSERT INTO referrer_stats 
+    SELECT site_id, hostname_id, pathname_id, groupname, pageviews, visitors, bounce_rate, known_durations, avg_duration, date || ' 00:00:00' 
+    FROM daily_referrer_stats s;
+DROP TABLE daily_referrer_stats;
+
+-- +migrate Down

pkg/datastore/sqlstore/migrations/sqlite3/24_recreate_stat_table_indices.sql

@@ -0,0 +1,11 @@
+-- +migrate Up
+DROP INDEX IF EXISTS unique_daily_page_stats;
+DROP INDEX IF EXISTS unique_daily_referrer_stats;
+CREATE UNIQUE INDEX unique_page_stats ON page_stats(site_id, hostname_id, pathname_id, ts);
+CREATE UNIQUE INDEX unique_referrer_stats ON referrer_stats(site_id, hostname_id, pathname_id, ts);
+CREATE UNIQUE INDEX unique_site_stats ON site_stats(site_id, ts);
+
+-- +migrate Down
+DROP INDEX IF EXISTS unique_page_stats;
+DROP INDEX IF EXISTS unique_referrer_stats;
+DROP INDEX IF EXISTS unique_site_stats;

pkg/datastore/sqlstore/page_stats.go

@@ -9,8 +9,8 @@ import (
 
 func (db *sqlstore) GetPageStats(siteID int64, date time.Time, hostnameID int64, pathnameID int64) (*models.PageStats, error) {
 	stats := &models.PageStats{New: false}
-	query := db.Rebind(`SELECT * FROM daily_page_stats WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND date = ? LIMIT 1`)
+	query := db.Rebind(`SELECT * FROM page_stats WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND ts = ? LIMIT 1`)
-	err := db.Get(stats, query, siteID, hostnameID, pathnameID, date.Format("2006-01-02"))
+	err := db.Get(stats, query, siteID, hostnameID, pathnameID, date.Format(DATE_FORMAT))
 	if err == sql.ErrNoRows {
 		return nil, ErrNoResults
 	}
@@ -27,14 +27,14 @@ func (db *sqlstore) SavePageStats(s *models.PageStats) error {
 }
 
 func (db *sqlstore) insertPageStats(s *models.PageStats) error {
-	query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, site_id, hostname_id, pathname_id, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+	query := db.Rebind(`INSERT INTO page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, known_durations, site_id, hostname_id, pathname_id, ts) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
-	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format(DATE_FORMAT))
 	return err
 }
 
 func (db *sqlstore) updatePageStats(s *models.PageStats) error {
-	query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ?, avg_duration = ?, known_durations = ? WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND date = ?`)
+	query := db.Rebind(`UPDATE page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ?, avg_duration = ?, known_durations = ? WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND ts = ?`)
-	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format(DATE_FORMAT))
 	return err
 }
 
@@ -48,19 +48,19 @@ func (db *sqlstore) GetAggregatedPageStats(siteID int64, startDate time.Time, en
 		SUM(entries) AS entries, 
 		COALESCE(SUM(entries*bounce_rate) / NULLIF(SUM(entries), 0), 0.00) AS bounce_rate, 
 		COALESCE(SUM(pageviews*avg_duration) / SUM(pageviews), 0.00) AS avg_duration 
-		FROM daily_page_stats s 
+		FROM page_stats s 
 			LEFT JOIN hostnames h ON h.id = s.hostname_id 
 			LEFT JOIN pathnames p ON p.id = s.pathname_id 
-		WHERE site_id = ? AND date >= ? AND date <= ? 
+		WHERE site_id = ? AND ts >= ? AND ts <= ? 
 		GROUP BY hostname, pathname 
 		ORDER BY pageviews DESC LIMIT ?`)
-	err := db.Select(&result, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit)
+	err := db.Select(&result, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT), limit)
 	return result, err
 }
 
 func (db *sqlstore) GetAggregatedPageStatsPageviews(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
 	var result int64
-	query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM daily_page_stats WHERE site_id = ? AND date >= ? AND date <= ?`)
+	query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM page_stats WHERE site_id = ? AND ts >= ? AND ts <= ?`)
-	err := db.Get(&result, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
+	err := db.Get(&result, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT))
 	return result, err
 }

pkg/datastore/sqlstore/referrer_stats.go

@@ -9,8 +9,8 @@ import (
 
 func (db *sqlstore) GetReferrerStats(siteID int64, date time.Time, hostnameID int64, pathnameID int64) (*models.ReferrerStats, error) {
 	stats := &models.ReferrerStats{New: false}
-	query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE site_id = ? AND date = ? AND hostname_id = ? AND pathname_id = ? LIMIT 1`)
+	query := db.Rebind(`SELECT * FROM referrer_stats WHERE site_id = ? AND ts = ? AND hostname_id = ? AND pathname_id = ? LIMIT 1`)
-	err := db.Get(stats, query, siteID, date.Format("2006-01-02"), hostnameID, pathnameID)
+	err := db.Get(stats, query, siteID, date.Format(DATE_FORMAT), hostnameID, pathnameID)
 	if err == sql.ErrNoRows {
 		return nil, ErrNoResults
 	}
@@ -27,14 +27,14 @@ func (db *sqlstore) SaveReferrerStats(s *models.ReferrerStats) error {
 }
 
 func (db *sqlstore) insertReferrerStats(s *models.ReferrerStats) error {
-	query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, site_id, hostname_id, pathname_id, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+	query := db.Rebind(`INSERT INTO referrer_stats(visitors, pageviews, bounce_rate, avg_duration, known_durations, groupname, site_id, hostname_id, pathname_id, ts) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
-	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format(DATE_FORMAT))
 	return err
 }
 
 func (db *sqlstore) updateReferrerStats(s *models.ReferrerStats) error {
-	query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ?, avg_duration = ?, known_durations = ?, groupname = ? WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND date = ?`)
+	query := db.Rebind(`UPDATE referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ?, avg_duration = ?, known_durations = ?, groupname = ? WHERE site_id = ? AND hostname_id = ? AND pathname_id = ? AND ts = ?`)
-	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Group, s.SiteID, s.HostnameID, s.PathnameID, s.Date.Format(DATE_FORMAT))
 	return err
 }
 
@@ -49,10 +49,10 @@ func (db *sqlstore) GetAggregatedReferrerStats(siteID int64, startDate time.Time
 		SUM(pageviews) AS pageviews, 
 		COALESCE(SUM(pageviews*NULLIF(bounce_rate, 0)) / SUM(pageviews), 0.00) AS bounce_rate, 
 		COALESCE(SUM(pageviews*avg_duration) / SUM(pageviews), 0.00) AS avg_duration 
-	FROM daily_referrer_stats s
+	FROM referrer_stats s
 		LEFT JOIN hostnames h ON h.id = s.hostname_id 
 		LEFT JOIN pathnames p ON p.id = s.pathname_id 
-	WHERE site_id = ? AND date >= ? AND date <= ? `
+	WHERE site_id = ? AND ts >= ? AND ts <= ? `
 
 	if db.Config.Driver == "sqlite3" {
 		sql = sql + `GROUP BY COALESCE(NULLIF(groupname, ''), hostname || pathname ) `
@@ -63,13 +63,13 @@ func (db *sqlstore) GetAggregatedReferrerStats(siteID int64, startDate time.Time
 
 	query := db.Rebind(sql)
 
-	err := db.Select(&result, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit)
+	err := db.Select(&result, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT), limit)
 	return result, mapError(err)
 }
 
 func (db *sqlstore) GetAggregatedReferrerStatsPageviews(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
 	var result int64
-	query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM daily_referrer_stats WHERE site_id = ? AND date >= ? AND date <= ?`)
+	query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM referrer_stats WHERE site_id = ? AND ts >= ? AND ts <= ?`)
-	err := db.Get(&result, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
+	err := db.Get(&result, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT))
 	return result, mapError(err)
 }

pkg/datastore/sqlstore/site_stats.go

@@ -10,9 +10,9 @@ import (
 
 func (db *sqlstore) GetSiteStats(siteID int64, date time.Time) (*models.SiteStats, error) {
 	stats := &models.SiteStats{New: false}
-	query := db.Rebind(`SELECT * FROM daily_site_stats WHERE site_id = ? AND date = ? LIMIT 1`)
+	query := db.Rebind(`SELECT * FROM site_stats WHERE site_id = ? AND ts = ? LIMIT 1`)
 
-	err := db.Get(stats, query, siteID, date.Format("2006-01-02"))
+	err := db.Get(stats, query, siteID, date.Format(DATE_FORMAT))
 	if err == sql.ErrNoRows {
 		return nil, ErrNoResults
 	}
@@ -29,22 +29,21 @@ func (db *sqlstore) SaveSiteStats(s *models.SiteStats) error {
 }
 
 func (db *sqlstore) insertSiteStats(s *models.SiteStats) error {
-	query := db.Rebind(`INSERT INTO daily_site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
+	query := db.Rebind(`INSERT INTO site_stats(site_id, visitors, sessions, pageviews, bounce_rate, avg_duration, known_durations, ts) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
-	_, err := db.Exec(query, s.SiteID, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.SiteID, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.Date.Format(DATE_FORMAT))
 	return err
 }
 
 func (db *sqlstore) updateSiteStats(s *models.SiteStats) error {
-	query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ?, avg_duration = ?, known_durations = ? WHERE site_id = ? AND date = ?`)
+	query := db.Rebind(`UPDATE site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ?, avg_duration = ?, known_durations = ? WHERE site_id = ? AND ts = ?`)
-	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Date.Format("2006-01-02"))
+	_, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.KnownDurations, s.SiteID, s.Date.Format(DATE_FORMAT))
 	return err
 }
 
 func (db *sqlstore) GetSiteStatsPerDay(siteID int64, startDate time.Time, endDate time.Time) ([]*models.SiteStats, error) {
 	results := []*models.SiteStats{}
-	sql := `SELECT * FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
+	query := db.Rebind(`SELECT * FROM site_stats WHERE site_id = ? AND ts >= ? AND ts <= ? ORDER BY ts DESC`)
-	query := db.Rebind(sql)
+	err := db.Select(&results, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT))
-	err := db.Select(&results, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
 	return results, err
 }
 
@@ -56,51 +55,12 @@ func (db *sqlstore) GetAggregatedSiteStats(siteID int64, startDate time.Time, en
 		COALESCE(SUM(sessions), 0) AS sessions,
 		COALESCE(SUM(pageviews*avg_duration) / NULLIF(SUM(pageviews), 0), 0.00) AS avg_duration,
 		COALESCE(SUM(sessions*bounce_rate) / NULLIF(SUM(sessions), 0), 0.00) AS bounce_rate
-	 FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ? LIMIT 1`)
+	 FROM site_stats 
-	err := db.Get(stats, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
+	 WHERE site_id = ? AND ts >= ? AND ts <= ? LIMIT 1`)
+	err := db.Get(stats, query, siteID, startDate.Format(DATE_FORMAT), endDate.Format(DATE_FORMAT))
 	return stats, mapError(err)
 }
 
-func (db *sqlstore) GetTotalSiteViews(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
-	sql := `SELECT COALESCE(SUM(pageviews), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
-	query := db.Rebind(sql)
-	var total int64
-	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
-	return total, mapError(err)
-}
-
-func (db *sqlstore) GetTotalSiteVisitors(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
-	sql := `SELECT COALESCE(SUM(visitors), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
-	query := db.Rebind(sql)
-	var total int64
-	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
-	return total, mapError(err)
-}
-
-func (db *sqlstore) GetTotalSiteSessions(siteID int64, startDate time.Time, endDate time.Time) (int64, error) {
-	sql := `SELECT COALESCE(SUM(sessions), 0) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
-	query := db.Rebind(sql)
-	var total int64
-	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
-	return total, mapError(err)
-}
-
-func (db *sqlstore) GetAverageSiteDuration(siteID int64, startDate time.Time, endDate time.Time) (float64, error) {
-	sql := `SELECT COALESCE(SUM(pageviews*avg_duration)/SUM(pageviews), 0.00) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
-	query := db.Rebind(sql)
-	var total float64
-	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
-	return total, mapError(err)
-}
-
-func (db *sqlstore) GetAverageSiteBounceRate(siteID int64, startDate time.Time, endDate time.Time) (float64, error) {
-	sql := `SELECT COALESCE(SUM(sessions*bounce_rate)/SUM(sessions), 4) FROM daily_site_stats WHERE site_id = ? AND date >= ? AND date <= ?`
-	query := db.Rebind(sql)
-	var total float64
-	err := db.Get(&total, query, siteID, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
-	return total, mapError(err)
-}
-
 func (db *sqlstore) GetRealtimeVisitorCount(siteID int64) (int64, error) {
 	var siteTrackingID string
 	if err := db.Get(&siteTrackingID, db.Rebind(`SELECT tracking_id FROM sites WHERE id = ? LIMIT 1`), siteID); err != nil && err != sql.ErrNoRows {

pkg/datastore/sqlstore/sqlstore.go

@@ -19,6 +19,8 @@ const (
 	MYSQL    = "mysql"
 	POSTGRES = "postgres"
 	SQLITE   = "sqlite3"
+
+	DATE_FORMAT = "2006-01-02 15:00:00"
 )
 
 type sqlstore struct {

pkg/models/page_stats.go

@@ -17,7 +17,7 @@ type PageStats struct {
 	BounceRate     float64   `db:"bounce_rate"`
 	AvgDuration    float64   `db:"avg_duration"`
 	KnownDurations int64     `db:"known_durations"`
-	Date           time.Time `db:"date" json:",omitempty"`
+	Date           time.Time `db:"ts" json:",omitempty"`
 }
 
 func (s *PageStats) HandlePageview(p *Pageview) {

pkg/models/referrer_stats.go

@@ -17,7 +17,7 @@ type ReferrerStats struct {
 	BounceRate     float64   `db:"bounce_rate"`
 	AvgDuration    float64   `db:"avg_duration"`
 	KnownDurations int64     `db:"known_durations"`
-	Date           time.Time `db:"date" json:",omitempty"`
+	Date           time.Time `db:"ts" json:",omitempty"`
 }
 
 func (s *ReferrerStats) HandlePageview(p *Pageview) {

pkg/models/site_stats.go

@@ -14,7 +14,7 @@ type SiteStats struct {
 	BounceRate     float64   `db:"bounce_rate"`
 	AvgDuration    float64   `db:"avg_duration"`
 	KnownDurations int64     `db:"known_durations" json:",omitempty"`
-	Date           time.Time `db:"date" json:",omitempty"`
+	Date           time.Time `db:"ts" json:",omitempty"`
 }
 
 func (s *SiteStats) FormattedDuration() string {