status-im
/
fathom
mirror of https://github.com/status-im/fathom.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

get rid of session_id and move knowledge of previous pageview to client. this also gets rid of the (only) index on the pageviews table, allowing for much faster INSERT's. closes #14

This commit is contained in:
Danny 2018-07-11 15:03:56 +02:00
parent d9bce6a0cf
commit 42008ab83f
10 changed files with 91 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
assets/src/js/lib/util.js
View File

@ -11,12 +11,12 @@ function stringifyObject(json) {
}).join('&');
}
function generateKey() {
function randomString(n) {
var s = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
return Array(16).join().split(',').map(function() { return s.charAt(Math.floor(Math.random() * s.length)); }).join('');
return Array(n).join().split(',').map(() => s.charAt(Math.floor(Math.random() * s.length))).join('');
}
export {
generateKey,
randomString,
stringifyObject
}

8
assets/src/js/tracker.js
View File

@ -13,10 +13,10 @@ const commands = {
function newVisitorData() {
return {
sid: util.generateKey(),
isNewVisitor: true,
isNewSession: true,
pagesViewed: [],
previousPageviewId: '',
lastSeen: +new Date(),
}
}
@ -97,7 +97,8 @@ function trackPageview() {
let data = getData();
const d = {
sid: data.sid,
id: util.randomString(20),
pid: data.previousPageviewId || '',
p: path,
h: hostname,
r: referrer,
@ -113,10 +114,11 @@ function trackPageview() {
let midnight = new Date(Date.UTC(now.getFullYear(), now.getMonth(), now.getDate(), 24, 0, 0));
let expires = Math.round((midnight - now) / 1000);
// update data in cookie
if( data.pagesViewed.indexOf(path) == -1 ) {
data.pagesViewed.push(path);
}
data.previousPageviewId = d.id;
data.isNewVisitor = false;
data.isNewSession = false;
data.lastSeen = +new Date();

7
pkg/api/collect.go
View File

@ -77,7 +77,7 @@ func (api *API) NewCollectHandler() http.Handler {
// get pageview details
pageview := &models.Pageview{
SessionID: q.Get("sid"),
ID: q.Get("id"),
Hostname: parseHostname(q.Get("h")),
Pathname: parsePathname(q.Get("p")),
IsNewVisitor: q.Get("nv") == "1",
@ -90,8 +90,9 @@ func (api *API) NewCollectHandler() http.Handler {
}
// find previous pageview by same visitor
if !pageview.IsNewSession {
previousPageview, err := api.database.GetMostRecentPageviewBySessionID(pageview.SessionID)
previousPageviewID := q.Get("pid")
if !pageview.IsNewSession && previousPageviewID != "" {
previousPageview, err := api.database.GetPageview(previousPageviewID)
if err != nil && err != datastore.ErrNoResults {
return err
}

2
pkg/datastore/datastore.go
View File

@ -32,7 +32,7 @@ type Datastore interface {
// pageviews
SavePageview(*models.Pageview) error
UpdatePageview(*models.Pageview) error
GetMostRecentPageviewBySessionID(string) (*models.Pageview, error)
GetPageview(string) (*models.Pageview, error)
GetProcessablePageviews() ([]*models.Pageview, error)
DeletePageviews([]*models.Pageview) error

14
pkg/datastore/sqlstore/migrations/mysql/4_pageview_id_column.sql Normal file
View File

@ -0,0 +1,14 @@
-- +migrate Up
ALTER TABLE pageviews DROP COLUMN session_id;
ALTER TABLE pageviews DROP COLUMN id;
ALTER TABLE pageviews ADD COLUMN id VARCHAR(31) NOT NULL FIRST;
-- +migrate Down
ALTER TABLE pageviews DROP COLUMN id;
ALTER TABLE pageviews ADD COLUMN id INT AUTO_INCREMENT PRIMARY KEY NOT NULL FIRST;
ALTER TABLE pageviews ADD COLUMN session_id VARCHAR(16) NOT NULL AFTER id;

15
pkg/datastore/sqlstore/migrations/postgres/4_pageview_id_column.sql Normal file
View File

@ -0,0 +1,15 @@
-- +migrate Up
TRUNCATE pageviews; -- postgres will fail because of NULL values otherwise
ALTER TABLE pageviews DROP COLUMN session_id;
ALTER TABLE pageviews DROP COLUMN id;
ALTER TABLE pageviews ADD COLUMN id VARCHAR(31) NOT NULL;
-- +migrate Down
ALTER TABLE pageviews DROP COLUMN id;
ALTER TABLE pageviews ADD COLUMN id INT AUTO_INCREMENT PRIMARY KEY NOT NULL;
ALTER TABLE pageviews ADD COLUMN session_id VARCHAR(16) NOT NULL;

32
pkg/datastore/sqlstore/migrations/sqlite3/4_pageview_id_column.sql Normal file
View File

@ -0,0 +1,32 @@
-- +migrate Up
DROP TABLE pageviews;
CREATE TABLE pageviews(
id VARCHAR(31) NOT NULL,
hostname VARCHAR(255) NOT NULL,
pathname VARCHAR(255) NOT NULL,
is_new_visitor TINYINT(1) NOT NULL,
is_new_session TINYINT(1) NOT NULL,
is_unique TINYINT(1) NOT NULL,
is_bounce TINYINT(1) NULL,
referrer VARCHAR(255) NULL,
duration INTEGER(4) NULL,
timestamp DATETIME NOT NULL
);
-- +migrate Down
DROP TABLE pageviews;
CREATE TABLE pageviews(
id INTEGER PRIMARY KEY,
hostname VARCHAR(255) NOT NULL,
pathname VARCHAR(255) NOT NULL,
session_id VARCHAR(16) NOT NULL,
is_new_visitor TINYINT(1) NOT NULL,
is_new_session TINYINT(1) NOT NULL,
is_unique TINYINT(1) NOT NULL,
is_bounce TINYINT(1) NULL,
referrer VARCHAR(255) NULL,
duration INTEGER(4) NULL,
timestamp DATETIME NOT NULL
);

29
pkg/datastore/sqlstore/pageviews.go
View File

@ -2,7 +2,6 @@ package sqlstore
import (
"database/sql"
"strconv"
"strings"
"time"
@ -11,26 +10,21 @@ import (
// SavePageview inserts a single pageview model into the connected database
func (db *sqlstore) SavePageview(p *models.Pageview) error {
query := db.Rebind(`INSERT INTO pageviews(hostname, pathname, session_id, is_new_visitor, is_new_session, is_unique, is_bounce, referrer, duration, timestamp) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
result, err := db.Exec(query, p.Hostname, p.Pathname, p.SessionID, p.IsNewVisitor, p.IsNewSession, p.IsUnique, p.IsBounce, p.Referrer, p.Duration, p.Timestamp)
query := db.Rebind(`INSERT INTO pageviews(id, hostname, pathname, is_new_visitor, is_new_session, is_unique, is_bounce, referrer, duration, timestamp) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
_, err := db.Exec(query, p.ID, p.Hostname, p.Pathname, p.IsNewVisitor, p.IsNewSession, p.IsUnique, p.IsBounce, p.Referrer, p.Duration, p.Timestamp)
if err != nil {
return err
}
p.ID, _ = result.LastInsertId()
return nil
}
func (db *sqlstore) UpdatePageview(p *models.Pageview) error {
query := db.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`)
_, err := db.Exec(query, p.IsBounce, p.Duration, p.ID)
return err
}
func (db *sqlstore) GetMostRecentPageviewBySessionID(sessionID string) (*models.Pageview, error) {
// GetPageview selects a single pageview by its string ID
func (db *sqlstore) GetPageview(id string) (*models.Pageview, error) {
result := &models.Pageview{}
query := db.Rebind(`SELECT * FROM pageviews WHERE session_id = ? ORDER BY id DESC LIMIT 1`)
err := db.Get(result, query, sessionID)
query := db.Rebind(`SELECT * FROM pageviews WHERE id = ? LIMIT 1`)
err := db.Get(result, query, id)
if err != nil {
if err == sql.ErrNoRows {
return nil, ErrNoResults
@ -42,6 +36,13 @@ func (db *sqlstore) GetMostRecentPageviewBySessionID(sessionID string) (*models.
return result, nil
}
func (db *sqlstore) UpdatePageview(p *models.Pageview) error {
query := db.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`)
_, err := db.Exec(query, p.IsBounce, p.Duration, p.ID)
return err
}
// GetProcessablePageviews selects all pageviews which are "done" (ie not still waiting for bounce flag or duration)
func (db *sqlstore) GetProcessablePageviews() ([]*models.Pageview, error) {
var results []*models.Pageview
thirtyMinsAgo := time.Now().Add(-30 * time.Minute)
@ -54,7 +55,7 @@ func (db *sqlstore) GetProcessablePageviews() ([]*models.Pageview, error) {
func (db *sqlstore) DeletePageviews(pageviews []*models.Pageview) error {
ids := []string{}
for _, p := range pageviews {
ids = append(ids, strconv.FormatInt(p.ID, 10))
ids = append(ids, "'"+p.ID+"'")
}
query := db.Rebind(`DELETE FROM pageviews WHERE id IN(` + strings.Join(ids, ",") + `)`)
_, err := db.Exec(query)

2
pkg/datastore/sqlstore/site_stats.go
View File

@ -93,7 +93,7 @@ func (db *sqlstore) GetAverageSiteBounceRate(startDate time.Time, endDate time.T
}
func (db *sqlstore) GetRealtimeVisitorCount() (int, error) {
sql := `SELECT COUNT(DISTINCT(session_id)) FROM pageviews WHERE timestamp > ?`
sql := `SELECT COUNT(*) FROM pageviews WHERE ( duration = 0 OR is_bounce = TRUE ) AND timestamp > ?`
query := db.Rebind(sql)
var total int
err := db.Get(&total, query, time.Now().Add(-5*time.Minute))

3
pkg/models/pageview.go
View File

@ -5,8 +5,7 @@ import (
)
type Pageview struct {
ID int64 `db:"id"`
SessionID string `db:"session_id"`
ID string `db:"id"`
Hostname string `db:"hostname"`
Pathname string `db:"pathname"`
IsNewVisitor bool `db:"is_new_visitor"`