status-im
/
fathom
mirror of https://github.com/status-im/fathom.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

move raw SQL from /api/collect handler to datastore package

This commit is contained in:
Danny van Kooten 2017-01-25 20:01:02 +01:00
parent 28fa8431ef
commit 6b5cccd147
9 changed files with 211 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
api/collect.go
View File

@ -1,9 +1,9 @@
package api
import (
"database/sql"
"crypto/md5"
"encoding/base64"
"log"
"encoding/hex"
"net/http"
"strings"
"time"
@ -35,53 +35,41 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
// find or insert page
page := models.Page{
Path: q.Get("p"),
Title: q.Get("t"),
Hostname: q.Get("h"),
}
stmt, _ := datastore.DB.Prepare("SELECT p.id FROM pages p WHERE p.hostname = ? AND p.path = ? LIMIT 1")
defer stmt.Close()
err := stmt.QueryRow(page.Hostname, page.Path).Scan(&page.ID)
if err != nil {
if err == sql.ErrNoRows {
page.Save(datastore.DB)
} else {
log.Fatal(err)
page, err := datastore.GetPageByHostnameAndPath(q.Get("h"), q.Get("p"))
if page.ID == 0 {
page = &models.Page{
Hostname: q.Get("h"),
Path: q.Get("p"),
Title: q.Get("t"),
}
err = datastore.SavePage(page)
}
checkError(err)
// find or insert visitor.
now := time.Now()
ip := getRequestIp(r)
visitor := models.Visitor{
IpAddress: ip,
BrowserLanguage: q.Get("l"),
ScreenResolution: q.Get("sr"),
DeviceOS: ua.OS(),
Country: "",
}
ipAddress := getRequestIp(r)
visitorKey := generateVisitorKey(now.Format("2006-01-02"), ipAddress, r.UserAgent())
// add browser details
visitor.BrowserName, visitor.BrowserVersion = ua.Browser()
visitor.BrowserName = parseMajorMinor(visitor.BrowserName)
// query by unique visitor key
visitor.Key = visitor.GenerateKey(now.Format("2006-01-02"), visitor.IpAddress, r.UserAgent())
stmt, _ = datastore.DB.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1")
defer stmt.Close()
err = stmt.QueryRow(visitor.Key).Scan(&visitor.ID)
if err != nil {
if err == sql.ErrNoRows {
err = visitor.Save(datastore.DB)
checkError(err)
} else {
log.Fatal(err)
visitor, err := datastore.GetVisitorByKey(visitorKey)
if visitor.ID == 0 {
visitor = &models.Visitor{
IpAddress: ipAddress,
BrowserLanguage: q.Get("l"),
ScreenResolution: q.Get("sr"),
DeviceOS: ua.OS(),
Country: "",
Key: visitorKey,
}
}
pageview := models.Pageview{
// add browser details
visitor.BrowserName, visitor.BrowserVersion = ua.Browser()
visitor.BrowserName = parseMajorMinor(visitor.BrowserName)
err = datastore.SaveVisitor(visitor)
}
checkError(err)
pageview := &models.Pageview{
PageID: page.ID,
VisitorID: visitor.ID,
ReferrerUrl: q.Get("ru"),
@ -94,7 +82,7 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
pageview.ReferrerUrl = ""
}
err = pageview.Save(datastore.DB)
err = datastore.SavePageview(pageview)
checkError(err)
// don't you cache this
@ -108,3 +96,9 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
b, _ := base64.StdEncoding.DecodeString("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7")
w.Write(b)
}
// generateVisitorKey generates the "unique" visitor key from date, user agent + screen resolution
func generateVisitorKey(date string, ipAddress string, userAgent string) string {
byteKey := md5.Sum([]byte(date + ipAddress + userAgent))
return hex.EncodeToString(byteKey[:])
}

42
datastore/pages.go Normal file
View File

@ -0,0 +1,42 @@
package datastore
import (
"github.com/dannyvankooten/ana/models"
)
var p models.Page
// GetPage ...
func GetPage(id int64) (*models.Page, error) {
return &p, err
}
// GetPageByHostnameAndPath ...
func GetPageByHostnameAndPath(hostname, path string) (*models.Page, error) {
stmt, err = DB.Prepare("SELECT p.id, p.hostname, p.path FROM pages p WHERE p.hostname = ? AND p.path = ? LIMIT 1")
defer stmt.Close()
err = stmt.QueryRow(hostname, path).Scan(&p.ID, &p.Hostname, &p.Path)
return &p, err
}
// SavePage ...
func SavePage(p *models.Page) error {
// prepare statement for inserting data
stmt, err = DB.Prepare(`INSERT INTO pages(
hostname,
path,
title
) VALUES( ?, ?, ? )`)
defer stmt.Close()
if err != nil {
return err
}
result, err = stmt.Exec(p.Hostname, p.Path, p.Title)
if err != nil {
return err
}
p.ID, err = result.LastInsertId()
return err
}

38
datastore/pageviews.go Normal file
View File

@ -0,0 +1,38 @@
package datastore
import (
"github.com/dannyvankooten/ana/models"
)
//var pv models.Pageview
// SavePageview ...
func SavePageview(pv *models.Pageview) error {
// prepare statement for inserting data
stmt, err = DB.Prepare(`INSERT INTO pageviews (
page_id,
visitor_id,
referrer_url,
referrer_keyword,
timestamp
) VALUES( ?, ?, ?, ?, ? )`)
defer stmt.Close()
if err != nil {
return err
}
result, err = stmt.Exec(
pv.PageID,
pv.VisitorID,
pv.ReferrerUrl,
pv.ReferrerKeyword,
pv.Timestamp,
)
if err != nil {
return err
}
pv.ID, err = result.LastInsertId()
return err
}

49
datastore/seed.go
View File

@ -1,6 +1,8 @@
package datastore
import (
"crypto/md5"
"encoding/hex"
"fmt"
"math/rand"
"time"
@ -101,23 +103,28 @@ func Seed(n int) {
// print a dot as progress indicator
fmt.Print(".")
date := randomDateBeforeNow()
ipAddress := randomdata.IpV4Address()
browserName := randSliceElement(browserNames)
browserVersion := "54.0"
deviceOS := "Linux"
// create or find visitor
visitor := models.Visitor{
IpAddress: randomdata.IpV4Address(),
DeviceOS: "Linux",
BrowserName: randSliceElement(browserNames),
BrowserVersion: "54.0",
BrowserLanguage: randSliceElement(browserLanguages),
ScreenResolution: randSliceElement(screenResolutions),
Country: randomdata.Country(randomdata.TwoCharCountry),
}
dummyUserAgent := visitor.BrowserName + visitor.BrowserVersion + visitor.DeviceOS
visitor.Key = visitor.GenerateKey(date.Format("2006-01-02"), visitor.IpAddress, dummyUserAgent)
dummyUserAgent := browserName + browserVersion + deviceOS
visitorKey := generateVisitorKey(date.Format("2006-01-02"), ipAddress, dummyUserAgent)
err := stmtVisitor.QueryRow(visitor.Key).Scan(&visitor.ID)
if err != nil {
visitor.Save(DB)
var visitor *models.Visitor
visitor, err = GetVisitorByKey(visitorKey)
if visitor == nil {
// create or find visitor
visitor := models.Visitor{
IpAddress: ipAddress,
DeviceOS: deviceOS,
BrowserName: browserName,
BrowserVersion: browserVersion,
BrowserLanguage: randSliceElement(browserLanguages),
ScreenResolution: randSliceElement(screenResolutions),
Country: randomdata.Country(randomdata.TwoCharCountry),
}
err = SaveVisitor(&visitor)
}
// generate random timestamp
@ -132,11 +139,11 @@ func Seed(n int) {
DB.Exec("START TRANSACTION")
// insert between 1-4 pageviews for this visitor
for j := 0; j <= randInt(1, 4); j++ {
// insert between 1-6 pageviews for this visitor
for j := 0; j <= randInt(1, 6); j++ {
page := pages[randInt(0, len(pages))]
pv.PageID = page.ID
pv.Save(DB)
SavePageview(&pv)
}
DB.Exec("COMMIT")
@ -167,3 +174,9 @@ func randSliceElement(slice []string) string {
func randInt(min int, max int) int {
return min + rand.Intn(max-min)
}
// generateVisitorKey generates the "unique" visitor key from date, user agent + screen resolution
func generateVisitorKey(date string, ipAddress string, userAgent string) string {
byteKey := md5.Sum([]byte(date + ipAddress + userAgent))
return hex.EncodeToString(byteKey[:])
}

12
datastore/store.go
View File

@ -12,6 +12,9 @@ import (
// DB ...
var DB *sql.DB
var err error
var stmt *sql.Stmt
var result sql.Result
// Init creates a database connection pool
func Init() *sql.DB {
@ -30,12 +33,13 @@ func Init() *sql.DB {
// New creates a new database pool
func New(driver string, config string) *sql.DB {
db, err := sql.Open(driver, config)
var db *sql.DB
db, err = sql.Open(driver, config)
if err != nil {
log.Fatal(err)
}
if err := db.Ping(); err != nil {
if err = db.Ping(); err != nil {
log.Fatal(err)
}
@ -67,7 +71,9 @@ func runMigrations(driver string) {
}
migrate.SetTable("migrations")
n, err := migrate.Exec(DB, driver, migrations, migrate.Up)
var n int
n, err = migrate.Exec(DB, driver, migrations, migrate.Up)
if err != nil {
log.Fatal("Database migrations failed: ", err)

7
datastore/users.go
View File

@ -1,22 +1,21 @@
package datastore
import (
"database/sql"
"github.com/dannyvankooten/ana/models"
)
var err error
var stmt *sql.Stmt
var u models.User
// GetUser retrieves user from datastore by its ID
func GetUser(id int64) (*models.User, error) {
stmt, err = DB.Prepare("SELECT id, email FROM users WHERE id = ? LIMIT 1")
err = stmt.QueryRow(id).Scan(&u.ID, &u.Email)
return &u, err
}
// GetUserByEmail retrieves user from datastore by its email
func GetUserByEmail(email string) (*models.User, error) {
stmt, err = DB.Prepare("SELECT id, email, password FROM users WHERE email = ? LIMIT 1")
err := stmt.QueryRow(email).Scan(&u.ID, &u.Email, &u.HashedPassword)
err = stmt.QueryRow(email).Scan(&u.ID, &u.Email, &u.HashedPassword)
return &u, err
}

52
datastore/visitors.go Normal file
View File

@ -0,0 +1,52 @@
package datastore
import (
"github.com/dannyvankooten/ana/models"
)
var v models.Visitor
// GetVisitorByKey ...
func GetVisitorByKey(key string) (*models.Visitor, error) {
// query by unique visitor key
stmt, err = DB.Prepare("SELECT v.id FROM visitors v WHERE v.visitor_key = ? LIMIT 1")
defer stmt.Close()
err = stmt.QueryRow(key).Scan(&v.ID)
return &v, err
}
// SaveVisitor ...
func SaveVisitor(v *models.Visitor) error {
// prepare statement for inserting data
stmt, err = DB.Prepare(`INSERT INTO visitors (
visitor_key,
ip_address,
device_os,
browser_name,
browser_version,
browser_language,
screen_resolution,
country
) VALUES( ?, ?, ?, ?, ?, ?, ?, ? )`)
defer stmt.Close()
if err != nil {
return err
}
result, err = stmt.Exec(
v.Key,
v.IpAddress,
v.DeviceOS,
v.BrowserName,
v.BrowserVersion,
v.BrowserLanguage,
v.ScreenResolution,
v.Country,
)
if err != nil {
return err
}
v.ID, err = result.LastInsertId()
return err
}

33
models/pageview.go
View File

@ -1,9 +1,5 @@
package models
import (
"database/sql"
)
type Pageview struct {
ID int64
PageID int64
@ -12,32 +8,3 @@ type Pageview struct {
ReferrerUrl string
Timestamp string
}
func (pv *Pageview) Save(conn *sql.DB) error {
// prepare statement for inserting data
stmt, err := conn.Prepare(`INSERT INTO pageviews (
page_id,
visitor_id,
referrer_url,
referrer_keyword,
timestamp
) VALUES( ?, ?, ?, ?, ? )`)
if err != nil {
return err
}
defer stmt.Close()
result, err := stmt.Exec(
pv.PageID,
pv.VisitorID,
pv.ReferrerUrl,
pv.ReferrerKeyword,
pv.Timestamp,
)
if err != nil {
return err
}
pv.ID, err = result.LastInsertId()
return err
}

46
models/visitor.go
View File

@ -1,11 +1,5 @@
package models
import (
"crypto/md5"
"database/sql"
"encoding/hex"
)
type Visitor struct {
ID int64
Key string
@ -17,43 +11,3 @@ type Visitor struct {
IpAddress string
ScreenResolution string
}
func (v *Visitor) Save(conn *sql.DB) error {
// prepare statement for inserting data
stmt, err := conn.Prepare(`INSERT INTO visitors (
visitor_key,
ip_address,
device_os,
browser_name,
browser_version,
browser_language,
screen_resolution,
country
) VALUES( ?, ?, ?, ?, ?, ?, ?, ? )`)
if err != nil {
return err
}
defer stmt.Close()
result, err := stmt.Exec(
v.Key,
v.IpAddress,
v.DeviceOS,
v.BrowserName,
v.BrowserVersion,
v.BrowserLanguage,
v.ScreenResolution,
v.Country,
)
if err != nil {
return err
}
v.ID, err = result.LastInsertId()
return err
}
// GenerateKey generates the "unique" visitor key from date, user agent + screen resolution
func (v *Visitor) GenerateKey(date string, ipAddress string, userAgent string) string {
byteKey := md5.Sum([]byte(date + ipAddress + userAgent))
return hex.EncodeToString(byteKey[:])
}