add tables for pages & sites & normalize data into those tables

ROADMAP.md

@@ -9,12 +9,10 @@ This is a general draft document for thoughts and todo's, without any structure
 - Reference site URL when tracking.
 - Reference path & title when tracking (indexed by path, update title when changes)
 - Track referrals, use tables from aforementioned points.
-- CLI commands for CRUD user.
 - Bulk process tracking requests (Redis or in-memory?)
 - Allow sorting in table overviews.
 - Choose a OS license & settle on name.
 - Envelope API responses & perhaps return total in table overview?
-- Track canonical URL's.
 - Show referrals.
 - Geolocate unknown IP addresses periodically.
 - Mask last part of IP address.

api/collect.go

@@ -20,12 +20,11 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
   // prepare statement for inserting data
   stmt, err := db.Conn.Prepare(`INSERT INTO visits(
     ip_address,
-    path,
+    page_id,
     referrer_url,
     browser_language,
     browser_name,
     browser_version,
-    device_os,
     screen_resolution
     ) VALUES( ?, ?, ?, ?, ?, ?, ?, ? )`)
   if err != nil {
@@ -41,9 +40,10 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
     ipAddress = headerForwardedFor
   }
 
+  // TODO: Query Path
+
   q := r.URL.Query()
   visit := models.Visit{
-    Path: q.Get("p"),
     IpAddress: ipAddress,
     ReferrerUrl: q.Get("r"),
     BrowserLanguage: q.Get("l"),
@@ -53,17 +53,12 @@ func CollectHandler(w http.ResponseWriter, r *http.Request) {
   // add browser details
   visit.BrowserName, visit.BrowserVersion = ua.Browser()
 
-  // add device details
-  visit.DeviceOS = ua.OS()
-
   _, err = stmt.Exec(
     visit.IpAddress,
-    visit.Path,
     visit.ReferrerUrl,
     visit.BrowserLanguage,
     visit.BrowserName,
     visit.BrowserVersion,
-    visit.DeviceOS,
     visit.ScreenResolution,
   )
   if err != nil {

api/visits.go

@@ -15,9 +15,7 @@ var GetVisitsHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Requ
     id,
     COALESCE(browser_name, '') AS browser_name,
     COALESCE(browser_language, '') AS browser_language,
-    COALESCE(device_os, '') AS device_os,
     ip_address,
-    path,
     COALESCE(screen_resolution, '') AS screen_resolution,
     timestamp
     FROM visits
@@ -35,7 +33,7 @@ var GetVisitsHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Requ
   defer rows.Close()
   for rows.Next() {
     var v models.Visit
-    err = rows.Scan(&v.ID, &v.BrowserName, &v.BrowserLanguage, &v.DeviceOS, &v.IpAddress, &v.Path, &v.ScreenResolution, &v.Timestamp);
+    err = rows.Scan(&v.ID, &v.BrowserName, &v.BrowserLanguage, &v.IpAddress, &v.ScreenResolution, &v.Timestamp);
     checkError(err)
     results = append(results, v)
   }

commands/users.go

@@ -2,6 +2,7 @@ package commands
 
 import(
   "github.com/dannyvankooten/ana/db"
+  "github.com/dannyvankooten/ana/models"
   "golang.org/x/crypto/bcrypt"
   "log"
 )
@@ -11,11 +12,14 @@ func createUser() {
     log.Fatal("Please supply -email and -password values")
   }
 
-  stmt2, _ := db.Conn.Prepare("INSERT INTO users(email, password) VALUES(?, ?)")
   hash, _ := bcrypt.GenerateFromPassword([]byte(passwordArg), 10)
-  stmt2.Exec(emailArg, hash)
+  user := models.User{
+    Email: emailArg,
+    Password: string(hash),
+  }
+  user.Save(db.Conn)
 
-    log.Printf("User %s created", emailArg)
+  log.Printf("User %s #%d created", emailArg, user.ID)
 }
 
 func deleteUser() {

db/migrations/0001_initial_tables.down.sql

@@ -1 +1,4 @@
-DROP TABLE visits;
+DROP TABLE IF EXISTS visits;
+DROP TABLE IF EXISTS pages;
+DROP TABLE IF EXISTS sites;
+DROP TABLE IF EXISTS users;

db/migrations/0001_initial_tables.up.sql

@@ -1,19 +1,32 @@
-CREATE TABLE visits (
-  id INTEGER AUTO_INCREMENT PRIMARY KEY,
-  path TEXT NOT NULL,
-  ip_address VARCHAR(100) NOT NULL,
-  referrer_keyword VARCHAR(255) NULL,
-  referrer_type VARCHAR(255) NULL,
-  referrer_url TEXT NULL,
-  device_brand VARCHAR(100) NULL,
-  device_model VARCHAR(100) NULL,
-  device_type VARCHAR(100) NULL,
-  device_os VARCHAR(100) NULL,
-  browser_name VARCHAR(31) NULL,
-  browser_version VARCHAR(31) NULL,
-  browser_language VARCHAR(31) NULL,
-  screen_resolution VARCHAR(9) NULL,
-  visitor_returning TINYINT(1) DEFAULT 0,
-  country CHAR(3) NULL,
+CREATE TABLE visits(
+  `id` INTEGER UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+  `page_id` INTEGER UNSIGNED NOT NULL,
+  `ip_address` VARCHAR(100) NOT NULL,
+  `referrer_keyword` TEXT NULL,
+  `referrer_url` TEXT NULL,
+  `device_os` VARCHAR(100) NULL,
+  `browser_name` VARCHAR(31) NULL,
+  `browser_version` VARCHAR(31) NULL,
+  `browser_language` VARCHAR(31) NULL,
+  `screen_resolution` VARCHAR(9) NULL,
+  `country` CHAR(3) NULL,
   timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-)
+);
+
+CREATE TABLE sites(
+  `id` INTEGER UNSIGNED AUTO_INCREMENT PRIMARY KEY NOT NULL,
+  `url` VARCHAR(255) NOT NULL
+);
+
+CREATE TABLE pages(
+  `id` INTEGER UNSIGNED AUTO_INCREMENT PRIMARY KEY NOT NULL,
+  `site_id` INTEGER UNSIGNED NOT NULL,
+  `path` VARCHAR(255) NOT NULL,
+  `title` VARCHAR(255) NULL
+);
+
+CREATE TABLE users (
+  `id` INTEGER UNSIGNED AUTO_INCREMENT PRIMARY KEY NOT NULL,
+  `email` VARCHAR(255) NOT NULL,
+  `password` VARCHAR(255) NOT NULL
+);

db/migrations/0002_users_table.down.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS sessions;
-DROP TABLE IF EXISTS users;

db/migrations/0002_users_table.up.sql

@@ -1,14 +0,0 @@
-CREATE TABLE users (
-  `id` INTEGER UNSIGNED AUTO_INCREMENT PRIMARY KEY NOT NULL,
-  `email` VARCHAR(255) NOT NULL,
-  `password` VARCHAR(255) NOT NULL
-);
-
-CREATE TABLE sessions (
-  `id` INTEGER UNSIGNED AUTO_INCREMENT PRIMARY KEY NOT NULL,
-  `user_id` INTEGER UNSIGNED NOT NULL,
-  `key` VARCHAR(255) NOT NULL,
-  `time_login` TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
-  `time_last_seen` TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
-  INDEX `index_session_key` (`key` ASC)
-);

db/seed.go

@@ -33,14 +33,6 @@ var months = []time.Month {
   time.December,
 }
 
-var paths = []string {
-  "/",
-  "/", // we need this to weigh more.
-  "/contact",
-  "/about",
-  "/checkout",
-}
-
 var browserLanguages = []string {
   "en-US",
   "en-US",
@@ -57,27 +49,82 @@ var screenResolutions = []string {
   "360x640",
 }
 
+func seedSite() models.Site {
+  // get first site or create one
+  var site models.Site
+  Conn.QueryRow("SELECT url FROM sites LIMIT 1").Scan(&site.Url)
+
+  if site.Url == "" {
+    site.Url = "http://local.wordpress.dev/"
+    site.Save(Conn)
+  }
+
+  return site
+}
+
+func seedPages(site models.Site) []models.Page {
+  var pages = make([]models.Page, 0)
+
+  homepage := models.Page{
+    SiteID: site.ID,
+    Path: "/",
+    Title: "Homepage",
+  }
+  homepage.Save(Conn)
+
+  contactPage := models.Page{
+    SiteID: site.ID,
+    Path: "/contact/",
+    Title: "Contact",
+  }
+  contactPage.Save(Conn)
+
+  aboutPage := models.Page{
+    SiteID: site.ID,
+    Path: "/about/",
+    Title: "About Me",
+  }
+  aboutPage.Save(Conn)
+
+  portfolioPage := models.Page{
+    SiteID: site.ID,
+    Path: "/portfolio/",
+    Title: "Portfolio",
+  }
+  portfolioPage.Save(Conn)
+
+  pages = append(pages, homepage)
+  pages = append(pages, homepage)
+  pages = append(pages, contactPage)
+  pages = append(pages, aboutPage)
+  pages = append(pages, portfolioPage)
+  return pages
+}
+
 func Seed(n int) {
 
+  site := seedSite()
+  pages := seedPages(site)
+
   // prepare statement for inserting data
   stmt, err := Conn.Prepare(`INSERT INTO visits(
+    page_id,
     browser_language,
     browser_name,
     browser_version,
     country,
-    device_os,
     ip_address,
-    path,
     referrer_url,
     screen_resolution,
     timestamp
-    ) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )`)
+    ) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ? )`)
   if err != nil {
       log.Fatal(err)
   }
   defer stmt.Close()
 
   // insert X random hits
+  log.Printf("Inserting %d visits", n)
   for i := 0; i < n; i++ {
 
     // print a dot as progress indicator
@@ -89,7 +136,6 @@ func Seed(n int) {
 
     visit := models.Visit{
       IpAddress: randomdata.IpV4Address(),
-      DeviceOS: "Linux x86_64",
       BrowserName: randSliceElement(browserNames),
       BrowserVersion: "54.0.2840.100",
       BrowserLanguage: randSliceElement(browserLanguages),
@@ -101,16 +147,16 @@ func Seed(n int) {
 
     // insert between 1-4 pageviews for this visitor
     for j := 0; j < randInt(1, 4); j++ {
-      visit.Path = randSliceElement(paths)
+      page := pages[randInt(0, len(pages))]
+      visit.PageID = page.ID
 
       _, err = stmt.Exec(
+        visit.PageID,
         visit.BrowserLanguage,
         visit.BrowserName,
         visit.BrowserVersion,
         visit.Country,
-        visit.DeviceOS,
         visit.IpAddress,
-        visit.Path,
         visit.ReferrerUrl,
         visit.ScreenResolution,
         visit.Timestamp,

do

@@ -5,7 +5,8 @@ bin() {
 }
 
 migrate() {
-	env $(cat .env | xargs) | $GOPATH/bin/migrate -url mysql://$ANA_DATABASE_USER:$ANA_DATABASE_PASSWORD@$ANA_DATABSE_HOST/$ANA_DATABASE_NAME -path ./db/migrations $1 $2 $3
+	export $(cat .env | xargs)
+	$GOPATH/bin/migrate -url mysql://$ANA_DATABASE_USER:$ANA_DATABASE_PASSWORD@$ANA_DATABSE_HOST/$ANA_DATABASE_NAME -path ./db/migrations $1 $2 $3
 }
 
 # call first argument

models/page.go

@@ -0,0 +1,30 @@
+package models
+
+import(
+  "database/sql"
+)
+
+type Page struct {
+  ID int64
+  SiteID int64
+  Path string
+  Title string
+}
+
+func (p *Page) Save(conn *sql.DB) error {
+    // prepare statement for inserting data
+    stmt, err := conn.Prepare(`INSERT INTO pages(
+      site_id,
+      path,
+      title
+      ) VALUES( ?, ?, ? )`)
+    if err != nil {
+        return err
+    }
+    defer stmt.Close()
+
+    result, err := stmt.Exec(p.SiteID, p.Path, p.Title)
+    p.ID, _ = result.LastInsertId()
+
+    return err
+  }

models/site.go

@@ -0,0 +1,26 @@
+package models
+
+import (
+  "database/sql"
+)
+
+type Site struct {
+  ID int64
+  Url string
+}
+
+func (s *Site) Save(conn *sql.DB) error {
+    // prepare statement for inserting data
+    stmt, err := conn.Prepare(`INSERT INTO sites(
+        url
+      ) VALUES(?)`)
+    if err != nil {
+        return err
+    }
+    defer stmt.Close()
+
+    result, err := stmt.Exec(s.Url)
+    s.ID, _ = result.LastInsertId()
+
+    return err
+}

models/user.go

@@ -1,6 +1,28 @@
 package models
 
+import (
+  "database/sql"
+)
+
 type User struct {
-  ID int
+  ID int64
   Email string
+  Password string `json:"-"`
+}
+
+func (u *User) Save(conn *sql.DB) error {
+  // prepare statement for inserting data
+  stmt, err := conn.Prepare(`INSERT INTO users(
+    email,
+    password
+    ) VALUES(?, ?)`)
+  if err != nil {
+      return err
+  }
+  defer stmt.Close()
+
+  result, err := stmt.Exec(u.Email, u.Password)
+  u.ID, _ = result.LastInsertId()
+
+  return err
 }

models/visit.go

@@ -1,21 +1,15 @@
 package models
 
 type Visit struct {
-  ID int
-  Path string
+  ID int64
+  PageID int64
   IpAddress string
   ReferrerKeyword string
-  ReferrerType string
   ReferrerUrl string
-  DeviceBrand string
-  DeviceModel string
-  DeviceType string
-  DeviceOS string
   BrowserName string
   BrowserVersion string
   BrowserLanguage string
   ScreenResolution string
-  VisitorReturning bool
   Country string
   Timestamp string
 }