diff --git a/cmd/fathom/config.go b/cmd/fathom/config.go index 25323f7..096676c 100644 --- a/cmd/fathom/config.go +++ b/cmd/fathom/config.go @@ -2,16 +2,15 @@ package main import ( "math/rand" - "os" "github.com/joho/godotenv" "github.com/kelseyhightower/envconfig" log "github.com/sirupsen/logrus" - "github.com/usefathom/fathom/pkg/datastore" + "github.com/usefathom/fathom/pkg/datastore/sqlstore" ) type Config struct { - Database *datastore.Config + Database *sqlstore.Config Secret string } @@ -37,10 +36,9 @@ func parseConfig(file string) *Config { cfg.Database.Driver = "sqlite3" } - // if secret key is empty, use a randomly generated one to ease first-time installation + // if secret key is empty, use a randomly generated one if cfg.Secret == "" { cfg.Secret = randomString(40) - os.Setenv("FATHOM_SECRET", cfg.Secret) } return &cfg diff --git a/cmd/fathom/fathom.go b/cmd/fathom/fathom.go index 7e2cc5e..1f8a877 100644 --- a/cmd/fathom/fathom.go +++ b/cmd/fathom/fathom.go @@ -4,12 +4,11 @@ import ( "log" "os" - "github.com/jmoiron/sqlx" "github.com/urfave/cli" "github.com/usefathom/fathom/pkg/datastore" ) -var db *sqlx.DB +var db datastore.Datastore var config *Config func main() { @@ -64,7 +63,8 @@ func main() { func before(c *cli.Context) error { config = parseConfig(c.String("config")) - db = datastore.Init(config.Database) + db = datastore.New(config.Database) + return nil } diff --git a/cmd/fathom/register.go b/cmd/fathom/register.go index ae19c60..23b3bdf 100644 --- a/cmd/fathom/register.go +++ b/cmd/fathom/register.go @@ -3,7 +3,6 @@ package main import ( log "github.com/sirupsen/logrus" "github.com/urfave/cli" - "github.com/usefathom/fathom/pkg/datastore" "github.com/usefathom/fathom/pkg/models" "golang.org/x/crypto/bcrypt" ) @@ -19,7 +18,7 @@ func register(c *cli.Context) error { Email: c.String("email"), Password: string(hash), } - err := datastore.SaveUser(user) + err := db.SaveUser(user) if err != nil { log.Errorf("error creating user: %s", err) diff --git a/cmd/fathom/server.go b/cmd/fathom/server.go index 2a90f86..6f580c4 100644 --- a/cmd/fathom/server.go +++ b/cmd/fathom/server.go @@ -13,7 +13,8 @@ import ( func server(c *cli.Context) error { var h http.Handler - h = api.Routes() + a := api.New(db, config.Secret) + h = a.Routes() // set debug log level if --debug was passed if c.Bool("debug") { diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go index 29f6509..8e72d2a 100644 --- a/pkg/aggregator/aggregator.go +++ b/pkg/aggregator/aggregator.go @@ -1,17 +1,25 @@ package aggregator import ( - "time" - "github.com/usefathom/fathom/pkg/datastore" "github.com/usefathom/fathom/pkg/models" log "github.com/sirupsen/logrus" ) -func Run() { +type aggregator struct { + database datastore.Datastore +} + +func New(db datastore.Datastore) *aggregator { + return &aggregator{ + database: db, + } +} + +func (agg *aggregator) Run() { // Get unprocessed pageviews - pageviews, err := datastore.GetProcessablePageviews() + pageviews, err := agg.database.GetProcessablePageviews() if err != nil && err != datastore.ErrNoResults { log.Error(err) return @@ -22,43 +30,43 @@ func Run() { return } - results := Process(pageviews) + results := agg.Process(pageviews) // update stats for _, site := range results.Sites { - err = datastore.UpdateSiteStats(site) + err = agg.database.UpdateSiteStats(site) if err != nil { log.Error(err) } } for _, pageStats := range results.Pages { - err = datastore.UpdatePageStats(pageStats) + err = agg.database.UpdatePageStats(pageStats) if err != nil { log.Error(err) } } for _, referrerStats := range results.Referrers { - err = datastore.UpdateReferrerStats(referrerStats) + err = agg.database.UpdateReferrerStats(referrerStats) if err != nil { log.Error(err) } } // finally, remove pageviews that we just processed - err = datastore.DeletePageviews(pageviews) + err = agg.database.DeletePageviews(pageviews) if err != nil { log.Error(err) } } -func Process(pageviews []*models.Pageview) *Results { +func (agg *aggregator) Process(pageviews []*models.Pageview) *Results { log.Debugf("processing %d pageviews", len(pageviews)) results := NewResults() for _, p := range pageviews { - site, err := results.GetSiteStats(p.Timestamp) + site, err := agg.getSiteStats(results, p.Timestamp) if err != nil { log.Error(err) continue @@ -85,7 +93,7 @@ func Process(pageviews []*models.Pageview) *Results { } } - pageStats, err := results.GetPageStats(p.Timestamp, p.Hostname, p.Pathname) + pageStats, err := agg.getPageStats(results, p.Timestamp, p.Hostname, p.Pathname) if err != nil { log.Error(err) continue @@ -112,7 +120,7 @@ func Process(pageviews []*models.Pageview) *Results { // referrer stats if p.Referrer != "" { - referrerStats, err := results.GetReferrerStats(p.Timestamp, p.Referrer) + referrerStats, err := agg.getReferrerStats(results, p.Timestamp, p.Referrer) if err != nil { log.Error(err) continue @@ -140,57 +148,3 @@ func Process(pageviews []*models.Pageview) *Results { return results } - -func getSiteStats(t time.Time) (*models.SiteStats, error) { - stats, err := datastore.GetSiteStats(t) - if err != nil && err != datastore.ErrNoResults { - return nil, err - } - - if stats != nil { - return stats, nil - } - - stats = &models.SiteStats{ - Date: t, - } - err = datastore.InsertSiteStats(stats) - return stats, err -} - -func getPageStats(date time.Time, hostname string, pathname string) (*models.PageStats, error) { - stats, err := datastore.GetPageStats(date, hostname, pathname) - if err != nil && err != datastore.ErrNoResults { - return nil, err - } - - if stats != nil { - return stats, nil - } - - stats = &models.PageStats{ - Hostname: hostname, - Pathname: pathname, - Date: date, - } - err = datastore.InsertPageStats(stats) - return stats, err -} - -func getReferrerStats(date time.Time, url string) (*models.ReferrerStats, error) { - stats, err := datastore.GetReferrerStats(date, url) - if err != nil && err != datastore.ErrNoResults { - return nil, err - } - - if stats != nil { - return stats, nil - } - - stats = &models.ReferrerStats{ - URL: url, - Date: date, - } - err = datastore.InsertReferrerStats(stats) - return stats, err -} diff --git a/pkg/aggregator/result.go b/pkg/aggregator/result.go index 39a1ec8..376fc84 100644 --- a/pkg/aggregator/result.go +++ b/pkg/aggregator/result.go @@ -1,8 +1,6 @@ package aggregator import ( - "time" - "github.com/usefathom/fathom/pkg/models" ) @@ -19,52 +17,3 @@ func NewResults() *Results { Referrers: map[string]*models.ReferrerStats{}, } } - -func (r *Results) GetSiteStats(t time.Time) (*models.SiteStats, error) { - var stats *models.SiteStats - var ok bool - var err error - - date := t.Format("2006-01-02") - if stats, ok = r.Sites[date]; !ok { - stats, err = getSiteStats(t) - if err != nil { - return nil, err - } - r.Sites[date] = stats - } - return stats, nil -} - -func (r *Results) GetPageStats(t time.Time, hostname string, pathname string) (*models.PageStats, error) { - var stats *models.PageStats - var ok bool - var err error - - date := t.Format("2006-01-02") - if stats, ok = r.Pages[date+hostname+pathname]; !ok { - stats, err = getPageStats(t, hostname, pathname) - if err != nil { - return nil, err - } - r.Pages[date+hostname+pathname] = stats - } - return stats, nil -} - -func (r *Results) GetReferrerStats(t time.Time, referrer string) (*models.ReferrerStats, error) { - var stats *models.ReferrerStats - var ok bool - var err error - - date := t.Format("2006-01-02") - if stats, ok = r.Referrers[date+referrer]; !ok { - stats, err = getReferrerStats(t, referrer) - if err != nil { - return nil, err - } - r.Referrers[date+referrer] = stats - } - - return stats, nil -} diff --git a/pkg/aggregator/store.go b/pkg/aggregator/store.go new file mode 100644 index 0000000..6c4742e --- /dev/null +++ b/pkg/aggregator/store.go @@ -0,0 +1,93 @@ +package aggregator + +import ( + "time" + + "github.com/usefathom/fathom/pkg/datastore" + "github.com/usefathom/fathom/pkg/models" +) + +func (agg *aggregator) getSiteStats(r *Results, t time.Time) (*models.SiteStats, error) { + // get from map + date := t.Format("2006-01-02") + if stats, ok := r.Sites[date]; ok { + return stats, nil + + } + + // get from db + stats, err := agg.database.GetSiteStats(t) + if err != nil && err != datastore.ErrNoResults { + return nil, err + } + + // create in db + if stats == nil { + stats = &models.SiteStats{ + Date: t, + } + + err = agg.database.InsertSiteStats(stats) + if err != nil { + return nil, err + } + } + + r.Sites[date] = stats + return stats, nil +} + +func (agg *aggregator) getPageStats(r *Results, t time.Time, hostname string, pathname string) (*models.PageStats, error) { + date := t.Format("2006-01-02") + if stats, ok := r.Pages[date+hostname+pathname]; ok { + return stats, nil + } + + stats, err := agg.database.GetPageStats(t, hostname, pathname) + if err != nil && err != datastore.ErrNoResults { + return nil, err + } + + if stats == nil { + stats = &models.PageStats{ + Hostname: hostname, + Pathname: pathname, + Date: t, + } + err = agg.database.InsertPageStats(stats) + if err != nil { + return nil, err + } + } + + r.Pages[date+hostname+pathname] = stats + return stats, nil +} + +func (agg *aggregator) getReferrerStats(r *Results, t time.Time, url string) (*models.ReferrerStats, error) { + date := t.Format("2006-01-02") + if stats, ok := r.Referrers[date+url]; ok { + return stats, nil + } + + // get from db + stats, err := agg.database.GetReferrerStats(t, url) + if err != nil && err != datastore.ErrNoResults { + return nil, err + } + + // create in db + if stats == nil { + stats = &models.ReferrerStats{ + URL: url, + Date: t, + } + err = agg.database.InsertReferrerStats(stats) + if err != nil { + return nil, err + } + } + + r.Referrers[date+url] = stats + return stats, nil +} diff --git a/pkg/api/api.go b/pkg/api/api.go new file mode 100644 index 0000000..8e9b110 --- /dev/null +++ b/pkg/api/api.go @@ -0,0 +1,19 @@ +package api + +import ( + "github.com/gorilla/sessions" + "github.com/usefathom/fathom/pkg/datastore" +) + +type API struct { + database datastore.Datastore + sessions sessions.Store +} + +// New instantiates a new API object +func New(db datastore.Datastore, secret string) *API { + return &API{ + database: db, + sessions: sessions.NewCookieStore([]byte(secret)), + } +} diff --git a/pkg/api/auth.go b/pkg/api/auth.go index cf42e00..8d4c3a2 100644 --- a/pkg/api/auth.go +++ b/pkg/api/auth.go @@ -4,9 +4,7 @@ import ( "context" "encoding/json" "net/http" - "os" - "github.com/gorilla/sessions" "github.com/usefathom/fathom/pkg/datastore" "golang.org/x/crypto/bcrypt" ) @@ -22,17 +20,14 @@ type login struct { Password string `json:"password"` } -var store = sessions.NewCookieStore([]byte(os.Getenv("FATHOM_SECRET"))) - // URL: POST /api/session -var LoginHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - +func (api *API) LoginHandler(w http.ResponseWriter, r *http.Request) error { // check login creds var l login json.NewDecoder(r.Body).Decode(&l) // find user with given email - u, err := datastore.GetUserByEmail(l.Email) + u, err := api.database.GetUserByEmail(l.Email) if err != nil && err != datastore.ErrNoResults { return err } @@ -43,7 +38,7 @@ var LoginHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) erro return respond(w, envelope{Error: "invalid_credentials"}) } - session, _ := store.Get(r, "auth") + session, _ := api.sessions.Get(r, "auth") session.Values["user_id"] = u.ID err = session.Save(r, w) if err != nil { @@ -51,11 +46,11 @@ var LoginHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) erro } return respond(w, envelope{Data: true}) -}) +} // URL: DELETE /api/session -var LogoutHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - session, _ := store.Get(r, "auth") +func (api *API) LogoutHandler(w http.ResponseWriter, r *http.Request) error { + session, _ := api.sessions.Get(r, "auth") if !session.IsNew { session.Options.MaxAge = -1 err := session.Save(r, w) @@ -65,12 +60,12 @@ var LogoutHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) err } return respond(w, envelope{Data: true}) -}) +} /* middleware */ -func Authorize(next http.Handler) http.Handler { +func (api *API) Authorize(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - session, err := store.Get(r, "auth") + session, err := api.sessions.Get(r, "auth") if err != nil { return } @@ -83,7 +78,7 @@ func Authorize(next http.Handler) http.Handler { } // find user - u, err := datastore.GetUser(userID.(int64)) + u, err := api.database.GetUser(userID.(int64)) if err != nil { w.WriteHeader(http.StatusUnauthorized) return diff --git a/pkg/api/collect.go b/pkg/api/collect.go index 8237e07..914fa4f 100644 --- a/pkg/api/collect.go +++ b/pkg/api/collect.go @@ -27,9 +27,8 @@ func ShouldCollect(r *http.Request) bool { return true } -/* middleware */ -func NewCollectHandler() http.Handler { - go aggregate() +func (api *API) NewCollectHandler() http.Handler { + go aggregate(api.database) return HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { if !ShouldCollect(r) { @@ -60,7 +59,7 @@ func NewCollectHandler() http.Handler { // find previous pageview by same visitor if !pageview.IsNewSession { - previousPageview, err := datastore.GetMostRecentPageviewBySessionID(pageview.SessionID) + previousPageview, err := api.database.GetMostRecentPageviewBySessionID(pageview.SessionID) if err != nil && err != datastore.ErrNoResults { return err } @@ -69,7 +68,7 @@ func NewCollectHandler() http.Handler { if previousPageview != nil && previousPageview.Timestamp.After(now.Add(-30*time.Minute)) { previousPageview.Duration = (now.Unix() - previousPageview.Timestamp.Unix()) previousPageview.IsBounce = false - err := datastore.UpdatePageview(previousPageview) + err := api.database.UpdatePageview(previousPageview) if err != nil { return err } @@ -77,7 +76,7 @@ func NewCollectHandler() http.Handler { } // save new pageview - err = datastore.SavePageview(pageview) + err = api.database.SavePageview(pageview) if err != nil { return err } @@ -97,14 +96,16 @@ func NewCollectHandler() http.Handler { } // runs the aggregate func every minute -func aggregate() { - aggregator.Run() +func aggregate(db datastore.Datastore) { + agg := aggregator.New(db) + agg.Run() + timeout := 1 * time.Minute for { select { case <-time.After(timeout): - aggregator.Run() + agg.Run() } } } diff --git a/pkg/api/page_stats.go b/pkg/api/page_stats.go index b26a0a2..23cf929 100644 --- a/pkg/api/page_stats.go +++ b/pkg/api/page_stats.go @@ -2,26 +2,23 @@ package api import ( "net/http" - - "github.com/usefathom/fathom/pkg/datastore" ) // URL: /api/stats/page -var GetPageStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { +func (api *API) GetPageStatsHandler(w http.ResponseWriter, r *http.Request) error { params := GetRequestParams(r) - result, err := datastore.GetAggregatedPageStats(params.StartDate, params.EndDate, params.Limit) + result, err := api.database.GetAggregatedPageStats(params.StartDate, params.EndDate, params.Limit) if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} -// URL: /api/stats/page/pageviews -var GetPageStatsPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { +func (api *API) GetPageStatsPageviewsHandler(w http.ResponseWriter, r *http.Request) error { params := GetRequestParams(r) - result, err := datastore.GetAggregatedPageStatsPageviews(params.StartDate, params.EndDate) + result, err := api.database.GetAggregatedPageStatsPageviews(params.StartDate, params.EndDate) if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} diff --git a/pkg/api/referrer_stats.go b/pkg/api/referrer_stats.go index 4e2431e..52b49fb 100644 --- a/pkg/api/referrer_stats.go +++ b/pkg/api/referrer_stats.go @@ -2,26 +2,24 @@ package api import ( "net/http" - - "github.com/usefathom/fathom/pkg/datastore" ) // URL: /api/stats/referrer -var GetReferrerStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { +func (api *API) GetReferrerStatsHandler(w http.ResponseWriter, r *http.Request) error { params := GetRequestParams(r) - result, err := datastore.GetAggregatedReferrerStats(params.StartDate, params.EndDate, params.Limit) + result, err := api.database.GetAggregatedReferrerStats(params.StartDate, params.EndDate, params.Limit) if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} // URL: /api/stats/referrer/pageviews -var GetReferrerStatsPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { +func (api *API) GetReferrerStatsPageviewsHandler(w http.ResponseWriter, r *http.Request) error { params := GetRequestParams(r) - result, err := datastore.GetAggregatedReferrerStatsPageviews(params.StartDate, params.EndDate) + result, err := api.database.GetAggregatedReferrerStatsPageviews(params.StartDate, params.EndDate) if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 1701e88..c01ddce 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -6,24 +6,24 @@ import ( "net/http" ) -func Routes() *mux.Router { +func (api *API) Routes() *mux.Router { // register routes r := mux.NewRouter() - r.Handle("/collect", NewCollectHandler()).Methods(http.MethodGet) - r.Handle("/api/session", LoginHandler).Methods(http.MethodPost) - r.Handle("/api/session", LogoutHandler).Methods(http.MethodDelete) + r.Handle("/collect", api.NewCollectHandler()).Methods(http.MethodGet) + r.Handle("/api/session", HandlerFunc(api.LoginHandler)).Methods(http.MethodPost) + r.Handle("/api/session", HandlerFunc(api.LogoutHandler)).Methods(http.MethodDelete) - r.Handle("/api/stats/site/pageviews", Authorize(GetSiteStatsPageviewsHandler)).Methods(http.MethodGet) - r.Handle("/api/stats/site/visitors", Authorize(GetSiteStatsVisitorsHandler)).Methods(http.MethodGet) - r.Handle("/api/stats/site/duration", Authorize(GetSiteStatsDurationHandler)).Methods(http.MethodGet) - r.Handle("/api/stats/site/bounces", Authorize(GetSiteStatsBouncesHandler)).Methods(http.MethodGet) - r.Handle("/api/stats/site/realtime", Authorize(GetSiteStatsRealtimeHandler)).Methods(http.MethodGet) + r.Handle("/api/stats/site/pageviews", api.Authorize(HandlerFunc(api.GetSiteStatsPageviewsHandler))).Methods(http.MethodGet) + r.Handle("/api/stats/site/visitors", api.Authorize(HandlerFunc(api.GetSiteStatsVisitorsHandler))).Methods(http.MethodGet) + r.Handle("/api/stats/site/duration", api.Authorize(HandlerFunc(api.GetSiteStatsDurationHandler))).Methods(http.MethodGet) + r.Handle("/api/stats/site/bounces", api.Authorize(HandlerFunc(api.GetSiteStatsBouncesHandler))).Methods(http.MethodGet) + r.Handle("/api/stats/site/realtime", api.Authorize(HandlerFunc(api.GetSiteStatsRealtimeHandler))).Methods(http.MethodGet) - r.Handle("/api/stats/pages", Authorize(GetPageStatsHandler)).Methods(http.MethodGet) - r.Handle("/api/stats/pages/pageviews", Authorize(GetPageStatsPageviewsHandler)).Methods(http.MethodGet) + r.Handle("/api/stats/pages", api.Authorize(HandlerFunc(api.GetPageStatsHandler))).Methods(http.MethodGet) + r.Handle("/api/stats/pages/pageviews", api.Authorize(HandlerFunc(api.GetPageStatsPageviewsHandler))).Methods(http.MethodGet) - r.Handle("/api/stats/referrers", Authorize(GetReferrerStatsHandler)).Methods(http.MethodGet) - r.Handle("/api/stats/referrers/pageviews", Authorize(GetReferrerStatsPageviewsHandler)).Methods(http.MethodGet) + r.Handle("/api/stats/referrers", api.Authorize(HandlerFunc(api.GetReferrerStatsHandler))).Methods(http.MethodGet) + r.Handle("/api/stats/referrers/pageviews", api.Authorize(HandlerFunc(api.GetReferrerStatsPageviewsHandler))).Methods(http.MethodGet) // static assets & 404 handler box := packr.NewBox("./../../build") diff --git a/pkg/api/site_stats.go b/pkg/api/site_stats.go index 36daca5..9427b67 100644 --- a/pkg/api/site_stats.go +++ b/pkg/api/site_stats.go @@ -2,65 +2,53 @@ package api import ( "net/http" - - "github.com/usefathom/fathom/pkg/datastore" - "github.com/usefathom/fathom/pkg/models" ) -// URL: /api/stats/site -var GetSiteStatsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - // before, after := getRequestedPeriods(r) - // limit := getRequestedLimit(r) - - var results []*models.SiteStats - return respond(w, envelope{Data: results}) -}) - // URL: /api/stats/site/pageviews -var GetSiteStatsPageviewsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { +func (api *API) GetSiteStatsPageviewsHandler(w http.ResponseWriter, r *http.Request) error { params := GetRequestParams(r) - result, err := datastore.GetTotalSiteViews(params.StartDate, params.EndDate) + result, err := api.database.GetTotalSiteViews(params.StartDate, params.EndDate) if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} // URL: /api/stats/site/visitors -var GetSiteStatsVisitorsHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { +func (api *API) GetSiteStatsVisitorsHandler(w http.ResponseWriter, r *http.Request) error { params := GetRequestParams(r) - result, err := datastore.GetTotalSiteVisitors(params.StartDate, params.EndDate) + result, err := api.database.GetTotalSiteVisitors(params.StartDate, params.EndDate) if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} // URL: /api/stats/site/duration -var GetSiteStatsDurationHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { +func (api *API) GetSiteStatsDurationHandler(w http.ResponseWriter, r *http.Request) error { params := GetRequestParams(r) - result, err := datastore.GetAverageSiteDuration(params.StartDate, params.EndDate) + result, err := api.database.GetAverageSiteDuration(params.StartDate, params.EndDate) if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} // URL: /api/stats/site/bounces -var GetSiteStatsBouncesHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { +func (api *API) GetSiteStatsBouncesHandler(w http.ResponseWriter, r *http.Request) error { params := GetRequestParams(r) - result, err := datastore.GetAverageSiteBounceRate(params.StartDate, params.EndDate) + result, err := api.database.GetAverageSiteBounceRate(params.StartDate, params.EndDate) if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} // URL: /api/stats/site/realtime -var GetSiteStatsRealtimeHandler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error { - result, err := datastore.GetRealtimeVisitorCount() +func (api *API) GetSiteStatsRealtimeHandler(w http.ResponseWriter, r *http.Request) error { + result, err := api.database.GetRealtimeVisitorCount() if err != nil { return err } return respond(w, envelope{Data: result}) -}) +} diff --git a/pkg/datastore/datastore.go b/pkg/datastore/datastore.go index 581e13e..bc63539 100644 --- a/pkg/datastore/datastore.go +++ b/pkg/datastore/datastore.go @@ -1,53 +1,55 @@ package datastore import ( - "errors" + "time" - _ "github.com/go-sql-driver/mysql" // mysql driver - "github.com/gobuffalo/packr" - "github.com/jmoiron/sqlx" - _ "github.com/lib/pq" // postgresql driver - _ "github.com/mattn/go-sqlite3" //sqlite3 driver - migrate "github.com/rubenv/sql-migrate" - log "github.com/sirupsen/logrus" + "github.com/usefathom/fathom/pkg/datastore/sqlstore" + "github.com/usefathom/fathom/pkg/models" ) -var dbx *sqlx.DB +var ErrNoResults = sqlstore.ErrNoResults // ??? -// ErrNoResults is returned when a query yielded 0 results -var ErrNoResults = errors.New("datastore: query returned 0 results") +type Datastore interface { + // users + GetUser(int64) (*models.User, error) + GetUserByEmail(string) (*models.User, error) + SaveUser(*models.User) error -// Init creates a database connection pool (using sqlx) -func Init(c *Config) *sqlx.DB { - dbx = New(c) + // site stats + GetSiteStats(time.Time) (*models.SiteStats, error) + InsertSiteStats(*models.SiteStats) error + UpdateSiteStats(*models.SiteStats) error + GetTotalSiteViews(time.Time, time.Time) (int, error) + GetTotalSiteVisitors(time.Time, time.Time) (int, error) + GetTotalSiteSessions(time.Time, time.Time) (int, error) + GetAverageSiteDuration(time.Time, time.Time) (float64, error) + GetAverageSiteBounceRate(time.Time, time.Time) (float64, error) + GetRealtimeVisitorCount() (int, error) - // run migrations - runMigrations(c.Driver) + // pageviews + SavePageview(*models.Pageview) error + UpdatePageview(*models.Pageview) error + GetMostRecentPageviewBySessionID(string) (*models.Pageview, error) + GetProcessablePageviews() ([]*models.Pageview, error) + DeletePageviews([]*models.Pageview) error - return dbx + // page stats + GetPageStats(time.Time, string, string) (*models.PageStats, error) + InsertPageStats(*models.PageStats) error + UpdatePageStats(*models.PageStats) error + GetAggregatedPageStats(time.Time, time.Time, int) ([]*models.PageStats, error) + GetAggregatedPageStatsPageviews(time.Time, time.Time) (int, error) + + // referrer stats + GetReferrerStats(time.Time, string) (*models.ReferrerStats, error) + InsertReferrerStats(*models.ReferrerStats) error + UpdateReferrerStats(*models.ReferrerStats) error + GetAggregatedReferrerStats(time.Time, time.Time, int) ([]*models.ReferrerStats, error) + GetAggregatedReferrerStatsPageviews(time.Time, time.Time) (int, error) + + Close() } -// New creates a new database pool -func New(c *Config) *sqlx.DB { - dbx := sqlx.MustConnect(c.Driver, c.DSN()) - return dbx -} - -// TODO: Move to command (but still auto-run on boot). -func runMigrations(driver string) { - migrations := &migrate.PackrMigrationSource{ - Box: packr.NewBox("./migrations"), - Dir: "./" + driver, - } - migrate.SetTable("migrations") - - n, err := migrate.Exec(dbx.DB, driver, migrations, migrate.Up) - - if err != nil { - log.Fatal("database migrations failed: ", err) - } - - if n > 0 { - log.Infof("applied %d database migrations", n) - } +func New(c *sqlstore.Config) Datastore { + return sqlstore.New(c) } diff --git a/pkg/datastore/page_stats.go b/pkg/datastore/page_stats.go deleted file mode 100644 index d77ab96..0000000 --- a/pkg/datastore/page_stats.go +++ /dev/null @@ -1,43 +0,0 @@ -package datastore - -import ( - "database/sql" - "github.com/usefathom/fathom/pkg/models" - "time" -) - -func GetPageStats(date time.Time, hostname string, pathname string) (*models.PageStats, error) { - stats := &models.PageStats{} - query := dbx.Rebind(`SELECT * FROM daily_page_stats WHERE hostname = ? AND pathname = ? AND date = ? LIMIT 1`) - err := dbx.Get(stats, query, hostname, pathname, date.Format("2006-01-02")) - if err != nil && err == sql.ErrNoRows { - return nil, ErrNoResults - } - return stats, err -} - -func InsertPageStats(s *models.PageStats) error { - query := dbx.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`) - _, err := dbx.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) - return err -} - -func UpdatePageStats(s *models.PageStats) error { - query := dbx.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE hostname = ? AND pathname = ? AND date = ?`) - _, err := dbx.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) - return err -} - -func GetAggregatedPageStats(startDate time.Time, endDate time.Time, limit int) ([]*models.PageStats, error) { - var result []*models.PageStats - query := dbx.Rebind(`SELECT hostname, pathname, SUM(pageviews) AS pageviews, SUM(visitors) AS visitors, SUM(entries) AS entries, COALESCE(ROUND(SUM(entries*bounce_rate)/SUM(entries), 4), 0.00) AS bounce_rate, COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration FROM daily_page_stats WHERE date >= ? AND date <= ? GROUP BY hostname, pathname ORDER BY pageviews DESC LIMIT ?`) - err := dbx.Select(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit) - return result, err -} - -func GetAggregatedPageStatsPageviews(startDate time.Time, endDate time.Time) (int, error) { - var result int - query := dbx.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM daily_page_stats WHERE date >= ? AND date <= ?`) - err := dbx.Get(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) - return result, err -} diff --git a/pkg/datastore/pageviews.go b/pkg/datastore/pageviews.go deleted file mode 100644 index 24b199a..0000000 --- a/pkg/datastore/pageviews.go +++ /dev/null @@ -1,61 +0,0 @@ -package datastore - -import ( - "database/sql" - "strconv" - "strings" - "time" - - "github.com/usefathom/fathom/pkg/models" -) - -// SavePageview inserts a single pageview model into the connected database -func SavePageview(p *models.Pageview) error { - query := dbx.Rebind(`INSERT INTO pageviews(hostname, pathname, session_id, is_new_visitor, is_new_session, is_unique, is_bounce, referrer, duration, timestamp) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) - result, err := dbx.Exec(query, p.Hostname, p.Pathname, p.SessionID, p.IsNewVisitor, p.IsNewSession, p.IsUnique, p.IsBounce, p.Referrer, p.Duration, p.Timestamp) - if err != nil { - return err - } - - p.ID, _ = result.LastInsertId() - return nil -} - -func UpdatePageview(p *models.Pageview) error { - query := dbx.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`) - _, err := dbx.Exec(query, p.IsBounce, p.Duration, p.ID) - return err -} - -func GetMostRecentPageviewBySessionID(sessionID string) (*models.Pageview, error) { - result := &models.Pageview{} - query := dbx.Rebind(`SELECT * FROM pageviews WHERE session_id = ? ORDER BY id DESC LIMIT 1`) - err := dbx.Get(result, query, sessionID) - if err != nil { - if err == sql.ErrNoRows { - return nil, ErrNoResults - } - - return nil, err - } - - return result, nil -} - -func GetProcessablePageviews() ([]*models.Pageview, error) { - var results []*models.Pageview - thirtyMinsAgo := time.Now().Add(-30 * time.Minute) - query := dbx.Rebind(`SELECT * FROM pageviews WHERE ( duration > 0 AND is_bounce = 0 ) OR timestamp < ? LIMIT 500`) - err := dbx.Select(&results, query, thirtyMinsAgo) - return results, err -} - -func DeletePageviews(pageviews []*models.Pageview) error { - ids := []string{} - for _, p := range pageviews { - ids = append(ids, strconv.FormatInt(p.ID, 10)) - } - query := dbx.Rebind(`DELETE FROM pageviews WHERE id IN(` + strings.Join(ids, ",") + `)`) - _, err := dbx.Exec(query) - return err -} diff --git a/pkg/datastore/referrer_stats.go b/pkg/datastore/referrer_stats.go deleted file mode 100644 index f9e3768..0000000 --- a/pkg/datastore/referrer_stats.go +++ /dev/null @@ -1,44 +0,0 @@ -package datastore - -import ( - "database/sql" - "time" - - "github.com/usefathom/fathom/pkg/models" -) - -func GetReferrerStats(date time.Time, url string) (*models.ReferrerStats, error) { - stats := &models.ReferrerStats{} - query := dbx.Rebind(`SELECT * FROM daily_referrer_stats WHERE url = ? AND date = ? LIMIT 1`) - err := dbx.Get(stats, query, url, date.Format("2006-01-02")) - if err != nil && err == sql.ErrNoRows { - return nil, ErrNoResults - } - return stats, err -} - -func InsertReferrerStats(s *models.ReferrerStats) error { - query := dbx.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, url, date) VALUES(?, ?, ?, ?, ?, ?)`) - _, err := dbx.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02")) - return err -} - -func UpdateReferrerStats(s *models.ReferrerStats) error { - query := dbx.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE url = ? AND date = ?`) - _, err := dbx.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02")) - return err -} - -func GetAggregatedReferrerStats(startDate time.Time, endDate time.Time, limit int) ([]*models.ReferrerStats, error) { - var result []*models.ReferrerStats - query := dbx.Rebind(`SELECT url, SUM(visitors) AS visitors, SUM(pageviews) AS pageviews, COALESCE(ROUND(SUM(pageviews*bounce_rate)/SUM(pageviews), 4), 0.00) AS bounce_rate, COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration FROM daily_referrer_stats WHERE date >= ? AND date <= ? GROUP BY url ORDER BY pageviews DESC LIMIT ?`) - err := dbx.Select(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit) - return result, err -} - -func GetAggregatedReferrerStatsPageviews(startDate time.Time, endDate time.Time) (int, error) { - var result int - query := dbx.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM daily_referrer_stats WHERE date >= ? AND date <= ?`) - err := dbx.Get(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) - return result, err -} diff --git a/pkg/datastore/site_stats.go b/pkg/datastore/site_stats.go deleted file mode 100644 index b4b2b55..0000000 --- a/pkg/datastore/site_stats.go +++ /dev/null @@ -1,77 +0,0 @@ -package datastore - -import ( - "database/sql" - "github.com/usefathom/fathom/pkg/models" - "time" -) - -func GetSiteStats(date time.Time) (*models.SiteStats, error) { - stats := &models.SiteStats{} - query := dbx.Rebind(`SELECT * FROM daily_site_stats WHERE date = ? LIMIT 1`) - err := dbx.Get(stats, query, date.Format("2006-01-02")) - if err != nil && err == sql.ErrNoRows { - return nil, ErrNoResults - } - return stats, err -} - -func InsertSiteStats(s *models.SiteStats) error { - query := dbx.Rebind(`INSERT INTO daily_site_stats(visitors, sessions, pageviews, bounce_rate, avg_duration, date) VALUES(?, ?, ?, ?, ?, ?)`) - _, err := dbx.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02")) - return err -} - -func UpdateSiteStats(s *models.SiteStats) error { - query := dbx.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE date = ?`) - _, err := dbx.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02")) - return err -} - -func GetTotalSiteViews(startDate time.Time, endDate time.Time) (int, error) { - sql := `SELECT COALESCE(SUM(pageviews), 0) FROM daily_site_stats WHERE date >= ? AND date <= ?` - query := dbx.Rebind(sql) - var total int - err := dbx.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) - return total, err -} - -func GetTotalSiteVisitors(startDate time.Time, endDate time.Time) (int, error) { - sql := `SELECT COALESCE(SUM(visitors), 0) FROM daily_site_stats WHERE date >= ? AND date <= ?` - query := dbx.Rebind(sql) - var total int - err := dbx.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) - return total, err -} - -func GetTotalSiteSessions(startDate time.Time, endDate time.Time) (int, error) { - sql := `SELECT COALESCE(SUM(sessions), 0) FROM daily_site_stats WHERE date >= ? AND date <= ?` - query := dbx.Rebind(sql) - var total int - err := dbx.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) - return total, err -} - -func GetAverageSiteDuration(startDate time.Time, endDate time.Time) (float64, error) { - sql := `SELECT COALESCE(ROUND(SUM(pageviews*avg_duration)/SUM(pageviews), 4), 0.00) FROM daily_site_stats WHERE date >= ? AND date <= ?` - query := dbx.Rebind(sql) - var total float64 - err := dbx.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) - return total, err -} - -func GetAverageSiteBounceRate(startDate time.Time, endDate time.Time) (float64, error) { - sql := `SELECT COALESCE(ROUND(SUM(sessions*bounce_rate)/SUM(sessions), 4), 0.00) FROM daily_site_stats WHERE date >= ? AND date <= ?` - query := dbx.Rebind(sql) - var total float64 - err := dbx.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) - return total, err -} - -func GetRealtimeVisitorCount() (int, error) { - sql := `SELECT COUNT(DISTINCT(session_id)) FROM pageviews WHERE timestamp > ?` - query := dbx.Rebind(sql) - var total int - err := dbx.Get(&total, query, time.Now().Add(-5*time.Minute)) - return total, err -} diff --git a/pkg/datastore/config.go b/pkg/datastore/sqlstore/config.go similarity index 96% rename from pkg/datastore/config.go rename to pkg/datastore/sqlstore/config.go index f715cc5..888eecf 100644 --- a/pkg/datastore/config.go +++ b/pkg/datastore/sqlstore/config.go @@ -1,4 +1,4 @@ -package datastore +package sqlstore import "fmt" diff --git a/pkg/datastore/migrations/mysql/1_initial_tables.sql b/pkg/datastore/sqlstore/migrations/mysql/1_initial_tables.sql similarity index 100% rename from pkg/datastore/migrations/mysql/1_initial_tables.sql rename to pkg/datastore/sqlstore/migrations/mysql/1_initial_tables.sql diff --git a/pkg/datastore/migrations/postgres/1_initial_tables.sql b/pkg/datastore/sqlstore/migrations/postgres/1_initial_tables.sql similarity index 100% rename from pkg/datastore/migrations/postgres/1_initial_tables.sql rename to pkg/datastore/sqlstore/migrations/postgres/1_initial_tables.sql diff --git a/pkg/datastore/migrations/sqlite3/1_initial_tables.sql b/pkg/datastore/sqlstore/migrations/sqlite3/1_initial_tables.sql similarity index 100% rename from pkg/datastore/migrations/sqlite3/1_initial_tables.sql rename to pkg/datastore/sqlstore/migrations/sqlite3/1_initial_tables.sql diff --git a/pkg/datastore/sqlstore/page_stats.go b/pkg/datastore/sqlstore/page_stats.go new file mode 100644 index 0000000..84c1567 --- /dev/null +++ b/pkg/datastore/sqlstore/page_stats.go @@ -0,0 +1,43 @@ +package sqlstore + +import ( + "database/sql" + "github.com/usefathom/fathom/pkg/models" + "time" +) + +func (db *sqlstore) GetPageStats(date time.Time, hostname string, pathname string) (*models.PageStats, error) { + stats := &models.PageStats{} + query := db.Rebind(`SELECT * FROM daily_page_stats WHERE hostname = ? AND pathname = ? AND date = ? LIMIT 1`) + err := db.Get(stats, query, hostname, pathname, date.Format("2006-01-02")) + if err != nil && err == sql.ErrNoRows { + return nil, ErrNoResults + } + return stats, err +} + +func (db *sqlstore) InsertPageStats(s *models.PageStats) error { + query := db.Rebind(`INSERT INTO daily_page_stats(pageviews, visitors, entries, bounce_rate, avg_duration, hostname, pathname, date) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`) + _, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) + return err +} + +func (db *sqlstore) UpdatePageStats(s *models.PageStats) error { + query := db.Rebind(`UPDATE daily_page_stats SET pageviews = ?, visitors = ?, entries = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE hostname = ? AND pathname = ? AND date = ?`) + _, err := db.Exec(query, s.Pageviews, s.Visitors, s.Entries, s.BounceRate, s.AvgDuration, s.Hostname, s.Pathname, s.Date.Format("2006-01-02")) + return err +} + +func (db *sqlstore) GetAggregatedPageStats(startDate time.Time, endDate time.Time, limit int) ([]*models.PageStats, error) { + var result []*models.PageStats + query := db.Rebind(`SELECT hostname, pathname, SUM(pageviews) AS pageviews, SUM(visitors) AS visitors, SUM(entries) AS entries, COALESCE(ROUND(SUM(entries*bounce_rate)/SUM(entries), 4), 0.00) AS bounce_rate, COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration FROM daily_page_stats WHERE date >= ? AND date <= ? GROUP BY hostname, pathname ORDER BY pageviews DESC LIMIT ?`) + err := db.Select(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit) + return result, err +} + +func (db *sqlstore) GetAggregatedPageStatsPageviews(startDate time.Time, endDate time.Time) (int, error) { + var result int + query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM daily_page_stats WHERE date >= ? AND date <= ?`) + err := db.Get(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) + return result, err +} diff --git a/pkg/datastore/sqlstore/pageviews.go b/pkg/datastore/sqlstore/pageviews.go new file mode 100644 index 0000000..311eac8 --- /dev/null +++ b/pkg/datastore/sqlstore/pageviews.go @@ -0,0 +1,61 @@ +package sqlstore + +import ( + "database/sql" + "strconv" + "strings" + "time" + + "github.com/usefathom/fathom/pkg/models" +) + +// SavePageview inserts a single pageview model into the connected database +func (db *sqlstore) SavePageview(p *models.Pageview) error { + query := db.Rebind(`INSERT INTO pageviews(hostname, pathname, session_id, is_new_visitor, is_new_session, is_unique, is_bounce, referrer, duration, timestamp) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + result, err := db.Exec(query, p.Hostname, p.Pathname, p.SessionID, p.IsNewVisitor, p.IsNewSession, p.IsUnique, p.IsBounce, p.Referrer, p.Duration, p.Timestamp) + if err != nil { + return err + } + + p.ID, _ = result.LastInsertId() + return nil +} + +func (db *sqlstore) UpdatePageview(p *models.Pageview) error { + query := db.Rebind(`UPDATE pageviews SET is_bounce = ?, duration = ? WHERE id = ?`) + _, err := db.Exec(query, p.IsBounce, p.Duration, p.ID) + return err +} + +func (db *sqlstore) GetMostRecentPageviewBySessionID(sessionID string) (*models.Pageview, error) { + result := &models.Pageview{} + query := db.Rebind(`SELECT * FROM pageviews WHERE session_id = ? ORDER BY id DESC LIMIT 1`) + err := db.Get(result, query, sessionID) + if err != nil { + if err == sql.ErrNoRows { + return nil, ErrNoResults + } + + return nil, err + } + + return result, nil +} + +func (db *sqlstore) GetProcessablePageviews() ([]*models.Pageview, error) { + var results []*models.Pageview + thirtyMinsAgo := time.Now().Add(-30 * time.Minute) + query := db.Rebind(`SELECT * FROM pageviews WHERE ( duration > 0 AND is_bounce = 0 ) OR timestamp < ? LIMIT 500`) + err := db.Select(&results, query, thirtyMinsAgo) + return results, err +} + +func (db *sqlstore) DeletePageviews(pageviews []*models.Pageview) error { + ids := []string{} + for _, p := range pageviews { + ids = append(ids, strconv.FormatInt(p.ID, 10)) + } + query := db.Rebind(`DELETE FROM pageviews WHERE id IN(` + strings.Join(ids, ",") + `)`) + _, err := db.Exec(query) + return err +} diff --git a/pkg/datastore/sqlstore/referrer_stats.go b/pkg/datastore/sqlstore/referrer_stats.go new file mode 100644 index 0000000..4de2dfe --- /dev/null +++ b/pkg/datastore/sqlstore/referrer_stats.go @@ -0,0 +1,44 @@ +package sqlstore + +import ( + "database/sql" + "time" + + "github.com/usefathom/fathom/pkg/models" +) + +func (db *sqlstore) GetReferrerStats(date time.Time, url string) (*models.ReferrerStats, error) { + stats := &models.ReferrerStats{} + query := db.Rebind(`SELECT * FROM daily_referrer_stats WHERE url = ? AND date = ? LIMIT 1`) + err := db.Get(stats, query, url, date.Format("2006-01-02")) + if err != nil && err == sql.ErrNoRows { + return nil, ErrNoResults + } + return stats, err +} + +func (db *sqlstore) InsertReferrerStats(s *models.ReferrerStats) error { + query := db.Rebind(`INSERT INTO daily_referrer_stats(visitors, pageviews, bounce_rate, avg_duration, url, date) VALUES(?, ?, ?, ?, ?, ?)`) + _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02")) + return err +} + +func (db *sqlstore) UpdateReferrerStats(s *models.ReferrerStats) error { + query := db.Rebind(`UPDATE daily_referrer_stats SET visitors = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE url = ? AND date = ?`) + _, err := db.Exec(query, s.Visitors, s.Pageviews, s.BounceRate, s.AvgDuration, s.URL, s.Date.Format("2006-01-02")) + return err +} + +func (db *sqlstore) GetAggregatedReferrerStats(startDate time.Time, endDate time.Time, limit int) ([]*models.ReferrerStats, error) { + var result []*models.ReferrerStats + query := db.Rebind(`SELECT url, SUM(visitors) AS visitors, SUM(pageviews) AS pageviews, COALESCE(ROUND(SUM(pageviews*bounce_rate)/SUM(pageviews), 4), 0.00) AS bounce_rate, COALESCE(ROUND(SUM(avg_duration*pageviews)/SUM(pageviews), 4), 0.00) AS avg_duration FROM daily_referrer_stats WHERE date >= ? AND date <= ? GROUP BY url ORDER BY pageviews DESC LIMIT ?`) + err := db.Select(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), limit) + return result, err +} + +func (db *sqlstore) GetAggregatedReferrerStatsPageviews(startDate time.Time, endDate time.Time) (int, error) { + var result int + query := db.Rebind(`SELECT COALESCE(SUM(pageviews), 0) FROM daily_referrer_stats WHERE date >= ? AND date <= ?`) + err := db.Get(&result, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) + return result, err +} diff --git a/pkg/datastore/seed/pageviews.sql b/pkg/datastore/sqlstore/seed/pageviews.sql similarity index 100% rename from pkg/datastore/seed/pageviews.sql rename to pkg/datastore/sqlstore/seed/pageviews.sql diff --git a/pkg/datastore/sqlstore/site_stats.go b/pkg/datastore/sqlstore/site_stats.go new file mode 100644 index 0000000..f18df0b --- /dev/null +++ b/pkg/datastore/sqlstore/site_stats.go @@ -0,0 +1,77 @@ +package sqlstore + +import ( + "database/sql" + "github.com/usefathom/fathom/pkg/models" + "time" +) + +func (db *sqlstore) GetSiteStats(date time.Time) (*models.SiteStats, error) { + stats := &models.SiteStats{} + query := db.Rebind(`SELECT * FROM daily_site_stats WHERE date = ? LIMIT 1`) + err := db.Get(stats, query, date.Format("2006-01-02")) + if err != nil && err == sql.ErrNoRows { + return nil, ErrNoResults + } + return stats, err +} + +func (db *sqlstore) InsertSiteStats(s *models.SiteStats) error { + query := db.Rebind(`INSERT INTO daily_site_stats(visitors, sessions, pageviews, bounce_rate, avg_duration, date) VALUES(?, ?, ?, ?, ?, ?)`) + _, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02")) + return err +} + +func (db *sqlstore) UpdateSiteStats(s *models.SiteStats) error { + query := db.Rebind(`UPDATE daily_site_stats SET visitors = ?, sessions = ?, pageviews = ?, bounce_rate = ROUND(?, 4), avg_duration = ROUND(?, 4) WHERE date = ?`) + _, err := db.Exec(query, s.Visitors, s.Sessions, s.Pageviews, s.BounceRate, s.AvgDuration, s.Date.Format("2006-01-02")) + return err +} + +func (db *sqlstore) GetTotalSiteViews(startDate time.Time, endDate time.Time) (int, error) { + sql := `SELECT COALESCE(SUM(pageviews), 0) FROM daily_site_stats WHERE date >= ? AND date <= ?` + query := db.Rebind(sql) + var total int + err := db.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) + return total, err +} + +func (db *sqlstore) GetTotalSiteVisitors(startDate time.Time, endDate time.Time) (int, error) { + sql := `SELECT COALESCE(SUM(visitors), 0) FROM daily_site_stats WHERE date >= ? AND date <= ?` + query := db.Rebind(sql) + var total int + err := db.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) + return total, err +} + +func (db *sqlstore) GetTotalSiteSessions(startDate time.Time, endDate time.Time) (int, error) { + sql := `SELECT COALESCE(SUM(sessions), 0) FROM daily_site_stats WHERE date >= ? AND date <= ?` + query := db.Rebind(sql) + var total int + err := db.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) + return total, err +} + +func (db *sqlstore) GetAverageSiteDuration(startDate time.Time, endDate time.Time) (float64, error) { + sql := `SELECT COALESCE(ROUND(SUM(pageviews*avg_duration)/SUM(pageviews), 4), 0.00) FROM daily_site_stats WHERE date >= ? AND date <= ?` + query := db.Rebind(sql) + var total float64 + err := db.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) + return total, err +} + +func (db *sqlstore) GetAverageSiteBounceRate(startDate time.Time, endDate time.Time) (float64, error) { + sql := `SELECT COALESCE(ROUND(SUM(sessions*bounce_rate)/SUM(sessions), 4), 0.00) FROM daily_site_stats WHERE date >= ? AND date <= ?` + query := db.Rebind(sql) + var total float64 + err := db.Get(&total, query, startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) + return total, err +} + +func (db *sqlstore) GetRealtimeVisitorCount() (int, error) { + sql := `SELECT COUNT(DISTINCT(session_id)) FROM pageviews WHERE timestamp > ?` + query := db.Rebind(sql) + var total int + err := db.Get(&total, query, time.Now().Add(-5*time.Minute)) + return total, err +} diff --git a/pkg/datastore/sqlstore/sqlstore.go b/pkg/datastore/sqlstore/sqlstore.go new file mode 100644 index 0000000..8b96f78 --- /dev/null +++ b/pkg/datastore/sqlstore/sqlstore.go @@ -0,0 +1,56 @@ +package sqlstore + +import ( + "errors" + + _ "github.com/go-sql-driver/mysql" // mysql driver + "github.com/gobuffalo/packr" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" // postgresql driver + _ "github.com/mattn/go-sqlite3" //sqlite3 driver + migrate "github.com/rubenv/sql-migrate" + log "github.com/sirupsen/logrus" +) + +type sqlstore struct { + *sqlx.DB + + Config *Config +} + +// ErrNoResults is returned when a query yielded 0 results +var ErrNoResults = errors.New("datastore: query returned 0 results") + +// New creates a new database pool +func New(c *Config) *sqlstore { + dbx := sqlx.MustConnect(c.Driver, c.DSN()) + db := &sqlstore{dbx, c} + + // run migrations + db.Migrate() + + return db +} + +func (db *sqlstore) Migrate() { + migrations := &migrate.PackrMigrationSource{ + Box: packr.NewBox("./migrations"), + Dir: "./" + db.Config.Driver, + } + migrate.SetTable("migrations") + + n, err := migrate.Exec(db.DB.DB, db.Config.Driver, migrations, migrate.Up) + + if err != nil { + log.Fatal("database migrations failed: ", err) + } + + if n > 0 { + log.Infof("applied %d database migrations", n) + } +} + +// Closes the db pool +func (db *sqlstore) Close() { + db.DB.Close() +} diff --git a/pkg/datastore/users.go b/pkg/datastore/sqlstore/users.go similarity index 53% rename from pkg/datastore/users.go rename to pkg/datastore/sqlstore/users.go index 69f5253..3f9f53d 100644 --- a/pkg/datastore/users.go +++ b/pkg/datastore/sqlstore/users.go @@ -1,4 +1,4 @@ -package datastore +package sqlstore import ( "database/sql" @@ -6,10 +6,10 @@ import ( ) // GetUser retrieves user from datastore by its ID -func GetUser(ID int64) (*models.User, error) { +func (db *sqlstore) GetUser(ID int64) (*models.User, error) { u := &models.User{} - query := dbx.Rebind("SELECT * FROM users WHERE id = ? LIMIT 1") - err := dbx.Get(u, query, ID) + query := db.Rebind("SELECT * FROM users WHERE id = ? LIMIT 1") + err := db.Get(u, query, ID) if err != nil { if err == sql.ErrNoRows { @@ -23,10 +23,10 @@ func GetUser(ID int64) (*models.User, error) { } // GetUserByEmail retrieves user from datastore by its email -func GetUserByEmail(email string) (*models.User, error) { +func (db *sqlstore) GetUserByEmail(email string) (*models.User, error) { u := &models.User{} - query := dbx.Rebind("SELECT * FROM users WHERE email = ? LIMIT 1") - err := dbx.Get(u, query, email) + query := db.Rebind("SELECT * FROM users WHERE email = ? LIMIT 1") + err := db.Get(u, query, email) if err != nil { if err == sql.ErrNoRows { return nil, ErrNoResults @@ -39,9 +39,9 @@ func GetUserByEmail(email string) (*models.User, error) { } // SaveUser inserts the user model in the connected database -func SaveUser(u *models.User) error { - var query = dbx.Rebind("INSERT INTO users(email, password) VALUES(?, ?)") - result, err := dbx.Exec(query, u.Email, u.Password) +func (db *sqlstore) SaveUser(u *models.User) error { + var query = db.Rebind("INSERT INTO users(email, password) VALUES(?, ?)") + result, err := db.Exec(query, u.Email, u.Password) if err != nil { return err }