status-im
/
timbre
mirror of https://github.com/status-im/timbre.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add unified arg hashing (rate limiter, Carmine appender, etc.)

This commit is contained in:
Peter Taoussanis 2013-12-01 20:37:29 +07:00
parent 34394319c2
commit 8e5329e79f
2 changed files with 27 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/taoensso/timbre.clj
View File

@ -98,6 +98,7 @@
:rate-limit ; (Optional) [ncalls-limit window-ms].
:fmt-output-opts ; (Optional) extra opts passed to `fmt-output-fn`.
:fn ; (fn [appender-args-map]), with keys described below.
:args-hash-fn ; Experimental. Used by rate-limiter, etc.
An appender's fn takes a single map with keys:
:level ; Keyword.
@ -179,10 +180,28 @@
;;;; Appender-fn decoration
(defn default-args-hash-fn
"Alpha - subject to change!!
Returns a hash identifier for given appender arguments in such a way that
(= (hash args-A) (hash args-B)) iff arguments A and B are \"the same\" by
some reasonable-in-the-general-case definition for logging arguments.
Useful in the context of rate limiting, deduplicating appenders, etc."
;; Things like dates & user ids user ids will still trip us up.
;; `[hostname ns line]` may be another idea?
;; Waiting on http://dev.clojure.org/jira/browse/CLJ-865.
[{:keys [hostname ns args] :as apfn-args}]
(str (or (some #(and (map? %) (:timbre/hash %)) args) ; Explicit hash given
[hostname ns args])))
(defn- wrap-appender-fn
"Wraps compile-time appender fn with additional runtime capabilities
controlled by compile-time config."
[config {apfn :fn :keys [async? rate-limit fmt-output-opts] :as appender}]
[config {:as appender apfn :fn
:keys [async? rate-limit fmt-output-opts args-hash-fn]
:or {args-hash-fn default-args-hash-fn}}]
(let [rate-limit (or rate-limit ; Backwards comp:
(if-let [x (:max-message-per-msecs appender)] [1 x]
(when-let [x (:limit-per-msecs appender)] [1 x])))]
@ -208,12 +227,13 @@
limiter-any (utils/rate-limiter ncalls-limit window-ms)
;; This is a little hand-wavy but it's a decent general
;; strategy and helps us from making this overly complex to
;; configure:
;; configure.
limiter-specific (utils/rate-limiter (quot ncalls-limit 4)
window-ms)]
(fn [{:keys [ns args] :as apfn-args}]
;; Runtime: (test smaller limit 1st):
(when-not (or (limiter-specific (str ns args)) (limiter-any))
(when-not (or (limiter-specific (args-hash-fn apfn-args))
(limiter-any))
(apfn apfn-args)))))))
;; Async (agent) support

16
src/taoensso/timbre/appenders/carmine.clj
View File

@ -17,14 +17,6 @@
(defn default-keyfn [level] {:pre [(string? level)]}
(format "carmine:timbre:default:%s" level))
(defn default-entry-hash-fn [{:keys [hostname ns args] :as apfn-args}]
;; We try choose a hashing strategy here that gives a reasonable
;; definition of 'uniqueness' for general entries. Things like dates
;; or user ids will still trip us up. `[hostname ns line]` may be another
;; idea? Waiting on http://dev.clojure.org/jira/browse/CLJ-865.
(or (some #(and (map? %) (:timbre/id %)) args)
[hostname ns args]))
(defn make-carmine-appender
"Alpha - subject to change!
Returns a Carmine Redis appender:
@ -38,9 +30,9 @@
also offer interesting opportunities here.
See accompanying `query-entries` fn to return deserialized log entries."
[& [appender-opts {:keys [conn keyfn entry-hash-fn nentries-by-level]
:or {keyfn default-keyfn
entry-hash-fn default-entry-hash-fn
[& [appender-opts {:keys [conn keyfn args-hash-fn nentries-by-level]
:or {keyfn default-keyfn
args-hash-fn timbre/default-args-hash-fn
nentries-by-level {:trace 50
:debug 50
:info 50
@ -56,7 +48,7 @@
(merge default-appender-opts appender-opts
{:fn
(fn [{:keys [level instant] :as apfn-args}]
(let [entry-hash (sha48 (entry-hash-fn apfn-args))
(let [entry-hash (sha48 (args-hash-fn apfn-args))
entry (select-keys apfn-args [:hostname :ns :args :throwable
:profile-stats])
k-zset (keyfn (name level))