Move balances update to 10-min update thread

Add Tufte profiling

project.clj

@@ -11,6 +11,7 @@
                  [org.clojure/clojurescript "1.9.946"]
                  [org.clojure/clojure "1.8.0"]
                  [selmer "1.11.1"]
+                 [com.taoensso/tufte "1.3.0"]
                  [markdown-clj "1.0.1"]
                  [ring-middleware-format "0.7.2"]
                  [ring/ring-core "1.6.2"]

src/clj/commiteth/eth/core.clj

@@ -4,8 +4,10 @@
             [clojure.java.io :as io]
             [commiteth.config :refer [env]]
             [clojure.string :refer [join]]
+            [taoensso.tufte :as tufte :refer (defnp p profiled profile)]
             [clojure.tools.logging :as log]
             [clojure.string :as str]
+            [mount.core :as mount]
             [pandect.core :as pandect]
             [commiteth.util.util :refer [json-api-request]])
   (:import [org.web3j
@@ -26,24 +28,30 @@
 (defn auto-gas-price? [] (env :auto-gas-price false))
 (defn offline-signing? [] (env :offline-signing true))
 
+(def web3j-obj (atom nil))
+(def creds-obj (atom nil))
+
 (defn wallet-file-path []
   (env :eth-wallet-file))
 
 (defn wallet-password []
   (env :eth-password))
 
+(defn create-web3j []
+ (or @web3j-obj
+      (swap! web3j-obj (constantly (Web3j/build (HttpService. (eth-rpc-url)))))))
+
 (defn creds []
+  (or @creds-obj
       (let [password  (wallet-password)
             file-path (wallet-file-path)]
         (if (and password file-path)
-      (WalletUtils/loadCredentials
+          (swap! creds-obj
+                 (constantly (WalletUtils/loadCredentials
                                password
-       file-path)
+                               file-path)))
           (throw (ex-info "Make sure you provided proper credentials in appropriate resources/config.edn"
-                      {:password password :file-path file-path})))))
+                          {:password password :file-path file-path}))))))
-
-(defn create-web3j []
-  (Web3j/build (HttpService. (eth-rpc-url))))
 
 (defn get-signed-tx [gas-price gas-limit to data]
   "Create a sign a raw transaction.
@@ -192,7 +200,8 @@
 
 (defn get-balance-eth
   [account digits]
-  (hex->eth (get-balance-hex account) digits))
+  (p :get-balance-eth
+     (hex->eth (get-balance-hex account) digits)))
 
 (defn- format-param
   [param]
@@ -296,3 +305,15 @@
             (filter true?)
             (empty?)))
           true))))
+
+(mount/defstate
+  eth-core
+  :start
+  (do
+    (swap! web3j-obj (constantly nil))
+    (swap! creds-obj (constantly nil))
+    (log/info "eth/core started"))
+  :stop
+  (log/info "eth/core stopped"))
+
+

src/clj/commiteth/eth/multisig_wallet.clj

@@ -3,6 +3,7 @@
              :refer [create-web3j creds]]
             [commiteth.config :refer [env]]
             [clojure.tools.logging :as log]
+            [taoensso.tufte :as tufte :refer (defnp p profiled profile)]
             [commiteth.eth.token-data :as token-data])
   (:import [org.web3j
             abi.datatypes.Address
@@ -147,6 +148,7 @@
   "Query (internal) ERC20 token balance from bounty contract for given
   token TLA."
   [bounty-addr token]
+  (p :token-balance-in-bounty
      (let [bounty-contract (load-bounty-contract bounty-addr)
         token-address (get-token-address token)
         token-addr-web3j (Address. token-address)]
@@ -154,12 +156,13 @@
         (.tokenBalances token-addr-web3j)
         .get
         .getValue
-        (convert-token-value token))))
+        (convert-token-value token)))))
 
 (defn token-balance
   "Query balance of given ERC20 token TLA for given address from ERC20
   contract."
   [bounty-addr token]
+  (p :token-balance
      (let [token-address (get-token-address token)]
     (log/debug "token-balance" bounty-addr token token-address)
     (try
@@ -170,25 +173,26 @@
           (convert-token-value token))
       (catch Throwable t
         (log/debug "Failed to query token balance " t)
-        0))))
+        0)))))
 
 
 (defn token-balances
   "Get a given bounty contract's token balances. Assumes contract's
   internal balances have been updated."
   [bounty-addr]
-  (let [bounty-contract (load-bounty-contract bounty-addr)
+  (p :token-balances
-        token-addresses (-> bounty-contract
+     (let [bounty-contract (p :load-bounty-contract (load-bounty-contract bounty-addr))
+        token-addresses (p :getTokenList (-> bounty-contract
                             (.getTokenList)
-                            .get)]
+                            .get))]
     (if token-addresses
       (let [addrs (map str
-                       (.getValue token-addresses))]
+                       (p :getValue (.getValue token-addresses)))]
         (into {}
               (map (fn [addr] (if-let [info (token-data/token-info-by-addr addr)]
                                (let [tla (first info)]
                                  [tla (token-balance bounty-addr tla)]))) addrs)))
-      {})))
+      {}))))
 
 (defn uint256 [x]
   (org.web3j.abi.datatypes.generated.Uint256. x))

src/clj/commiteth/scheduler.clj

@@ -4,6 +4,7 @@
             [commiteth.eth.token-data :as token-data]
             [commiteth.github.core :as github]
             [commiteth.db.issues :as issues]
+            [taoensso.tufte :as tufte :refer (defnp p profiled profile)]
             [commiteth.db.bounties :as db-bounties]
             [commiteth.bounties :as bounties]
             [commiteth.util.crypto-fiat-value :as fiat-util]
@@ -15,12 +16,34 @@
             [clj-time.periodic :refer [periodic-seq]]
             [chime :refer [chime-at]]))
 
+(tufte/add-basic-println-handler! {})
+(tufte/add-handler! :file (fn [{stats :stats}]
+                            (log/info "Profiling stats:" stats)))
 
+(comment
+
+  (profile {} (update-issue-contract-address))
+  (profile {} (deploy-pending-contracts))
+  (profile {} (self-sign-bounty))
+  (profile {} (update-confirm-hash))
+  (profile {} (update-watch-hash))
+  (profile {} (update-payout-receipt))
+  (profile {} (update-contract-internal-balances))
+  (profile {} (update-open-issue-usd-values))
+  (profile {} (update-balances))
+  (profile {}
+           (doseq [i (range 5)]
+             (update-contract-internal-balances) 
+             (update-open-issue-usd-values) 
+             (update-balances)))
+
+  )
 (defn update-issue-contract-address
   "For each pending deployment: gets transaction receipt, updates db
   state (contract-address, comment-id) and posts github comment"
   []
   (log/info "In update-issue-contract-address")
+  (p :update-issue-contract-address
      (doseq [{issue-id         :issue_id
            transaction-hash :transaction_hash} (issues/list-pending-deployments)]
     (log/info "pending deployment:" transaction-hash)
@@ -57,7 +80,7 @@
           (log/error "Failed to find contract address in tx logs")))
       (catch Throwable ex 
         (do (log/error "update-issue-contract-address exception:" ex)
-            (clojure.stacktrace/print-stack-trace ex)))))
+            (clojure.stacktrace/print-stack-trace ex))))))
   (log/info "Exit update-issue-contract-address"))
 
 
@@ -84,15 +107,17 @@
 (defn deploy-pending-contracts
   "Under high-concurrency circumstances or in case geth is in defunct state, a bounty contract may not deploy successfully when the bounty label is addded to an issue. This function deploys such contracts."
   []
+  (p :deploy-pending-contracts
      (doseq [{issue-id :issue_id
            owner-address :owner_address} (db-bounties/pending-contracts)]
     (log/debug "Trying to re-deploy failed bounty contract deployment, issue-id:" issue-id)
-    (deploy-contract owner-address issue-id)))
+    (deploy-contract owner-address issue-id))))
 
 (defn self-sign-bounty
   "Walks through all issues eligible for bounty payout and signs corresponding transaction"
   []
   (log/info "In self-sign-bounty")
+  (p :self-sign-bounty
      (doseq [{contract-address :contract_address
            issue-id         :issue_id
            payout-address   :payout_address
@@ -130,7 +155,7 @@
                                                 false))))
       (catch Throwable ex 
         (do (log/error "self-sign-bounty exception:" ex)
-            (clojure.stacktrace/print-stack-trace ex)))))
+            (clojure.stacktrace/print-stack-trace ex))))))
   (log/info "Exit self-sign-bounty")
   )
 
@@ -138,6 +163,7 @@
   "Gets transaction receipt for each pending payout and updates DB confirm_hash with tranaction ID of commiteth bot account's confirmation."
   []
   (log/info "In update-confirm-hash")
+  (p :update-confirm-hash
      (doseq [{issue-id     :issue_id
            execute-hash :execute_hash} (db-bounties/pending-payouts)]
     (log/info "pending payout:" execute-hash)
@@ -145,18 +171,19 @@
       (log/info "execution receipt for issue #" issue-id ": " receipt)
       (when-let [confirm-hash (multisig/find-confirmation-tx-id receipt)]
         (log/info "confirm hash:" confirm-hash)
-        (db-bounties/update-confirm-hash issue-id confirm-hash))))
+        (db-bounties/update-confirm-hash issue-id confirm-hash)))))
   (log/info "Exit update-confirm-hash"))
 
 
 (defn update-watch-hash
   "Sets watch-hash to NULL for bounties where watch tx has been mined. Used to avoid unneeded watch transactions in update-bounty-token-balances"
   []
+  (p :update-watch-hash
      (doseq [{issue-id :issue_id
            watch-hash :watch_hash} (db-bounties/pending-watch-calls)]
     (log/info "pending watch call" watch-hash)
     (when-let [receipt (eth/get-transaction-receipt watch-hash)]
-      (db-bounties/update-watch-hash issue-id nil))))
+      (db-bounties/update-watch-hash issue-id nil)))))
 
 
 (defn older-than-3h?
@@ -171,6 +198,7 @@
   "Gets transaction receipt for each confirmed payout and updates payout_hash"
   []
   (log/info "In update-payout-receipt")
+  (p :update-payout-receipt
      (doseq [{issue-id    :issue_id
            payout-hash :payout_hash
            contract-address :contract_address
@@ -213,7 +241,7 @@
           (db-bounties/reset-payout-hash issue-id)))
       (catch Throwable ex 
         (do (log/error "update-payout-receipt exception:" ex)
-            (clojure.stacktrace/print-stack-trace ex)))))
+            (clojure.stacktrace/print-stack-trace ex))))))
   (log/info "Exit update-payout-receipt")
   )
 
@@ -258,12 +286,14 @@
 (defn update-contract-internal-balances
   "It is required in our current smart contract to manually update it's internal balance when some tokens have been added."
   []
+  (log/info "In update-contract-internal-balances")
+  (p :update-contract-internal-balances
      (doseq [{issue-id :issue_id
               bounty-address :contract_address
               watch-hash :watch_hash}
              (db-bounties/open-bounty-contracts)]
        (update-bounty-token-balances issue-id bounty-address watch-hash)))
-
+  (log/info "Exit update-contract-internal-balances"))
 
 (defn get-bounty-funds
   "Get funds in given bounty contract.
@@ -287,13 +317,15 @@
 (defn update-open-issue-usd-values
   "Sum up current USD values of all crypto assets in a bounty and store to DB"
   []
+  (p :update-open-issue-usd-values
      (doseq [{bounty-addr :contract_address}
           (db-bounties/open-bounty-contracts)]
-    (update-issue-usd-value bounty-addr)))
+    (update-issue-usd-value bounty-addr))))
 
 (defn update-balances
   []
   (log/info "In update-balances")
+  (p :update-balances
      (doseq [{contract-address :contract_address
            owner            :owner
            repo             :repo
@@ -342,7 +374,7 @@
             (update-issue-usd-value contract-address))))
       (catch Throwable ex 
         (do (log/error "update-balances exception:" ex)
-            (clojure.stacktrace/print-stack-trace ex)))))
+            (clojure.stacktrace/print-stack-trace ex))))))
   (log/info "Exit update-balances"))
 
 
@@ -370,17 +402,18 @@
       update-payout-receipt
       update-watch-hash
       self-sign-bounty
-      update-contract-internal-balances
+      ])
-      update-balances])
+    (log/info "run-1-min-interval-tasks done")))
-    (log/debug "run-1-min-interval-tasks done")))
 
 
 (defn run-10-min-interval-tasks [time]
   (do
-    (log/debug "run-1-min-interval-tasks" time)
+    (log/info "run-10-min-interval-tasks" time)
     (run-tasks
-     [update-open-issue-usd-values])
+     [update-contract-internal-balances
-    (log/debug "run-10-min-interval-tasks done")))
+      update-balances
+      update-open-issue-usd-values])
+    (log/info "run-10-min-interval-tasks done")))
 
 
 (mount/defstate scheduler