[#5749] transaction history fixes

Signed-off-by: Goran Jovic <goranjovic@gmail.com>

src/status_im/accounts/logout/core.cljs

@@ -4,7 +4,8 @@
             [status-im.init.core :as init]
             [status-im.transport.core :as transport]
             [status-im.ui.screens.navigation :as navigation]
-            [status-im.utils.fx :as fx]))
+            [status-im.utils.fx :as fx]
+            [status-im.models.transactions :as transactions]))
 
 (fx/defn logout
   [{:keys [db] :as cofx}]
@@ -12,6 +13,7 @@
     (fx/merge cofx
               {:keychain/clear-user-password (get-in db [:account/account :address])
                :dev-server/stop              nil}
+              (transactions/stop-sync)
               (navigation/navigate-to-clean :login {})
               (transport/stop-whisper)
               (init/initialize-keychain))))

src/status_im/chat/commands/impl/transactions.cljs

@@ -293,9 +293,7 @@
           (let [updated-request-message (assoc-in request-message [:content :params :answered?] true)]
             {:db            (assoc-in db [:chats chat-id :messages responding-to] updated-request-message)
              :data-store/tx [(messages-store/save-message-tx updated-request-message)]})))))
-  (on-receive [_ command-message cofx]
+  (on-receive [_ command-message cofx])
-    (when-let [tx-hash (get-in command-message [:content :params :tx-hash])]
-      (wallet.transactions/store-chat-transaction-hash tx-hash cofx)))
   (short-preview [_ command-message]
     (personal-send-request-short-preview :command-sending command-message))
   (preview [_ command-message]

src/status_im/init/core.cljs

@@ -181,7 +181,6 @@
 (defn initialize-wallet [cofx]
   (fx/merge cofx
             (models.wallet/update-wallet)
-            (transactions/run-update)
             (transactions/start-sync)))
 
 (defn login-only-events [cofx address]

src/status_im/models/transactions.cljs

@@ -1,116 +1,439 @@
 (ns status-im.models.transactions
   (:require [clojure.set :as set]
-            [status-im.utils.datetime :as time]
+            [cljs.core.async :as async]
+            [clojure.string :as string]
+            [status-im.utils.async :as async-util]
             [status-im.utils.ethereum.core :as ethereum]
+            [status-im.constants :as constants]
+            [status-im.native-module.core :as status]
             [status-im.utils.ethereum.tokens :as tokens]
-            [status-im.utils.semaphores :as semaphores]
+            [status-im.utils.http :as http]
+            [status-im.utils.types :as types]
             [taoensso.timbre :as log]
-            [status-im.utils.fx :as fx]))
+            [status-im.utils.fx :as fx]
+            [re-frame.core :as re-frame]
+            [re-frame.db])
+  (:require-macros
+   [cljs.core.async.macros :refer [go-loop go]]))
 
 (def sync-interval-ms 15000)
+(def sync-timeout-ms  20000)
 (def confirmations-count-threshold 12)
+(def block-query-limit 100000)
 
-;; Detects if some of the transactions have less than 12 confirmations
+;; ----------------------------------------------------------------------------
-(defn- have-unconfirmed-transactions? [cofx]
+;; token transfer event logs from eth-node
-  (->> (get-in cofx [:db :wallet :transactions])
+;; ----------------------------------------------------------------------------
+
+(defn- parse-json [s]
+  {:pre [(string? s)]}
+  (try
+    (let [res (-> s
+                  js/JSON.parse
+                  (js->clj :keywordize-keys true))]
+      (if (= (:error res) "")
+        {:result true}
+        res))
+    (catch :default e
+      {:error (.-message e)})))
+
+(defn- add-padding [address]
+  {:pre [(string? address)]}
+  (str "0x000000000000000000000000" (subs address 2)))
+
+(defn- remove-padding [topic]
+  {:pre [(string? topic)]}
+  (str "0x" (subs topic 26)))
+
+(defn- parse-transaction-entries [current-block-number block-info chain-tokens direction transfers]
+  {:pre [(integer? current-block-number) (map? block-info)
+         (map? chain-tokens) (every? (fn [[k v]] (and (string? k) (map? v))) chain-tokens)
+         (keyword? direction)
+         (every? map? transfers)]}
+  (into {}
+        (keep identity
+              (for [transfer transfers]
+                (when-let [token (->> transfer :address (get chain-tokens))]
+                  (when-not (:nft? token)
+                    [(:transactionHash transfer)
+                     {:block         (-> block-info :number str)
+                      :hash          (:transactionHash transfer)
+                      :symbol        (:symbol token)
+                      :from          (some-> transfer :topics second remove-padding)
+                      :to            (some-> transfer :topics last remove-padding)
+                      :value         (-> transfer :data ethereum/hex->bignumber)
+                      :type          direction
+
+                      :confirmations (str (- current-block-number (-> transfer :blockNumber ethereum/hex->int)))
+
+                      :gas-price     nil
+                      :nonce         nil
+                      :data          nil
+
+                      :gas-limit     nil
+                      :timestamp     (-> block-info :timestamp (* 1000) str)
+
+                      :gas-used      nil
+
+                      ;; NOTE(goranjovic) - metadata on the type of token: contains name, symbol, decimas, address.
+                      :token         token
+
+                      ;; NOTE(goranjovic) - if an event has been emitted, we can say there was no error
+                      :error?        false
+
+                      ;; NOTE(goranjovic) - just a flag we need when we merge this entry with the existing entry in
+                      ;; the app, e.g. transaction info with gas details, or a previous transfer entry with old
+                      ;; confirmations count.
+                      :transfer      true}]))))))
+
+(defn- add-block-info [web3 current-block-number chain-tokens direction result success-fn]
+  {:pre [web3 (integer? current-block-number) (map? chain-tokens) (keyword? direction)
+         (every? map? result)
+         (fn? success-fn)]}
+  (let [transfers-by-block (group-by :blockNumber result)]
+    (doseq [[block-number transfers] transfers-by-block]
+      (ethereum/get-block-info web3 (ethereum/hex->int block-number)
+                               (fn [block-info]
+                                 (if-not (map? block-info)
+                                   (log/error "Request for block info failed")
+                                   (success-fn (parse-transaction-entries current-block-number
+                                                                          block-info
+                                                                          chain-tokens
+                                                                          direction
+                                                                          transfers))))))))
+
+(defn- response-handler [web3 current-block-number chain-tokens direction error-fn success-fn]
+  (fn handle-response
+    ([response]
+     #_(log/debug "Token transaction logs recieved --" (pr-str response))
+     (let [{:keys [error result]} (parse-json response)]
+       (handle-response error result)))
+    ([error result]
+     (if error
+       (error-fn error)
+       (add-block-info web3 current-block-number chain-tokens direction result success-fn)))))
+
+(defn- limited-from-block [current-block-number]
+  {:pre [(integer? current-block-number)]
+   ;; needs to be a positive etherium hex
+   :post [(string? %) (string/starts-with? % "0x")]}
+  (-> current-block-number (- block-query-limit) (max 0) ethereum/int->hex))
+
+;; Here we are querying event logs for Transfer events.
+;;
+;; The parameters are as follows:
+;; - address - token smart contract address
+;; - fromBlock - we need to specify it, since default is latest
+;; - topics[0] - hash code of the Transfer event signature
+;; - topics[1] - address of token sender with leading zeroes padding up to 32 bytes
+;; - topics[2] - address of token sender with leading zeroes padding up to 32 bytes
+;;
+
+(defn- get-token-transfer-logs
+  ;; NOTE(goranjovic): here we use direct JSON-RPC calls to get event logs because of web3 event issues with infura
+  ;; we still use web3 to get other data, such as block info
+  [web3 current-block-number chain-tokens direction address cb]
+  {:pre [web3 (integer? current-block-number) (map? chain-tokens) (keyword? direction) (string? address) (fn? cb)]}
+  (let [[from to] (if (= :inbound direction)
+                    [nil (add-padding (ethereum/normalized-address address))]
+                    [(add-padding (ethereum/normalized-address address)) nil])
+        from-block (limited-from-block current-block-number)
+        args {:jsonrpc "2.0"
+              :id      2
+              :method  constants/web3-get-logs
+              :params  [{:address (keys chain-tokens)
+                         :fromBlock from-block
+                         :topics    [constants/event-transfer-hash from to]}]}
+        payload (.stringify js/JSON (clj->js args))]
+    (status/call-private-rpc payload
+                             (response-handler web3 current-block-number chain-tokens direction ethereum/handle-error cb))))
+
+(defn- get-token-transactions
+  [web3 chain-tokens direction address cb]
+  {:pre [web3 (map? chain-tokens) (keyword? direction) (string? address) (fn? cb)]}
+  (ethereum/get-block-number web3
+                             #(get-token-transfer-logs web3 % chain-tokens direction address cb)))
+
+;; --------------------------------------------------------------------------
+;; etherscan transactions
+;; --------------------------------------------------------------------------
+
+(def etherscan-supported? #{:testnet :mainnet :rinkeby})
+
+(let [network->subdomain {:testnet "ropsten" :rinkeby "rinkeby"}]
+  (defn get-transaction-details-url [chain hash]
+    {:pre [(keyword? chain) (string? hash)]
+     :post [(or (nil? %) (string? %))]}
+    (when (etherscan-supported? chain)
+      (let [network-subdomain (when-let [subdomain (network->subdomain chain)]
+                                (str subdomain "."))]
+        (str "https://" network-subdomain  "etherscan.io/tx/" hash)))))
+
+(def etherscan-api-key "DMSI4UAAKUBVGCDMVP3H2STAMSAUV7BYFI")
+
+(defn- get-api-network-subdomain [chain]
+  (case chain
+    (:testnet) "api-ropsten"
+    (:mainnet) "api"
+    (:rinkeby) "api-rinkeby"))
+
+(defn- get-transaction-url [chain account]
+  {:pre [(keyword? chain) (string? account)]
+   :post [(string? %)]}
+  (let [network-subdomain (get-api-network-subdomain chain)]
+    (str "https://" network-subdomain ".etherscan.io/api?module=account&action=txlist&address=0x"
+         account "&startblock=0&endblock=99999999&sort=desc&apikey=" etherscan-api-key "&q=json")))
+
+(defn- format-transaction [account
+                           {:keys [value timeStamp blockNumber hash from to
+                                   gas gasPrice gasUsed nonce confirmations
+                                   input isError]}]
+  (let [inbound? (= (str "0x" account) to)
+        error?   (= "1" isError)]
+    {:value         value
+     ;; timestamp is in seconds, we convert it in ms
+     :timestamp     (str timeStamp "000")
+     :symbol        :ETH
+     :type          (cond error?   :failed
+                          inbound? :inbound
+                          :else    :outbound)
+     :block         blockNumber
+     :hash          hash
+     :from          from
+     :to            to
+     :gas-limit     gas
+     :gas-price     gasPrice
+     :gas-used      gasUsed
+     :nonce         nonce
+     :confirmations confirmations
+     :data          input}))
+
+(defn- format-transactions-response [response account]
+  (let [{:keys [result]} (types/json->clj response)]
+    (cond-> {}
+      (vector? result)
+      (into (comp
+             (map (partial format-transaction account))
+             (map (juxt :hash identity)))
+            result))))
+
+(defn- etherscan-transactions [chain account on-success on-error]
+  (if (etherscan-supported? chain)
+    (let [url (get-transaction-url chain account)]
+      (log/debug "HTTP GET" url)
+      (http/get url
+                #(on-success (format-transactions-response % account))
+                on-error))
+    (log/info "Etherscan not supported for " chain)))
+
+(defn- get-transactions [{:keys [web3 chain chain-tokens account-address success-fn error-fn]}]
+  (log/debug "Syncing transactions data..")
+  (etherscan-transactions chain
+                          account-address
+                          success-fn
+                          error-fn)
+  (doseq [direction [:inbound :outbound]]
+    (get-token-transactions web3
+                            chain-tokens
+                            direction
+                            account-address
+                            success-fn)))
+
+;; ---------------------------------------------------------------------------
+;; Periodic background job
+;; ---------------------------------------------------------------------------
+
+(defn- async-periodic-run!
+  ([async-periodic-chan]
+   (async-periodic-run! async-periodic-chan true))
+  ([async-periodic-chan worker-fn]
+   (async/put! async-periodic-chan worker-fn)))
+
+(defn- async-periodic-stop! [async-periodic-chan]
+  (async/close! async-periodic-chan))
+
+(defn- async-periodic-exec
+  "Periodically execute an function.
+  Takes a work-fn of one argument `finished-fn -> any` this function
+  is passed a finished-fn that must be called to signal that the work
+  being performed in the work-fn is finished.
+
+  The work-fn can be forced to run immediately "
+  [work-fn interval-ms timeout-ms]
+  {:pre [(fn? work-fn) (integer? interval-ms) (integer? timeout-ms)]}
+  (let [do-now-chan (async/chan (async/sliding-buffer 1))]
+    (go-loop []
+      (let [timeout (async-util/timeout interval-ms)
+            finished-chan (async/promise-chan)
+            [v ch] (async/alts! [do-now-chan timeout])
+            worker (if (and (= ch do-now-chan) (fn? v))
+                     v work-fn)]
+        (when-not (and (= ch do-now-chan) (nil? v))
+          (try
+            (worker #(async/put! finished-chan true))
+            ;; if an error occurs in work-fn log it and consider it done
+            (catch :default e
+              (log/error "failed to run transaction sync" e)
+              (async/put! finished-chan true)))
+          ;; sanity timeout for work-fn
+          (async/alts! [finished-chan (async-util/timeout timeout-ms)])
+          (recur))))
+    do-now-chan))
+
+;; -----------------------------------------------------------------------------
+;; Helpers functions that help determine if a background sync should execute
+;; -----------------------------------------------------------------------------
+
+(defn- keyed-memoize
+  "Space bounded memoize.
+
+  Takes a key-function that decides the key in the cache for the
+  memoized value. Takes a value function that will extract the value
+  that will invalidate the cache if it changes.  And finally the
+  function to memoize.
+
+  Memoize that doesn't grow bigger than the number of keys."
+  [key-fn val-fn f]
+  (let [val-store (atom {})
+        res-store (atom {})]
+    (fn [arg]
+      (let [k (key-fn arg)
+            v (val-fn arg)]
+        (if (not= (get @val-store k) v)
+          (let [res (f arg)]
+            #_(prn "storing!!!!" res)
+            (swap! val-store assoc k v)
+            (swap! res-store assoc k res)
+            res)
+          (get @res-store k))))))
+
+;; Map[id, chat] -> Set[transaction-id]
+;; chat may or may not have a :messages Map
+(let [chat-map-entry->transaction-ids
+      (keyed-memoize key (comp :messages val)
+                     (fn [[_ chat]]
+                       (some->> (:messages chat)
                                 vals
+                                (filter #(= "command" (:content-type %)))
+                                (keep #(select-keys (get-in % [:content :params]) [:tx-hash :network])))))]
+  (defn- chat-map->transaction-ids [network chat-map]
+    {:pre  [(string? network) (every? map? (vals chat-map))]
+     :post [(set? %)]}
+    (let [network (string/replace network "_rpc" "")]
+      (->> chat-map
+           (remove (comp :public? val))
+           (mapcat chat-map-entry->transaction-ids)
+           (filter #(= network (:network %)))
+           (map :tx-hash)
+           set))))
+
+;; Seq[transaction] -> truthy
+(defn- have-unconfirmed-transactions?
+  "Detects if some of the transactions have less than 12 confirmations"
+  [transactions]
+  {:pre [(every? string? (map :confirmations transactions))]}
+  (->> transactions
        (map :confirmations)
        (map int)
        (some #(< % confirmations-count-threshold))))
 
-(defn- wallet-transactions-set [db]
+(letfn [(combine-entries [transaction token-transfer]
-  (-> db
+          (merge transaction (select-keys token-transfer [:symbol :from :to :value :type :token :transfer])))
-      (get-in [:wallet :transactions])
+        (update-confirmations [tx1 tx2]
-      keys
+                              (assoc tx1 :confirmations (str (max (int (:confirmations tx1))
-      set))
+                                                                  (int (:confirmations tx2))))))
+        (tx-and-transfer? [tx1 tx2]
+                          (and (not (:transfer tx1)) (:transfer tx2)))
+        (both-transfer?
+         [tx1 tx2]
+         (and (:transfer tx1) (:transfer tx2)))]
+  (defn- dedupe-transactions [tx1 tx2]
+    (cond (tx-and-transfer? tx1 tx2) (combine-entries tx1 tx2)
+          (tx-and-transfer? tx2 tx1) (combine-entries tx2 tx1)
+          (both-transfer? tx1 tx2)   (update-confirmations tx1 tx2)
+          :else tx2)))
 
-;; Detects if some of missing chat transactions are missing from wallet
+;; ----------------------------------------------------------------------------
-(defn- have-missing-chat-transactions? [{:keys [db]}]
+;; The following Code represents how fetching transactions is
-  (let [chat-transactions (get-in db [:wallet :chat-transactions])]
+;; complected with the rest of the application
-    (not= (count chat-transactions)
+;; ----------------------------------------------------------------------------
-          (count (set/intersection
-                  chat-transactions
-                  (wallet-transactions-set db))))))
 
-(fx/defn schedule-sync [cofx]
+(defonce polling-executor (atom nil))
-  {:utils/dispatch-later [{:ms       sync-interval-ms
-                           :dispatch [:sync-wallet-transactions]}]})
 
-(defn store-chat-transaction-hash [tx-hash {:keys [db]}]
+(defn transactions-query-helper [web3 account-address chain done-fn]
-  {:db (update-in db [:wallet :chat-transactions] conj tx-hash)})
+  (get-transactions
-
+   {:account-address account-address
-(defn- missing-chat-transactions [{:keys [db] :as cofx}]
-  (let [chat-transactions (->> db
-                               :chats
-                               vals
-                               (remove :public?)
-                               (mapcat :messages)
-                               vals
-                               flatten
-                               (filter #(= "command" (:content-type %)))
-                               (map #(get-in % [:content :params :tx-hash]))
-                               (filter identity)
-                               set)]
-    (set/difference
-     chat-transactions
-     (wallet-transactions-set db))))
-
-(fx/defn load-missing-chat-transactions
-  "Find missing chat transactions and store them at [:wallet :chat-transactions]
-  to be used later by have-missing-chat-transactions? on every sync request"
-  [{:keys [db] :as cofx}]
-  (when (nil? (get-in db [:wallet :chat-transactions]))
-    {:db (assoc-in db
-                   [:wallet :chat-transactions]
-                   (missing-chat-transactions cofx))}))
-
-(fx/defn run-update [{{:keys [network network-status web3] :as db} :db}]
-  (when (not= network-status :offline)
-    (let [network (get-in db [:account/account :networks network])
-          chain (ethereum/network->chain-keyword network)]
-      (when-not (= :custom chain)
-        (let [all-tokens (tokens/tokens-for chain)
-              token-addresses (map :address all-tokens)]
-          (log/debug "Syncing transactions data..")
-          {:get-transactions {:account-id      (get-in db [:account/account :address])
-                              :token-addresses token-addresses
     :chain           chain
+    :chain-tokens    (into {} (map (juxt :address identity) (tokens/tokens-for chain)))
     :web3            web3
-                              :success-event   :update-transactions-success
+    :success-fn (fn [transactions]
-                              :error-event     :update-transactions-fail}
+                  #_(log/debug "Transactions received: " (pr-str (keys transactions)))
-           :db               (-> db
+                  (swap! re-frame.db/app-db
-                                 (update-in [:wallet :errors] dissoc :transactions-update)
+                         (fn [app-db]
-                                 (assoc-in [:wallet :transactions-loading?] true)
+                           (when (= (get-in app-db [:account/account :address])
-                                 (assoc-in [:wallet :transactions-last-updated-at] (time/timestamp)))})))))
+                                    account-address)
+                             (update-in app-db
+                                        [:wallet :transactions]
+                                        #(merge-with dedupe-transactions % transactions)))))
+                  (done-fn))
+    :error-fn   (fn [http-error]
+                  (log/debug "Unable to get transactions: " http-error)
+                  (done-fn))}))
 
-(defn- time-to-sync? [cofx]
+(defn- sync-now! [{:keys [network-status :account/account app-state network web3] :as opts}]
-  (let [last-updated-at (get-in cofx [:db :wallet :transactions-last-updated-at])]
+  (when @polling-executor
-    (or (nil? last-updated-at)
+    (let [chain (ethereum/network->chain-keyword (get-in account [:networks network]))
-        (< sync-interval-ms
+          account-address (:address account)]
-           (- (time/timestamp) last-updated-at)))))
+      (when (and (not= network-status :offline)
-
-(fx/defn sync
-  "Fetch updated data for any unconfirmed transactions or incoming chat transactions missing in wallet
-  and schedule new recurring sync request"
-  [{:keys [db] :as cofx}]
-  (if (:account/account db)
-    (let [in-progress? (get-in db [:wallet :transactions-loading?])
-          {:keys [app-state network-status]} db]
-      (if (and (not= network-status :offline)
                  (= app-state "active")
-               (not in-progress?)
+                 (not= :custom chain))
-               (time-to-sync? cofx)
+        (async-periodic-run!
-               (or (have-unconfirmed-transactions? cofx)
+         @polling-executor
-                   (have-missing-chat-transactions? cofx)))
+         (partial transactions-query-helper web3 account-address chain))))))
-        (fx/merge cofx
-                  (run-update)
-                  (schedule-sync))
-        (schedule-sync cofx)))
-    (semaphores/free cofx :sync-wallet-transactions?)))
 
-(fx/defn start-sync [cofx]
+;; this function handles background syncing of transactions
-  (when-not (semaphores/locked? cofx :sync-wallet-transactions?)
+(defn- background-sync [web3 account-address done-fn]
-    (fx/merge cofx
+  (let [{:keys [network network-status :account/account app-state wallet chats]} @re-frame.db/app-db
-              (load-missing-chat-transactions)
+        chain (ethereum/network->chain-keyword (get-in account [:networks network]))]
-              (semaphores/lock :sync-wallet-transactions?)
+    (assert (and web3 account-address network network-status account app-state wallet chats)
-              (sync))))
+            "Must have all necessary data to run background transaction sync")
+    (if-not (and (not= network-status :offline)
+                 (= app-state "active")
+                 (not= :custom chain))
+      (done-fn)
+      (let [chat-transaction-ids (chat-map->transaction-ids network chats)
+            transaction-map (:transactions wallet)
+            transaction-ids (set (keys transaction-map))]
+        (if-not (or (have-unconfirmed-transactions? (vals transaction-map))
+                    (not-empty (set/difference chat-transaction-ids transaction-ids)))
+          (done-fn)
+          (transactions-query-helper web3 account-address chain done-fn))))))
+
+(defn- start-sync! [{:keys [:account/account network web3] :as options}]
+  (let [account-address (:address account)]
+    (when @polling-executor
+      (async-periodic-stop! @polling-executor))
+    (reset! polling-executor
+            (async-periodic-exec
+             (partial #'background-sync web3 account-address)
+             sync-interval-ms
+             sync-timeout-ms)))
+  (sync-now! options))
+
+(re-frame/reg-fx
+ ::sync-transactions-now
+ (fn [db] (sync-now! db)))
+
+(re-frame/reg-fx ::start-sync-transactions
+                 (fn [db] (start-sync! db)))
+
+(fx/defn start-sync [{:keys [db]}]
+  {::start-sync-transactions (select-keys db [:network-status :account/account :app-state :network :web3])})
+
+(re-frame/reg-fx
+ ::stop-sync-transactions
+ #(when @polling-executor
+    (async-periodic-stop! @polling-executor)))
+
+(fx/defn stop-sync [_]
+  {::stop-sync-transactions nil})

src/status_im/ui/screens/wallet/db.cljs

@@ -10,26 +10,25 @@
 (spec/def :wallet/send (spec/keys :req-un [:wallet.send/recipient]))
 
 (spec/def :wallet/balance-loading? (spec/nilable boolean?))
-(spec/def :wallet/transactions-loading? (spec/nilable boolean?))
-(spec/def :wallet/transactions-sync-started? (spec/nilable boolean?))
-(spec/def :wallet/errors (spec/nilable any?))
-(spec/def :wallet/transactions-last-updated-at (spec/nilable any?))
-(spec/def :wallet/chat-transactions (spec/nilable any?))
-(spec/def :wallet/transactions (spec/nilable any?))
-(spec/def :wallet/transactions-queue (spec/nilable any?))
-(spec/def :wallet/edit (spec/nilable any?))
-(spec/def :wallet/current-tab (spec/nilable any?))
-(spec/def :wallet/current-transaction (spec/nilable any?))
-(spec/def :wallet/modal-history? (spec/nilable any?))
-(spec/def :wallet/visible-tokens (spec/nilable any?))
-(spec/def :wallet/currency (spec/nilable any?))
-(spec/def :wallet/balance (spec/nilable any?))
 
-(spec/def :wallet/wallet (spec/keys :opt-un [:wallet/send-transaction :wallet/request-transaction
+;; TODO these key specs are not needed, they don't do anything
+(spec/def :wallet/errors any?)
+(spec/def :wallet/transactions any?)
+(spec/def :wallet/transactions-queue any?)
+(spec/def :wallet/edit any?)
+(spec/def :wallet/current-tab any?)
+(spec/def :wallet/current-transaction any?)
+(spec/def :wallet/modal-history? any?)
+(spec/def :wallet/visible-tokens any?)
+(spec/def :wallet/currency any?)
+(spec/def :wallet/balance any?)
+
+(spec/def :wallet/wallet (spec/keys :opt-un [:wallet/send-transaction
+                                             :wallet/request-transaction
                                              :wallet/transactions-queue
-                                             :wallet/balance-loading? :wallet/errors :wallet/transactions-loading?
+                                             :wallet/balance-loading?
-                                             :wallet/transactions-last-updated-at :wallet/chat-transactions
+                                             :wallet/errors
-                                             :wallet/transactions-sync-started? :wallet/transactions
+                                             :wallet/transactions
                                              :wallet/edit
                                              :wallet/current-tab
                                              :wallet/current-transaction

src/status_im/ui/screens/wallet/events.cljs

@@ -10,7 +10,6 @@
             [status-im.utils.handlers :as handlers]
             [status-im.utils.money :as money]
             [status-im.utils.prices :as prices]
-            [status-im.utils.transactions :as transactions]
             [taoensso.timbre :as log]
             [status-im.utils.fx :as fx]))
 
@@ -61,21 +60,6 @@
                            :on-success #(re-frame/dispatch [success-event symbol %])
                            :on-error   #(re-frame/dispatch [error-event symbol %])})))))
 
-(re-frame/reg-fx
- :get-transactions
- (fn [{:keys [web3 chain account-id token-addresses success-event error-event]}]
-   (transactions/get-transactions chain
-                                  account-id
-                                  #(re-frame/dispatch [success-event % account-id])
-                                  #(re-frame/dispatch [error-event %]))
-   (doseq [direction [:inbound :outbound]]
-     (erc20/get-token-transactions web3
-                                   chain
-                                   token-addresses
-                                   direction
-                                   account-id
-                                   #(re-frame/dispatch [success-event % account-id])))))
-
 ;; TODO(oskarth): At some point we want to get list of relevant assets to get prices for
 (re-frame/reg-fx
  :get-prices
@@ -104,55 +88,9 @@
 
 (handlers/register-handler-fx
  :update-transactions
- (fn [cofx _]
+ (fn [{:keys [db]} _]
-   (wallet.transactions/run-update cofx)))
+   {::wallet.transactions/sync-transactions-now
-
+    (select-keys db [:network-status :account/account :app-state :network :web3])}))
-(defn combine-entries [transaction token-transfer]
-  (merge transaction (select-keys token-transfer [:symbol :from :to :value :type :token :transfer])))
-
-(defn update-confirmations [tx1 tx2]
-  (assoc tx1 :confirmations (max (:confirmations tx1)
-                                 (:confirmations tx2))))
-
-(defn- tx-and-transfer?
-  "A helper function that checks if first argument is a transaction and the second argument a token transfer object."
-  [tx1 tx2]
-  (and (not (:transfer tx1)) (:transfer tx2)))
-
-(defn- both-transfer?
-  [tx1 tx2]
-  (and (:transfer tx1) (:transfer tx2)))
-
-(defn dedupe-transactions [tx1 tx2]
-  (cond (tx-and-transfer? tx1 tx2) (combine-entries tx1 tx2)
-        (tx-and-transfer? tx2 tx1) (combine-entries tx2 tx1)
-        (both-transfer? tx1 tx2)   (update-confirmations tx1 tx2)
-        :else tx2))
-
-(defn own-transaction? [address [_ {:keys [type to from]}]]
-  (let [normalized (ethereum/normalized-address address)]
-    (or (and (= :inbound type) (= normalized (ethereum/normalized-address to)))
-        (and (= :outbound type) (= normalized (ethereum/normalized-address from)))
-        (and (= :failed type) (= normalized (ethereum/normalized-address from))))))
-
-(handlers/register-handler-fx
- :update-transactions-success
- (fn [{:keys [db]} [_ transactions address]]
-   ;; NOTE(goranjovic): we want to only show transactions that belong to the current account
-   ;; this filter is to prevent any late transaction updates initated from another account on the same
-   ;; device from being applied in the current account.
-   (let [own-transactions (into {} (filter #(own-transaction? address %) transactions))]
-     {:db (-> db
-              (update-in [:wallet :transactions] #(merge-with dedupe-transactions % own-transactions))
-              (assoc-in [:wallet :transactions-loading?] false))})))
-
-(handlers/register-handler-fx
- :update-transactions-fail
- (fn [{:keys [db]} [_ err]]
-   (log/debug "Unable to get transactions: " err)
-   {:db (-> db
-            (assoc-error-message :transactions-update :error-unable-to-get-transactions)
-            (assoc-in [:wallet :transactions-loading?] false))}))
 
 (handlers/register-handler-fx
  :update-balance-success

src/status_im/ui/screens/wallet/send/events.cljs

@@ -269,7 +269,3 @@
              {:dispatch-later [{:ms 400 :dispatch [:check-dapps-transactions-queue]}]}
              (navigation/navigate-back))))
 
-(handlers/register-handler-fx
- :sync-wallet-transactions
- (fn [cofx _]
-   (wallet.transactions/sync cofx)))

src/status_im/ui/screens/wallet/transactions/subs.cljs

@@ -4,16 +4,11 @@
             [status-im.utils.datetime :as datetime]
             [status-im.utils.hex :as utils.hex]
             [status-im.utils.money :as money]
-            [status-im.utils.transactions :as transactions]
+            [status-im.models.transactions :as transactions]
             [status-im.utils.ethereum.core :as ethereum]
             [status-im.utils.ethereum.tokens :as tokens]
             [status-im.ui.screens.wallet.utils :as wallet.utils]))
 
-(reg-sub :wallet.transactions/transactions-loading?
-         :<- [:wallet]
-         (fn [wallet]
-           (:transactions-loading? wallet)))
-
 (reg-sub :wallet.transactions/current-tab
          :<- [:wallet]
          (fn [wallet]
@@ -130,8 +125,3 @@
 (reg-sub :wallet.transactions/filters
          (fn [db]
            (get-in db [:wallet.transactions :filters])))
-
-(reg-sub :wallet.transactions/error-message?
-         :<- [:wallet]
-         (fn [wallet]
-           (get-in wallet [:errors :transactions-update])))

src/status_im/utils/ethereum/erc20.cljs

@@ -17,10 +17,6 @@
   "
   (:require [status-im.utils.ethereum.core :as ethereum]
             [status-im.native-module.core :as status]
-            [status-im.utils.ethereum.tokens :as tokens]
-            [status-im.constants :as constants]
-            [status-im.utils.datetime :as datetime]
-            [clojure.string :as string]
             [status-im.utils.security :as security]
             [status-im.utils.types :as types])
   (:refer-clojure :exclude [name symbol]))
@@ -67,115 +63,3 @@
   (ethereum/call web3
                  (ethereum/call-params contract "allowance(address,address)" (ethereum/normalized-address owner-address) (ethereum/normalized-address spender-address))
                  #(cb %1 (ethereum/hex->bignumber %2))))
-
-(defn- parse-json [s]
-  (try
-    (let [res (-> s
-                  js/JSON.parse
-                  (js->clj :keywordize-keys true))]
-      (if (= (:error res) "")
-        {:result true}
-        res))
-    (catch :default e
-      {:error (.-message e)})))
-
-(defn- add-padding [address]
-  (when address
-    (str "0x000000000000000000000000" (subs address 2))))
-
-(defn- remove-padding [topic]
-  (if topic
-    (str "0x" (subs topic 26))))
-
-(defn- parse-transaction-entries [current-block-number block-info chain direction transfers]
-  (into {}
-        (keep identity
-              (for [transfer transfers]
-                (if-let [token (->> transfer :address (tokens/address->token chain))]
-                  (when-not (:nft? token)
-                    [(:transactionHash transfer)
-                     {:block         (-> block-info :number str)
-                      :hash          (:transactionHash transfer)
-                      :symbol        (:symbol token)
-                      :from          (-> transfer :topics second remove-padding)
-                      :to            (-> transfer :topics last remove-padding)
-                      :value         (-> transfer :data ethereum/hex->bignumber)
-                      :type          direction
-
-                      :confirmations (str (- current-block-number (-> transfer :blockNumber ethereum/hex->int)))
-
-                      :gas-price     nil
-                      :nonce         nil
-                      :data          nil
-
-                      :gas-limit     nil
-                      :timestamp     (-> block-info :timestamp (* 1000) str)
-
-                      :gas-used      nil
-
-                      ;; NOTE(goranjovic) - metadata on the type of token: contains name, symbol, decimas, address.
-                      :token         token
-
-                      ;; NOTE(goranjovic) - if an event has been emitted, we can say there was no error
-                      :error?        false
-
-                      ;; NOTE(goranjovic) - just a flag we need when we merge this entry with the existing entry in
-                      ;; the app, e.g. transaction info with gas details, or a previous transfer entry with old
-                      ;; confirmations count.
-                      :transfer      true}]))))))
-
-(defn add-block-info [web3 current-block-number chain direction result success-fn]
-  (let [transfers-by-block (group-by :blockNumber result)]
-    (doseq [[block-number transfers] transfers-by-block]
-      (ethereum/get-block-info web3 (ethereum/hex->int block-number)
-                               (fn [block-info]
-                                 (success-fn (parse-transaction-entries current-block-number
-                                                                        block-info
-                                                                        chain
-                                                                        direction
-                                                                        transfers)))))))
-
-(defn- response-handler [web3 current-block-number chain direction error-fn success-fn]
-  (fn handle-response
-    ([response]
-     (let [{:keys [error result]} (parse-json response)]
-       (handle-response error result)))
-    ([error result]
-     (if error
-       (error-fn error)
-       (add-block-info web3 current-block-number chain direction result success-fn)))))
-
-;;
-;; Here we are querying event logs for Transfer events.
-;;
-;; The parameters are as follows:
-;; - address - token smart contract address
-;; - fromBlock - we need to specify it, since default is latest
-;; - topics[0] - hash code of the Transfer event signature
-;; - topics[1] - address of token sender with leading zeroes padding up to 32 bytes
-;; - topics[2] - address of token sender with leading zeroes padding up to 32 bytes
-;;
-
-(defn get-token-transfer-logs
-  ;; NOTE(goranjovic): here we use direct JSON-RPC calls to get event logs because of web3 event issues with infura
-  ;; we still use web3 to get other data, such as block info
-  [web3 current-block-number chain contracts direction address cb]
-  (let [[from to] (if (= :inbound direction)
-                    [nil (ethereum/normalized-address address)]
-                    [(ethereum/normalized-address address) nil])
-        args {:jsonrpc "2.0"
-              :id      2
-              :method  constants/web3-get-logs
-              :params  [{:address   (map string/lower-case contracts)
-                         :fromBlock "0x0"
-                         :topics    [constants/event-transfer-hash
-                                     (add-padding from)
-                                     (add-padding to)]}]}
-        payload (.stringify js/JSON (clj->js args))]
-    (status/call-private-rpc payload
-                             (response-handler web3 current-block-number chain direction ethereum/handle-error cb))))
-
-(defn get-token-transactions
-  [web3 chain contracts direction address cb]
-  (ethereum/get-block-number web3
-                             #(get-token-transfer-logs web3 % chain contracts direction address cb)))

src/status_im/utils/ethereum/tokens.cljs

@@ -495,12 +495,15 @@
                    {:symbol   :KDO
                     :nft?     true
                     :name     "KudosToken"
-                    :address "0x93bB0AFbd0627Bbd3a6C72Bc318341D3A22e254a"}])
+                    :address  "0x93bb0afbd0627bbd3a6c72bc318341d3a22e254a"}])
 
    :custom  []})
 
-(defn tokens-for [chain]
+(defn tokens-for
-  (get all chain))
+  "makes sure all addresses are lower-case
+   TODO: token list should be speced and not accept non-lower-cased addresses"
+  [chain]
+  (mapv #(update % :address string/lower-case) (get all chain)))
 
 (defn all-assets-for [chain]
   (concat [(native-currency chain)]

src/status_im/utils/transactions.cljs

@@ -1,69 +0,0 @@
-(ns status-im.utils.transactions
-  (:require [status-im.utils.http :as http]
-            [status-im.utils.types :as types]
-            [taoensso.timbre :as log]))
-
-(def etherscan-supported?
-  #{:testnet :mainnet :rinkeby})
-
-(defn- get-network-subdomain [chain]
-  (case chain
-    (:testnet) "ropsten"
-    (:mainnet) nil
-    (:rinkeby) "rinkeby"))
-
-(defn get-transaction-details-url [chain hash]
-  (when (etherscan-supported? chain)
-    (let [network-subdomain (get-network-subdomain chain)]
-      (str "https://" (when network-subdomain (str network-subdomain ".")) "etherscan.io/tx/" hash))))
-
-(def etherscan-api-key "DMSI4UAAKUBVGCDMVP3H2STAMSAUV7BYFI")
-
-(defn- get-api-network-subdomain [chain]
-  (case chain
-    (:testnet) "api-ropsten"
-    (:mainnet) "api"
-    (:rinkeby) "api-rinkeby"))
-
-(defn get-transaction-url [chain account]
-  (let [network-subdomain (get-api-network-subdomain chain)]
-    (str "https://" network-subdomain ".etherscan.io/api?module=account&action=txlist&address=0x"
-         account "&startblock=0&endblock=99999999&sort=desc&apikey=" etherscan-api-key "?q=json")))
-
-(defn- format-transaction [account {:keys [value timeStamp blockNumber hash from to gas gasPrice gasUsed nonce confirmations input isError]}]
-  (let [inbound? (= (str "0x" account) to)
-        error?   (= "1" isError)]
-    {:value         value
-     ;; timestamp is in seconds, we convert it in ms
-     :timestamp     (str timeStamp "000")
-     :symbol        :ETH
-     :type          (cond error?   :failed
-                          inbound? :inbound
-                          :else    :outbound)
-     :block         blockNumber
-     :hash          hash
-     :from          from
-     :to            to
-     :gas-limit     gas
-     :gas-price     gasPrice
-     :gas-used      gasUsed
-     :nonce         nonce
-     :confirmations confirmations
-     :data          input}))
-
-(defn- format-transactions-response [response account]
-  (->> response
-       types/json->clj
-       :result
-       (reduce (fn [transactions {:keys [hash] :as transaction}]
-                 (assoc transactions hash (format-transaction account transaction)))
-               {})))
-
-(defn get-transactions [chain account on-success on-error]
-  (if (etherscan-supported? chain)
-    (let [url (get-transaction-url chain account)]
-      (log/debug "HTTP GET" url)
-      (http/get url
-                #(on-success (format-transactions-response % account))
-                on-error))
-    (log/info "Etherscan not supported for " chain)))

test/cljs/status_im/test/runner.cljs

@@ -7,6 +7,7 @@
             [status-im.test.browser.core]
             [status-im.test.browser.permissions]
             [status-im.test.wallet.subs]
+            [status-im.test.wallet.transactions]
             [status-im.test.wallet.transactions.subs]
             [status-im.test.wallet.transactions.views]
             [status-im.test.mailserver.core]
@@ -84,6 +85,7 @@
  'status-im.test.models.network
  'status-im.test.models.wallet
  'status-im.test.wallet.subs
+ 'status-im.test.wallet.transactions
  'status-im.test.wallet.transactions.subs
  'status-im.test.wallet.transactions.views
  'status-im.test.chat.models.loading

test/cljs/status_im/test/utils/ethereum/eip681.cljs

@@ -37,7 +37,7 @@
          (eip681/generate-erc20-uri "0x89205a3a3b2a69de6dbf7f01ed13b2108b2c43e7" {:symbol :SNT :value 5 :gas 10000 :gasPrice 10000})))
   (is (= "ethereum:0x744d70fdbe2ba4cf95131626614a1763df805b9e/transfer?uint256=5&address=0x89205a3a3b2a69de6dbf7f01ed13b2108b2c43e7"
          (eip681/generate-erc20-uri "0x89205a3a3b2a69de6dbf7f01ed13b2108b2c43e7" {:symbol :SNT :chain-id 1 :value 5})))
-  (is (= "ethereum:0xc55cF4B03948D7EBc8b9E8BAD92643703811d162@3/transfer?uint256=5&address=0x89205a3a3b2a69de6dbf7f01ed13b2108b2c43e7"
+  (is (= "ethereum:0xc55cf4b03948d7ebc8b9e8bad92643703811d162@3/transfer?uint256=5&address=0x89205a3a3b2a69de6dbf7f01ed13b2108b2c43e7"
          (eip681/generate-erc20-uri "0x89205a3a3b2a69de6dbf7f01ed13b2108b2c43e7" {:symbol :STT :chain-id 3 :value 5}))))
 
 (deftest generate-uri