Merge branch 'fix/token-formatting' of github.com:status-im/open-bounty into fix/token-formatting

Dockerfile

@@ -23,7 +23,12 @@ RUN lein uberjar
 FROM clojure
 WORKDIR /root/
 
+RUN apt-get update
+RUN apt-get -y install xvfb
+RUN apt-get -y install wkhtmltopdf
+
 COPY --from=builder /usr/src/app/target/uberjar/commiteth.jar .
+COPY html2png.sh .
 
 CMD [""]
 ENTRYPOINT ["/usr/bin/java", "-Duser.timezone=UTC", "-Dconf=config-test.edn", "-jar", "/root/commiteth.jar"]

README.md

@@ -1,4 +1,5 @@
 # Status Open Bounty
+[![Riot Chat Badge](https://img.shields.io/badge/join%20%23openbounty-riot-green.svg)](https://chat.status.im/#/room/#openbounty:status.im)
 
 Allows you to set bounties for Github issues, paid out in Ether or any ERC-20 token.
 
@@ -71,6 +72,7 @@ eth-account | Ethereum account ID for the bot
 eth-password | Ethereum account password for the bot
 eth-rpc-url | RPC URL to Ethereum node, e.g. Geth. Either local or remote
 eth-wallet-file | Location of wallet file. If Geth is run with the parameters as given below, it will reside under `$HOME/.ropsten/keystore`
+offline-signing | Specifies whether to sign transactions locally before sending. Default is true. Set to false when connecting to local Geth node that unlocks accounts
 tokenreg-base-format | Should be set to `:status`
 github-client-id | Related to OAuth. Copied from GitHub account Settings->Developer settings->OAuth Apps
 github-client-secret | Related to OAuth. Copied from GitHub account Settings->Developer settings->OAuth Apps
@@ -91,14 +93,22 @@ Follow the steps [here](https://developer.github.com/apps/building-github-apps/c
 
 ## Running
 
-### Geth
+### Ethereum node
-Launch a local geth node with the bot account unlocked:
+There are two options for connecting to an Ethereum node: either run a local node with an unlocked account, or connect to a remote Geth node or Infura. We will be connecting to Ropsten which is a test Ethereum network.
+
+#### Local
+
+In order to launch a local geth node with the bot account unlocked issue the following command:
 
 ```
 #!/bin/bash
 geth --fast --testnet --cache=1024 --datadir=$HOME/.ropsten --verbosity 4 --port 50100 --ipcpath ~/.ropsten/geth.ipc --rpc --rpcaddr 127.0.0.1 --rpcport 8545 --rpcapi db,eth,net,web3,personal --rpccorsdomain "https://wallet.ethereum.org" --unlock "0xYOUR_ADDR" --password <(echo "YOUR_PASSPHRASE")
 ```
 
+#### Remote
+Register at [Infura](https://infura.io/signup). You will receive an email with provider URLs. Paste an URL for the Ropsten network into `config.edn` under `:eth-rpc-url` key, and set `:offline-signing` to true.
+
+
 ### CSS auto-compilation
 Launch the following command in a separate shell:
 

env/dev/resources/config.edn

@@ -14,6 +14,9 @@
  :eth-rpc-url            "http://localhost:8545"
  :eth-wallet-file        "/some/location"
 
+ ;; Specifies whether to sign transactions before sending. Should be false for local Geth, true for remote Infura
+ :offline-signing true
+
  ;; address of token registry to be used
  ;; this is the default value for ropsten
  :tokenreg-addr "0x7d127a3e3b5e72cd8f15e7dee650abe4fcced2b9"

html2png.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -eu
+# need to run wkhtmltoimage in quiet mode because it misuses stdout
+xvfb-run -a -s "-screen 0 640x480x16" wkhtmltoimage -q "$@" 2>/dev/null

src/clj/commiteth/eth/core.clj

@@ -7,14 +7,65 @@
             [clojure.tools.logging :as log]
             [clojure.string :as str]
             [pandect.core :as pandect]
-            [commiteth.util.util :refer [json-api-request]]))
+            [commiteth.util.util :refer [json-api-request]])
+  (:import [org.web3j
+            protocol.Web3j
+            protocol.http.HttpService
+            protocol.core.DefaultBlockParameterName
+            protocol.core.methods.response.EthGetTransactionCount
+            protocol.core.methods.request.RawTransaction
+            utils.Numeric
+            crypto.Credentials
+            crypto.TransactionEncoder
+            crypto.WalletUtils]))
 
 (defn eth-rpc-url [] (env :eth-rpc-url "http://localhost:8545"))
 (defn eth-account [] (:eth-account env))
 (defn eth-password [] (:eth-password env))
 (defn gas-estimate-factor [] (env :gas-estimate-factor 1.0))
-(defn auto-gas-price? [] (env :auto-gas-price? false))
+(defn auto-gas-price? [] (env :auto-gas-price false))
+(defn offline-signing? [] (env :offline-signing true))
 
+(defn wallet-file-path []
+  (env :eth-wallet-file))
+
+(defn wallet-password []
+  (env :eth-password))
+
+(defn creds []
+  (let [password  (wallet-password)
+        file-path (wallet-file-path)]
+    (if (and password file-path)
+      (WalletUtils/loadCredentials
+       password
+       file-path)
+      (throw (ex-info "Make sure you provided proper credentials in appropriate resources/config.edn"
+                      {:password password :file-path file-path})))))
+
+(defn create-web3j []
+  (Web3j/build (HttpService. (eth-rpc-url))))
+
+(defn get-signed-tx [gas-price gas-limit to data]
+  "Create a sign a raw transaction.
+   'From' argument is not needed as it's already
+   encoded in credentials.
+   See https://web3j.readthedocs.io/en/latest/transactions.html#offline-transaction-signing"
+  (let [web3j (create-web3j)
+        nonce (.. (.ethGetTransactionCount web3j 
+                                           (env :eth-account) 
+                                           DefaultBlockParameterName/LATEST)
+                  sendAsync
+                  get
+                  getTransactionCount)
+        tx (RawTransaction/createTransaction
+             nonce
+             gas-price
+             gas-limit
+             to
+             data)
+        signed (TransactionEncoder/signMessage tx (creds))
+        hex-string (Numeric/toHexString signed)]
+    hex-string))
 (defn eth-gasstation-gas-price
   []
   (let [data (json-api-request "https://ethgasstation.info/json/ethgasAPI.json")
@@ -40,6 +91,21 @@
         (gas-price-from-config)))
     (gas-price-from-config)))
 
+(defn safe-read-str [s]
+  (if (nil? s)
+    (do
+      (log/error "JSON response is nil")
+      nil)
+    (try
+      (json/read-str s :key-fn keyword)
+      (catch Exception ex
+        (do (log/error "Exception when parsing json string:"
+                       s
+                       "message:"
+                       ex)
+
+            nil)))))
+
 (defn eth-rpc
   [method params]
   (let [request-id (rand-int 4096)
@@ -49,8 +115,8 @@
                                   :id      request-id})
         options  {:headers {"content-type" "application/json"}
                   :body body}
-        response (:body @(post (eth-rpc-url) options))
+        response  @(post (eth-rpc-url) options)
-        result   (json/read-str response :key-fn keyword)]
+        result   (safe-read-str (:body response))]
     (log/debug body "\n" result)
 
     (if (= (:id result) request-id)
@@ -128,54 +194,16 @@
   [account digits]
   (hex->eth (get-balance-hex account) digits))
 
-(defn send-transaction
-  "Send transaction using default commiteth bot account."
-  [from to value & [params]]
-  (let [args (merge params
-                    {:from  from
-                     :value value}
-                    (when-not (nil? (gas-price))
-                      {:gasPrice (integer->hex (gas-price))})
-                    (when-not (contains? params :gas)
-                      {:gas
-                       (estimate-gas from to value params)}))]
-    (log/debug "args:" args)
-    (eth-rpc
-     "personal_sendTransaction"
-     [(if-not (nil? to)
-        (merge args {:to to})
-        args)
-      (eth-password)])))
-
-(defn send-transaction-using-from-account
-  "Send transaction using account address in parameter from. Assumes
-  account has been unlocked."
-  [from to value & [params]]
-  (let [args (merge params
-                    {:from  from
-                     :value value}
-                    (when-not (nil? (gas-price))
-                      {:gasPrice (integer->hex (gas-price))})
-                    (when-not (contains? params :gas)
-                      {:gas
-                       (estimate-gas from to value params)}))]
-    (log/debug "args:" args)
-    (eth-rpc
-     "eth_sendTransaction"
-     [(if-not (nil? to)
-        (merge args {:to to})
-        args)])))
-
-(defn get-transaction-receipt
-  [hash]
-  (eth-rpc "eth_getTransactionReceipt" [hash]))
-
 (defn- format-param
   [param]
   (if (number? param)
     (format "%064x" param)
     (clojure.string/replace (format "%64s" (subs param 2)) " " "0")))
 
+(defn get-transaction-receipt
+  [hash]
+  (eth-rpc "eth_getTransactionReceipt" [hash]))
+
 (defn format-call-params
   [method-id & params]
   (let [params (join (map format-param params))]
@@ -189,33 +217,30 @@
 (defn execute
   [from contract method-id gas-limit & params]
   (let [data (apply format-call-params method-id params)
-        value (format "0x%x" 0)]
+        gas-price (gas-price)
-    (send-transaction from contract value (merge
+        value (format "0x%x" 0)
-                                           {:data data}
+        params (cond-> {:data data
-                                           (when gas-limit
+                        :from  from
-                                             {:gas gas-limit})))))
+                        :value value}
-
+                 gas-price
-(defn execute-using-addr
+                 (merge {:gasPrice (integer->hex gas-price)})
-  [from-addr from-passphrase contract method-id & params]
-  (eth-rpc "personal_unlockAccount" [from-addr from-passphrase 30])
-  (let [data (apply format-call-params method-id params)
-        value (format "0x%x" 0)]
-    (send-transaction-using-from-account from-addr
                  contract
-                                         value
+                 (merge {:to contract}))
-                                         {:data data})))
+        gas (if gas-limit gas-limit 
-
+              (estimate-gas from contract value params))
-(defn transfer-eth
+        params (if (offline-signing?)
-  "Transfer amount-wei of ETH from from-addr to to-addr."
+                 (get-signed-tx (biginteger gas-price)
-  [from-addr from-passphrase to-addr amount-wei]
+                                      (hex->big-integer gas)
-  (eth-rpc "personal_unlockAccount" [from-addr from-passphrase 30])
+                                      contract
-  (let [data "0x"
+                                      data) 
-        value (integer->hex amount-wei)]
+                 (assoc params :gas gas))]
-    (send-transaction-using-from-account from-addr
+    (if (offline-signing?)
-                                         to-addr
+      (eth-rpc
-                                         value
+        "eth_sendRawTransaction"
-                                         {:data data})))
+        [params])
-
+      (eth-rpc
+        "personal_sendTransaction"
+        [params (eth-password)]))))
 
 (defn hex-ch->num
   [ch]

src/clj/commiteth/eth/multisig_wallet.clj

@@ -1,8 +1,7 @@
 (ns commiteth.eth.multisig-wallet
-  (:require [commiteth.eth.core :as eth]
+  (:require [commiteth.eth.core :as eth
-            [commiteth.config :refer [env]]
-            [commiteth.eth.web3j
              :refer [create-web3j creds]]
+            [commiteth.config :refer [env]]
             [clojure.tools.logging :as log]
             [commiteth.eth.token-data :as token-data])
   (:import [org.web3j
@@ -191,30 +190,6 @@
                                  [tla (token-balance bounty-addr tla)]))) addrs)))
       {})))
 
-(defn transfer-tokens
-  "Transfer mount of given ERC20 token from from-addr to
-  to-addr. Connected geth needs to have keys for the account and
-  passphrase needs to be supplied. Returns transaction ID."
-  [from-addr from-passphrase token to-addr amount]
-  (let [token-addr (get-token-address token)]
-    (eth/execute-using-addr from-addr
-                            from-passphrase
-                            token-addr
-                            (method-ids :transfer)
-                            to-addr
-                            amount)))
-
-(defn get-owners
-  "Return vector of multisig owner addresses."
-  [bounty-addr]
-  (let [bounty-contract (load-bounty-contract bounty-addr)
-        owner-addresses (-> bounty-contract
-                            (.getOwners)
-                            .get)]
-    (if owner-addresses
-      (mapv #(.toString %) (.getValue owner-addresses))
-      [])))
-
 (defn uint256 [x]
   (org.web3j.abi.datatypes.generated.Uint256. x))
 

src/clj/commiteth/eth/token_data.clj

@@ -15,7 +15,7 @@
 (defn update-data []
   (let [test-data (env :testnet-token-data)
         token-data
-        (if (and (env :on-testnet true)
+        (if (and #_(env :on-testnet true)
                  test-data)
           test-data
           (into {}

src/clj/commiteth/eth/token_registry.clj

@@ -1,6 +1,5 @@
 (ns commiteth.eth.token-registry
-  (:require [commiteth.eth.core :as eth]
+  (:require [commiteth.eth.core :as eth
-            [commiteth.eth.web3j
              :refer [create-web3j creds]]
             [commiteth.config :refer [env]]
             [clojure.tools.logging :as log])

src/clj/commiteth/eth/web3j.clj

@@ -1,28 +0,0 @@
-(ns commiteth.eth.web3j
-    (:require [commiteth.eth.core :as eth]
-            [commiteth.config :refer [env]])
-  (:import [org.web3j
-            protocol.Web3j
-            protocol.http.HttpService
-            crypto.Credentials
-            crypto.WalletUtils]))
-
-
-(defn wallet-file-path []
-  (env :eth-wallet-file))
-
-(defn wallet-password []
-  (env :eth-password))
-
-(defn creds []
-  (let [password  (wallet-password)
-        file-path (wallet-file-path)]
-    (if (and password file-path)
-      (WalletUtils/loadCredentials
-       password
-       file-path)
-      (throw (ex-info "Make sure you provided proper credentials in appropriate resources/config.edn"
-                      {:password password :file-path file-path})))))
-
-(defn create-web3j []
-  (Web3j/build (HttpService. (eth/eth-rpc-url))))

src/clj/commiteth/github/core.clj

@@ -3,6 +3,7 @@
              [core :as tentacles]
              [repos :as repos]
              [oauth :as oauth]
+             [search :as search]
              [users :as users]
              [repos :as repos]
              [issues :as issues]
@@ -27,6 +28,8 @@
 (defn on-testnet? [] (env :on-testnet))
 (defn webhook-secret [] (env :webhook-secret))
 
+
+
 (defn authorize-url [scope]
   (let [params (codec/form-encode {:client_id    (client-id)
                                    :redirect_uri (redirect-uri)
@@ -360,3 +363,32 @@
   (let [[owner repo] (str/split full-repo #"/")]
     (log/debug "creating bounty label" (str owner "/" repo) token)
     (issues/create-label owner repo "bounty" "fafad2" (auth-params token))))
+
+(defn get-labeled-issues-for-repos [repos auth-params]
+  "Find all issues with a bounty label in provided repos"
+  (let [get-last-part (fn get-last-part [s]
+                        (subs s (inc (str/last-index-of s "/"))))
+        get-issue-info (fn get-issue-info [r issue]
+                         (hash-map :owner r
+                                   :repo (get-last-part (:repository_url issue))
+                                   :id (:id issue)
+                                   :number (:number issue)
+                                   :title (:title issue)
+                                   :url (:html_url issue)
+                                   :created_at (:created_at issue)
+                                   :closed_at (:closed_at issue))) 
+        issues (for [r repos]
+                 (loop [repo-issues [] i 1]
+                   (let [params (into auth-params
+                                      {:sort "created" :order "desc" :page i})
+                         issues-page (-> (search/search-issues 
+                                           nil
+                                           {:label "bounty" :user r :type "issue"}
+                                           params)
+                                         :items)]
+                     (if (first issues-page)
+                       (recur (into repo-issues 
+                                    (map (partial get-issue-info r) issues-page)) 
+                              (inc i))
+                       repo-issues))))]
+    (apply concat issues)))

src/clj/commiteth/routes/redirect.clj

@@ -39,6 +39,7 @@
           body         (keywordize-keys (codec/form-decode (:body resp)))
           scope        (:scope body)
           access-token (:access_token body)]
+      (log/info "access-token:" access-token)
       (log/debug "github sign-in callback, response body:" body)
       (if (:error body)
         ;; Why does Mist browser sends two redirects at the same time? The latter results in 401 error.

src/clj/commiteth/scheduler.clj

@@ -15,13 +15,16 @@
             [clj-time.periodic :refer [periodic-seq]]
             [chime :refer [chime-at]]))
 
+
 (defn update-issue-contract-address
   "For each pending deployment: gets transaction receipt, updates db
   state (contract-address, comment-id) and posts github comment"
   []
+  (log/info "In update-issue-contract-address")
   (doseq [{issue-id         :issue_id
            transaction-hash :transaction_hash} (issues/list-pending-deployments)]
-    (log/debug "pending deployment:" transaction-hash)
+    (log/info "pending deployment:" transaction-hash)
+    (try 
       (when-let [receipt (eth/get-transaction-receipt transaction-hash)]
         (log/info "update-issue-contract-address: transaction receipt for issue #"
                   issue-id ": " receipt)
@@ -51,7 +54,11 @@
                                    balance-eth
                                    balance-eth-str
                                    {}))
-        (log/error "Failed to find contract address in tx logs")))))
+          (log/error "Failed to find contract address in tx logs")))
+      (catch Throwable ex 
+        (do (log/error "update-issue-contract-address exception:" ex)
+            (clojure.stacktrace/print-stack-trace ex)))))
+  (log/info "Exit update-issue-contract-address"))
 
 
 (defn deploy-contract [owner-address issue-id]
@@ -85,6 +92,7 @@
 (defn self-sign-bounty
   "Walks through all issues eligible for bounty payout and signs corresponding transaction"
   []
+  (log/info "In self-sign-bounty")
   (doseq [{contract-address :contract_address
            issue-id         :issue_id
            payout-address   :payout_address
@@ -94,8 +102,9 @@
            issue-number :issue_number
            balance-eth :balance_eth
            tokens :tokens
-           winner-login :winner_login} (db-bounties/pending-bounties)
+           winner-login :winner_login} (db-bounties/pending-bounties)]
-          :let [value (eth/get-balance-hex contract-address)]]
+    (try
+      (let [value (eth/get-balance-hex contract-address)]
         (if (empty? payout-address)
           (do
             (log/error "Cannot sign pending bounty - winner has no payout address")
@@ -118,11 +127,17 @@
                                                 (eth-decimal->str balance-eth)
                                                 tokens
                                                 winner-login
-                                            false)))))
+                                                false))))
+      (catch Throwable ex 
+        (do (log/error "self-sign-bounty exception:" ex)
+            (clojure.stacktrace/print-stack-trace ex)))))
+  (log/info "Exit self-sign-bounty")
+  )
 
 (defn update-confirm-hash
   "Gets transaction receipt for each pending payout and updates DB confirm_hash with tranaction ID of commiteth bot account's confirmation."
   []
+  (log/info "In update-confirm-hash")
   (doseq [{issue-id     :issue_id
            execute-hash :execute_hash} (db-bounties/pending-payouts)]
     (log/info "pending payout:" execute-hash)
@@ -130,7 +145,8 @@
       (log/info "execution receipt for issue #" issue-id ": " receipt)
       (when-let [confirm-hash (multisig/find-confirmation-tx-id receipt)]
         (log/info "confirm hash:" confirm-hash)
-        (db-bounties/update-confirm-hash issue-id confirm-hash)))))
+        (db-bounties/update-confirm-hash issue-id confirm-hash))))
+  (log/info "Exit update-confirm-hash"))
 
 
 (defn update-watch-hash
@@ -154,6 +170,7 @@
 (defn update-payout-receipt
   "Gets transaction receipt for each confirmed payout and updates payout_hash"
   []
+  (log/info "In update-payout-receipt")
   (doseq [{issue-id    :issue_id
            payout-hash :payout_hash
            contract-address :contract_address
@@ -167,6 +184,7 @@
            payee-login :payee_login
            updated :updated} (db-bounties/confirmed-payouts)]
     (log/debug "confirmed payout:" payout-hash)
+    (try
       (if-let [receipt (eth/get-transaction-receipt payout-hash)]
         (let [contract-tokens (multisig/token-balances contract-address)
               contract-eth-balance (eth/get-balance-wei contract-address)]
@@ -192,7 +210,12 @@
                                                 payee-login))))
         (when (older-than-3h? updated)
           (log/info "Resetting payout hash for issue" issue-id "as it has not been mined in 3h")
-        (db-bounties/reset-payout-hash issue-id)))))
+          (db-bounties/reset-payout-hash issue-id)))
+      (catch Throwable ex 
+        (do (log/error "update-payout-receipt exception:" ex)
+            (clojure.stacktrace/print-stack-trace ex)))))
+  (log/info "Exit update-payout-receipt")
+  )
 
 (defn abs
   "(abs n) is the absolute value of n"
@@ -213,17 +236,24 @@
 (defn update-bounty-token-balances
   "Helper function for updating internal ERC20 token balances to token multisig contract. Will be called periodically for all open bounty contracts."
   [issue-id bounty-addr watch-hash]
+  #_(log/info "In update-bounty-token-balances for issue" issue-id)
   (doseq [[tla token-data] (token-data/as-map)]
+    (try
       (let [balance (multisig/token-balance bounty-addr tla)]
         (when (> balance 0)
           (do
-          (log/debug "bounty at" bounty-addr "has" balance "of token" tla)
+            (log/info "bounty at" bounty-addr "has" balance "of token" tla)
             (let [internal-balance (multisig/token-balance-in-bounty bounty-addr tla)]
               (when (and (nil? watch-hash)
                          (not= balance internal-balance))
                 (log/info "balances not in sync, calling watch")
                 (let [hash (multisig/watch-token bounty-addr tla)]
-                (db-bounties/update-watch-hash issue-id hash)))))))))
+                  (db-bounties/update-watch-hash issue-id hash)))))))
+      (catch Throwable ex 
+        (do (log/error "update-bounty-token-balances exception:" ex)
+            (clojure.stacktrace/print-stack-trace ex)))))
+  #_(log/info "Exit update-bounty-token-balances"))
+  
 
 (defn update-contract-internal-balances
   "It is required in our current smart contract to manually update it's internal balance when some tokens have been added."
@@ -263,6 +293,7 @@
 
 (defn update-balances
   []
+  (log/info "In update-balances")
   (doseq [{contract-address :contract_address
            owner            :owner
            repo             :repo
@@ -271,6 +302,7 @@
            db-balance-eth   :balance_eth
            db-tokens        :tokens
            issue-number     :issue_number} (db-bounties/open-bounty-contracts)]
+    (try
       (when comment-id
         (let [balance-eth-str (eth/get-balance-eth contract-address 6)
               balance-eth (read-string balance-eth-str)
@@ -307,7 +339,11 @@
                                    balance-eth
                                    balance-eth-str
                                    token-balances)
-          (update-issue-usd-value contract-address))))))
+            (update-issue-usd-value contract-address))))
+      (catch Throwable ex 
+        (do (log/error "update-balances exception:" ex)
+            (clojure.stacktrace/print-stack-trace ex)))))
+  (log/info "Exit update-balances"))
 
 
 (defn wrap-in-try-catch [func]
@@ -323,7 +359,7 @@
 
 (defn run-1-min-interval-tasks [time]
   (do
-    (log/debug "run-1-min-interval-tasks" time)
+    (log/info "run-1-min-interval-tasks" time)
     ;; TODO: disabled for now. looks like it may cause extraneus
     ;; contract deployments and costs
     (run-tasks