From 8cb5995d5ae2dcdb4ecadfb846aa752ab83730c3 Mon Sep 17 00:00:00 2001 From: Teemu Patja Date: Sat, 21 Jan 2017 18:06:37 +0200 Subject: [PATCH 1/6] Use SQL timestamp instead of time * Use SQL data type timestamp for users.created and repositories.updated since we want to store datetimes, not time of day * Fix :migration-dir in project file Fixes: #15 --- project.clj | 2 +- .../20170121175409-fix-timestamp-column-datatypes.down.sql | 0 .../20170121175409-fix-timestamp-column-datatypes.up.sql | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 resources/migrations/20170121175409-fix-timestamp-column-datatypes.down.sql create mode 100644 resources/migrations/20170121175409-fix-timestamp-column-datatypes.up.sql diff --git a/project.clj b/project.clj index 821593b..54e3a0f 100644 --- a/project.clj +++ b/project.clj @@ -48,7 +48,7 @@ :target-path "target/%s/" :main commiteth.core :migratus {:store :database - :migration-dir "resources/migrations" + :migration-dir "migrations" :db ~(get (System/getenv) "DATABASE_URL")} :plugins [[lein-cprop "1.0.1"] diff --git a/resources/migrations/20170121175409-fix-timestamp-column-datatypes.down.sql b/resources/migrations/20170121175409-fix-timestamp-column-datatypes.down.sql new file mode 100644 index 0000000..e69de29 diff --git a/resources/migrations/20170121175409-fix-timestamp-column-datatypes.up.sql b/resources/migrations/20170121175409-fix-timestamp-column-datatypes.up.sql new file mode 100644 index 0000000..2165aac --- /dev/null +++ b/resources/migrations/20170121175409-fix-timestamp-column-datatypes.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.users ALTER COLUMN created SET DATA TYPE timestamp without time zone using date('20170120') + created; +ALTER TABLE public.repositories ALTER COLUMN updated SET DATA TYPE timestamp without time zone using date('20170120') + updated; From d39ccd15313f1cc98855eb3a5893f4ce57ff7b8e Mon Sep 17 00:00:00 2001 From: Teemu Patja Date: Sat, 21 Jan 2017 19:09:45 +0200 Subject: [PATCH 2/6] Update test+prod config.edn files --- env/prod/resources/config.edn | 9 ++++++--- env/test/resources/config.edn | 10 ++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/env/prod/resources/config.edn b/env/prod/resources/config.edn index f5ce248..2a92f75 100644 --- a/env/prod/resources/config.edn +++ b/env/prod/resources/config.edn @@ -1,3 +1,6 @@ -{:production true - :port 3000 - :server-address "http://commiteth.com"} +{ + :production true + :port 3000 + :nrepl-port 7000 + :server-address "https://commiteth.com" +} diff --git a/env/test/resources/config.edn b/env/test/resources/config.edn index 40d4a13..6e65a3d 100644 --- a/env/test/resources/config.edn +++ b/env/test/resources/config.edn @@ -1,6 +1,4 @@ -{:test true - :port 3001 - ;; when :nrepl-port is set the application starts the nREPL server on load - :nrepl-port 7001 - :jdbc-database-url "jdbc:postgresql://localhost/commiteth?user=commiteth&password=commiteth" - :server-address "http://localhost:3000"} +{ + :test true + :jdbc-database-url "jdbc:postgresql://localhost/commiteth_test?user=commiteth&password=commiteth" +} From 1c26d17ad9111e89e844d6805ac32d3f963faa75 Mon Sep 17 00:00:00 2001 From: Teemu Patja Date: Sat, 21 Jan 2017 19:11:17 +0200 Subject: [PATCH 3/6] Run pending DB migrations at startup Run pending DB migrations at startup to make deployement simpler (eliminates need to clone repo and run "lein migratus migrate" on the server) --- src/clj/commiteth/db/core.clj | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/clj/commiteth/db/core.clj b/src/clj/commiteth/db/core.clj index cb9c69e..2af3561 100644 --- a/src/clj/commiteth/db/core.clj +++ b/src/clj/commiteth/db/core.clj @@ -4,7 +4,8 @@ [clojure.java.jdbc :as jdbc] [conman.core :as conman] [commiteth.config :refer [env]] - [mount.core :refer [defstate]]) + [mount.core :refer [defstate]] + [migratus.core :as migratus]) (:import org.postgresql.util.PGobject java.sql.Array clojure.lang.IPersistentMap @@ -15,8 +16,18 @@ Timestamp PreparedStatement])) + +(defn start [] + (let [db (env :jdbc-database-url) + migratus-config {:store :database + :migration-dir "migrations/" + :migration-table-name "schema_migrations" + :db db}] + (migratus/migrate migratus-config) + (conman/connect! {:jdbc-url db}))) + (defstate ^:dynamic *db* - :start (conman/connect! {:jdbc-url (env :jdbc-database-url)}) + :start (start) :stop (conman/disconnect! *db*)) (conman/bind-connection *db* "sql/queries.sql") From f89a83ea32156acb7f92b2785f96c657533c05d7 Mon Sep 17 00:00:00 2001 From: Teemu Patja Date: Sun, 22 Jan 2017 07:57:05 +0200 Subject: [PATCH 4/6] Create github issue label with different color --- src/clj/commiteth/github/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/commiteth/github/core.clj b/src/clj/commiteth/github/core.clj index 91f0bdb..c52a2d5 100644 --- a/src/clj/commiteth/github/core.clj +++ b/src/clj/commiteth/github/core.clj @@ -178,4 +178,4 @@ [full-repo token] (let [[user repo] (str/split full-repo #"/")] (log/debug "creating bounty label" (str user "/" repo) token) - (issues/create-label user repo "bounty" "00ff00" (auth-params token)))) + (issues/create-label user repo "bounty" "fafad2" (auth-params token)))) From 985b72754d670639ebf9e9d39badd92c8ce47eb6 Mon Sep 17 00:00:00 2001 From: Teemu Patja Date: Sun, 22 Jan 2017 10:59:40 +0200 Subject: [PATCH 5/6] Create bounties for existing issues when enabling a repository * Create bounties for existing bounty-tagged issues when a repository is toggled on * added commiteth.bounties ns for sharing code Fixes: #12 --- src/clj/commiteth/bounties.clj | 46 ++++++++++++++ src/clj/commiteth/core.clj | 3 +- src/clj/commiteth/github/core.clj | 8 ++- src/clj/commiteth/routes/services.clj | 18 +++--- src/clj/commiteth/routes/webhooks.clj | 88 +++++++++++++-------------- 5 files changed, 108 insertions(+), 55 deletions(-) create mode 100644 src/clj/commiteth/bounties.clj diff --git a/src/clj/commiteth/bounties.clj b/src/clj/commiteth/bounties.clj new file mode 100644 index 0000000..4c2926b --- /dev/null +++ b/src/clj/commiteth/bounties.clj @@ -0,0 +1,46 @@ +(ns commiteth.bounties + (:require [commiteth.db.issues :as issues] + [commiteth.db.users :as users] + [commiteth.db.repositories :as repos] + [commiteth.eth.core :as eth] + [commiteth.github.core :as github] + [commiteth.eth.core :as eth] + [clojure.tools.logging :as log])) + + +(def ^:const label-name "bounty") + +(defn has-bounty-label? + [issue] + (let [labels (:labels issue)] + (some #(= label-name (:name %)) labels))) + + +(defn add-bounty-for-issue [repo-map issue] + (log/debug "add-bounty-for-issue" issue) + (let [{issue-id :id + issue-number :number + issue-title :title} issue + {repo :repo + repo-id :repo_id + user :login} repo-map + created-issue (issues/create repo-id issue-id issue-number issue-title) + repo-owner (:address (users/get-repo-owner repo-id))] + (log/info (format "Issue %s/%s/%s labeled as bounty" user repo issue-number)) + (if (= 1 created-issue) + (let [transaction-hash (eth/deploy-contract repo-owner)] + (log/info "Contract deployed, transaction-hash:" transaction-hash ) + (issues/update-transaction-hash issue-id transaction-hash)) + (log/debug "Issue already exists in DB, ignoring")))) + + +(defn add-bounties-for-existing-issues [repo-map] + (let [{repo :repo + user :login} repo-map + issues (github/get-issues user repo) + bounty-issues (filter has-bounty-label? issues)] + (log/debug bounty-issues) + (log/debug "adding bounties for" + (count bounty-issues) " existing issues") + (doall + (map (partial add-bounty-for-issue repo-map) bounty-issues)))) diff --git a/src/clj/commiteth/core.clj b/src/clj/commiteth/core.clj index a59eabc..da9f353 100644 --- a/src/clj/commiteth/core.clj +++ b/src/clj/commiteth/core.clj @@ -10,7 +10,7 @@ [mount.core :as mount]) (:gen-class)) -(def cli-options +(def ^:const cli-options [["-p" "--port PORT" "Port number" :parse-fn #(Integer/parseInt %)]]) @@ -58,4 +58,3 @@ repl-server (System/exit 0)) :else (start-app args))) - diff --git a/src/clj/commiteth/github/core.clj b/src/clj/commiteth/github/core.clj index c52a2d5..84322ea 100644 --- a/src/clj/commiteth/github/core.clj +++ b/src/clj/commiteth/github/core.clj @@ -160,7 +160,8 @@ (defn update-comment [user repo comment-id issue-number balance] (let [comment (generate-comment user repo issue-number balance)] - (log/debug (str "Updating " user "/" repo "/" issue-number " comment #" comment-id " with contents: " comment)) + (log/debug (str "Updating " user "/" repo "/" issue-number + " comment #" comment-id " with contents: " comment)) (let [req (make-patch-request "repos/%s/%s/issues/comments/%s" [user repo comment-id] (assoc (self-auth-params) :body comment))] @@ -170,6 +171,11 @@ [user repo issue-number] (issues/specific-issue user repo issue-number (self-auth-params))) +(defn get-issues + [user repo] + (issues/issues user repo)) + + (defn get-issue-events [user repo issue-id] (issues/issue-events user repo issue-id (self-auth-params))) diff --git a/src/clj/commiteth/routes/services.clj b/src/clj/commiteth/routes/services.clj index 364fbb7..c636671 100644 --- a/src/clj/commiteth/routes/services.clj +++ b/src/clj/commiteth/routes/services.clj @@ -7,7 +7,8 @@ [buddy.auth :refer [authenticated?]] [commiteth.db.users :as users] [commiteth.db.repositories :as repositories] - [commiteth.db.bounties :as bounties] + [commiteth.db.bounties :as bounties-db] + [commiteth.bounties :as bounties] [commiteth.github.core :as github] [clojure.tools.logging :as log] [commiteth.eth.core :as eth])) @@ -56,13 +57,15 @@ :current-user user (ok (repositories/get-enabled (:id user)))) (GET "/bounties" [] - (ok (bounties/list-all-bounties))) + (ok (bounties-db/list-all-bounties))) (POST "/bounty/:issue{[0-9]{1,9}}/payout" {:keys [params]} :auth-rules authenticated? :current-user user (let [{issue :issue payout-hash :payout-hash} params - result (bounties/update-payout-hash (Integer/parseInt issue) payout-hash)] + result (bounties-db/update-payout-hash + (Integer/parseInt issue) + payout-hash)] (if (= 1 result) (ok) (internal-server-error)))) @@ -72,7 +75,7 @@ (ok (map #(conj % (let [balance (:balance %)] {:balance-eth (eth/hex->eth balance 6) :balance-wei (eth/hex->big-integer balance)})) - (bounties/list-owner-bounties (:id user))))) + (bounties-db/list-owner-bounties (:id user))))) (POST "/repository/toggle" {:keys [params]} :auth-rules authenticated? :current-user user @@ -85,10 +88,11 @@ (repositories/create (merge params {:user_id user-id})) (repositories/toggle repo-id))] (if (:enabled result) - ;; @todo: do we really want to make this call at this moment? (let [created-hook (github/add-webhook repo token)] (log/debug "Created webhook:" created-hook) - (github/create-label repo token) - (repositories/update-hook-id repo-id (:id created-hook))) + (future + (github/create-label repo token) + (repositories/update-hook-id repo-id (:id created-hook)) + (bounties/add-bounties-for-existing-issues result))) (github/remove-webhook repo (:hook_id result) token)) result))))) diff --git a/src/clj/commiteth/routes/webhooks.clj b/src/clj/commiteth/routes/webhooks.clj index 7a0a8af..ebc665a 100644 --- a/src/clj/commiteth/routes/webhooks.clj +++ b/src/clj/commiteth/routes/webhooks.clj @@ -4,44 +4,48 @@ [commiteth.db.pull-requests :as pull-requests] [commiteth.db.issues :as issues] [commiteth.db.users :as users] - [commiteth.eth.core :as eth] + [commiteth.bounties :as bounties] [ring.util.http-response :refer [ok]] [clojure.string :refer [join]] [clojure.tools.logging :as log]) (:import [java.lang Integer])) -(def label-name "bounty") (defn find-issue-event [events type owner] (first (filter #(and (= owner (get-in % [:actor :login])) (= type (:event %))) - events))) + events))) + + +(defn labeled-as-bounty? + [action issue] + (and + (= "labeled" action) + (= bounties/label-name (get-in issue [:label :name])))) (defn find-commit-id [user repo issue-number event-types] + (log/debug "find-commit-id" user repo issue-number event-types) (some identity (map #(-> (github/get-issue-events user repo issue-number) (find-issue-event % user) :commit_id) - event-types))) + event-types))) + (defn handle-issue-labeled - [issue] - (let [{repo-id :id - repo :name - {user :login} :owner} (:repository issue) - {issue-id :id - issue-number :number - issue-title :title} (:issue issue) - created-issue (issues/create repo-id issue-id issue-number issue-title) - repo-owner (:address (users/get-repo-owner repo-id))] - (log/debug (format "Issue %s/%s/%s labeled as bounty" user repo issue-number)) - (when (= 1 created-issue) - (issues/update-transaction-hash issue-id (eth/deploy-contract repo-owner))))) + [webhook-payload] + (let [{issue :issue} webhook-payload + {repo-id :id + repo-name :name + {login :login} :owner} (:repository webhook-payload) + repo-map {:repo repo-name :login login :repo_id repo-id}] + (bounties/add-bounty-for-issue repo-map issue))) (defn handle-issue-closed + ;; TODO: does not work in case the issue is closed on github web ui [{{{user :login} :owner repo :name} :repository {issue-id :id issue-number :number} :issue}] (future @@ -49,7 +53,7 @@ (log/debug (format "Issue %s/%s/%s closed with commit %s" user repo issue-number commit-id)) (issues/close commit-id issue-id)))) -(def keywords +(def ^:const keywords [#"(?i)close\s+#(\d+)" #"(?i)closes\s+#(\d+)" #"(?i)closed\s+#(\d+)" @@ -70,16 +74,12 @@ (catch NumberFormatException _))) (re-seq % pr-body)) keywords)) -(defn has-bounty-label? - [issue] - (let [labels (:labels issue)] - (some #(= label-name (:name %)) labels))) (defn validate-issue-number "Checks if an issue has a bounty label attached and returns its number" [user repo issue-number] (when-let [issue (github/get-issue user repo issue-number)] - (when (has-bounty-label? issue) + (when (bounties/has-bounty-label? issue) issue-number))) (defn handle-pull-request-closed @@ -92,40 +92,38 @@ id :id pr-number :number pr-body :body} :pull_request}] + (log/debug "handle-pull-request-closed" owner repo repo-id login pr-body) (future (let [commit-id (find-commit-id owner repo pr-number ["merged"]) issue-number (->> - (extract-issue-number pr-body) - (first) - (validate-issue-number owner repo)) + (extract-issue-number pr-body) + (first) + (validate-issue-number owner repo)) m {:commit_id commit-id :issue_number issue-number}] + (log/debug "handle-pull-request-closed" commit-id issue-number) (when (or commit-id issue-number) (log/debug (format "Pull request %s/%s/%s closed with reference to %s" - login repo pr-number - (if commit-id (str "commit-id " commit-id) - (str "issue-number " issue-number)))) + login repo pr-number + (if commit-id (str "commit-id " commit-id) + (str "issue-number " issue-number)))) (pull-requests/create (merge m {:repo_id repo-id :pr_id id :pr_number pr-number :user_id user-id})) (users/create-user user-id login name nil nil))))) -(defn labeled-as-bounty? - [action issue] - (and - (= "labeled" action) - (= label-name (get-in issue [:label :name])))) (defn handle-issue - [issue] - (when-let [action (:action issue)] - (when (labeled-as-bounty? action issue) - (handle-issue-labeled issue)) + [webhook-payload] + (when-let [action (:action webhook-payload)] + (log/debug "handle-issue") + (when (labeled-as-bounty? action webhook-payload) + (handle-issue-labeled webhook-payload)) (when (and - (= "closed" action) - (has-bounty-label? (:issue issue))) - (handle-issue-closed issue))) - (ok (str issue))) + (= "closed" action) + (bounties/has-bounty-label? (:issue webhook-payload))) + (handle-issue-closed webhook-payload))) + (ok (str webhook-payload))) (defn handle-pull-request [pull-request] @@ -135,7 +133,7 @@ (defroutes webhook-routes (POST "/webhook" {:keys [params headers]} - (case (get headers "x-github-event") - "issues" (handle-issue params) - "pull_request" (handle-pull-request params) - (ok)))) + (case (get headers "x-github-event") + "issues" (handle-issue params) + "pull_request" (handle-pull-request params) + (ok)))) From c12e5d1311ab7c5751c53c8f9aa8f64d9b2ccde3 Mon Sep 17 00:00:00 2001 From: Teemu Patja Date: Sun, 22 Jan 2017 12:26:32 +0200 Subject: [PATCH 6/6] Contract-address to comment field + disable QR image caching * include contract address in github comment * set cache-control: no-cache header for QR image response to avoid caching on Github's side. Also no longer including balance in comment hash function. Fixes: #16 --- src/clj/commiteth/github/core.clj | 23 ++++++++++++----------- src/clj/commiteth/routes/qrcodes.clj | 13 ++++++++----- src/clj/commiteth/scheduler.clj | 13 +++++++++++-- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/clj/commiteth/github/core.clj b/src/clj/commiteth/github/core.clj index 84322ea..e4832cd 100644 --- a/src/clj/commiteth/github/core.clj +++ b/src/clj/commiteth/github/core.clj @@ -108,12 +108,12 @@ (repos/delete-hook user repo hook-id (auth-params token)))) (defn github-comment-hash - [user repo issue-number balance] - (digest/sha-256 (str "SALT_Yoh2looghie9jishah7aiphahphoo6udiju" user repo issue-number balance))) + [user repo issue-number] + (digest/sha-256 (str "SALT_Yoh2looghie9jishah7aiphahphoo6udiju" user repo issue-number))) (defn- get-qr-url - [user repo issue-number balance] - (let [hash (github-comment-hash user repo issue-number balance)] + [user repo issue-number] + (let [hash (github-comment-hash user repo issue-number)] (str (server-address) (format "/qr/%s/%s/bounty/%s/%s/qr.png" user repo issue-number hash)))) (defn- md-url @@ -127,15 +127,16 @@ (str "!" (md-url alt src))) (defn generate-comment - [user repo issue-number balance] - (let [image-url (md-image "QR Code" (get-qr-url user repo issue-number balance)) + [user repo issue-number contract-address balance] + (let [image-url (md-image "QR Code" (get-qr-url user repo issue-number)) balance (str balance " ETH") site-url (md-url (server-address) (server-address))] - (format "Current balance: %s\n%s\n%s" balance image-url site-url))) + (format "Current balance: %s\nContract address: %s\n%s\n%s" + balance contract-address image-url site-url))) (defn post-comment - [user repo issue-number balance] - (let [comment (generate-comment user repo issue-number balance)] + [user repo issue-number contract-address balance] + (let [comment (generate-comment user repo issue-number contract-address balance)] (log/debug "Posting comment to" (str user "/" repo "/" issue-number) ":" comment) (issues/create-comment user repo issue-number comment (self-auth-params)))) @@ -158,8 +159,8 @@ (assoc req :body (json/generate-string (or raw-query proper-query))))) (defn update-comment - [user repo comment-id issue-number balance] - (let [comment (generate-comment user repo issue-number balance)] + [user repo comment-id issue-number contract-address balance] + (let [comment (generate-comment user repo issue-number contract-address balance)] (log/debug (str "Updating " user "/" repo "/" issue-number " comment #" comment-id " with contents: " comment)) (let [req (make-patch-request "repos/%s/%s/issues/comments/%s" diff --git a/src/clj/commiteth/routes/qrcodes.clj b/src/clj/commiteth/routes/qrcodes.clj index 44296a4..6940ef1 100644 --- a/src/clj/commiteth/routes/qrcodes.clj +++ b/src/clj/commiteth/routes/qrcodes.clj @@ -33,7 +33,7 @@ (context "/qr" [] ;; user may be an organization here (GET "/:user/:repo/bounty/:issue{[0-9]{1,9}}/:hash/qr.png" [user repo issue hash] - (log/debug "qr PNG GET") + (log/debug "qr PNG GET" user repo issue hash) (let [{address :contract_address login :login repo :repo @@ -45,8 +45,11 @@ (log/debug "address:" address "balance:" balance) (if (and address - (= hash (github/github-comment-hash user repo issue balance))) - (let [issue-url (str login "/" repo "/issues/" issue-number)] - (log/debug "balance:" address) - (ok (generate-image address balance issue-url 768 256))) + (= hash (github/github-comment-hash user repo issue))) + (let [issue-url (str login "/" repo "/issues/" issue-number) + image-url (generate-image address balance issue-url 768 256) + response (assoc-in (ok image-url) + [:headers "cache-control"] "no-cache")] + (log/debug "balance:" address "response" response) + response) (bad-request)))))) diff --git a/src/clj/commiteth/scheduler.clj b/src/clj/commiteth/scheduler.clj index 9731b0a..7d3c3e0 100644 --- a/src/clj/commiteth/scheduler.clj +++ b/src/clj/commiteth/scheduler.clj @@ -24,7 +24,11 @@ repo :repo issue-number :issue_number} issue balance (eth/get-balance-eth contract-address 4) - {comment-id :id} (github/post-comment user repo issue-number balance)] + {comment-id :id} (github/post-comment user + repo + issue-number + contract-address + balance)] (issues/update-comment-id issue-id comment-id)))))) (defn self-sign-bounty @@ -72,7 +76,12 @@ current-balance-eth (eth/hex->eth current-balance-hex 8)] (when-not (= old-balance current-balance-hex) (issues/update-balance contract-address current-balance-hex) - (github/update-comment login repo comment-id issue-number current-balance-eth)))))) + (github/update-comment login + repo + comment-id + issue-number + contract-address + current-balance-eth)))))) (def scheduler-thread-name "SCHEDULER_THREAD")