diff --git a/src/clj/commiteth/github/core.clj b/src/clj/commiteth/github/core.clj
index f92b276..b90237c 100644
--- a/src/clj/commiteth/github/core.clj
+++ b/src/clj/commiteth/github/core.clj
@@ -8,7 +8,7 @@
             [commiteth.config :refer [env]])
   (:import [java.util UUID]))
 
-(defn server-address []  (:server-address env))
+(defn server-address [] (:server-address env))
 (defn client-id [] (:github-client-id env))
 (defn client-secret [] (:github-client-secret env))
 (defn redirect-uri [] (str (server-address) "/callback"))
@@ -101,6 +101,10 @@
   [user repo commit-id]
   (repos/specific-commit user repo commit-id (self-auth-params)))
 
+(defn get-issue
+  [user repo issue-number]
+  (issues/specific-issue user repo issue-number (self-auth-params)))
+
 (defn get-issue-events
   [user repo issue-id]
   (issues/issue-events user repo issue-id (self-auth-params)))
diff --git a/src/clj/commiteth/routes/webhooks.clj b/src/clj/commiteth/routes/webhooks.clj
index 889182f..4147f0c 100644
--- a/src/clj/commiteth/routes/webhooks.clj
+++ b/src/clj/commiteth/routes/webhooks.clj
@@ -53,6 +53,18 @@
                    (catch NumberFormatException _)))
             (re-seq % pr-body)) keywords))
 
+(defn has-bounty-label?
+  [issue]
+  (let [labels (:labels issue)]
+    (some #(= label-name (:name %)) labels)))
+
+(defn validate-issue-number
+  "Checks if an issue has a bounty label attached and returns its number"
+  [user repo issue-number]
+  (when-let [issue (github/get-issue user repo issue-number)]
+    (when (has-bounty-label? issue)
+      issue-number)))
+
 (defn handle-pull-request-closed
   [{{{owner :login} :owner
      repo           :name
@@ -65,7 +77,10 @@
      pr-body         :body} :pull_request}]
   (future
     (let [commit-id    (find-commit-id owner repo pr-number "merged")
-          issue-number (first (extract-issue-number pr-body))
+          issue-number (->>
+                         (extract-issue-number pr-body)
+                         (first)
+                         (validate-issue-number owner repo))
           m            {:commit_id commit-id :issue_number issue-number}]
       (when (or commit-id issue-number)
         (pull-requests/create (merge m {:repo_id   repo-id
@@ -80,11 +95,6 @@
     (= "labeled" action)
     (= label-name (get-in issue [:label :name]))))
 
-(defn has-bounty-label?
-  [issue]
-  (let [labels (get-in issue [:issue :labels])]
-    (some #(= label-name (:name %)) labels)))
-
 (defn gen-address []
   (UUID/randomUUID))
 
@@ -105,7 +115,7 @@
         (issues/create repo-id issue-id issue-number issue-title issue-address)))
     (when (and
             (= "closed" action)
-            (has-bounty-label? issue))
+            (has-bounty-label? (:issue issue)))
       (handle-issue-closed issue)))
   (ok (str issue)))