Continuation of https://github.com/nim-lang/packages/pull/916 (#2027)

package_scanner.nim

@@ -1,4 +1,3 @@
-
 # A very simple Nim package scanner.
 #
 # Scans the package list from this repository.
@@ -12,50 +11,89 @@
 #  * Missing/unknown license
 #  * Insecure git:// url on GitHub
 #
-# Usage: nim c -d:ssl -r package_scanner.nim
+# Usage: nim r -d:ssl package_scanner.nim
 #
 # Copyright 2015 Federico Ceratto <federico.ceratto@gmail.com>
 # Released under GPLv3 License, see /usr/share/common-licenses/GPL-3
 
-import httpclient
+import std/[httpclient, net, json, os, sets, strutils]
-import net
-import json
-import os
-import sets
-import strutils
 
-const
+const licenses = [
-  LICENSES = @[
+    "allegro 4 giftware",
-    "Allegro 4 Giftware",
+    "apache license 2.0",
-    "Apache License 2.0",
+    "apache",
-    "BSD",
+    "apache2",
-    "BSD2",
+    "apache 2.0",
-    "BSD3",
+    "apache-2.0",
-    "CC0",
+    "apache-2.0 license",
-    "GPL",
+    "apache version 2.0",
-    "GPLv2",
+    "mit or apache 2.0",
-    "GPLv3",
+    "apache license 2.0 or mit",
-    "LGPLv2",
+    "mit or apache license 2.0",
-    "LGPLv3",
+    "(mit or apache license 2.0) and simplified bsd",
-    "MIT",
+    "lxxsdt-mit",
-    "MS-PL",
+    "lgplv2.1",
-    "MPL",
+    "0bsd",
-    "WTFPL",
+    "bsd",
+    "bsd2",
+    "bsd-2",
+    "bsd3",
+    "bsd-3",
+    "bsd 3-clause",
+    "bsd-3-clause",
+    "2-clause bsd",
+    "cc0",
+    "cc0-1.0",
+    "gpl",
+    "gpl2",
+    "gpl3",
+    "gplv2",
+    "gplv3",
+    "gplv3+",
+    "gpl-2.0",
+    "agpl-3.0",
+    "gpl-3.0",
+    "gpl-3.0-or-later",
+    "gpl-3.0-only",
+    "lgplv3 or gplv2",
+    "apache 2.0 or gplv2",
+    "lgpl-2.1-or-later",
+    "lgpl with static linking exception",
+    "gnu lesser general public license v2.1",
+    "openldap",
+    "lgpl",
+    "lgplv2",
+    "lgplv3",
+    "lgpl-2.1",
+    "lgpl-3.0",
+    "agplv3",
+    "mit",
+    "mit/isc",
+    "ms-pl",
+    "mpl",
+    "mplv2",
+    "mpl-2.0",
+    "mpl 2.0",
+    "epl-2.0",
+    "wtfpl",
     "libpng",
+    "fontconfig",
     "zlib",
-    "ISC",
+    "isc",
-    "Unlicense"
+    "ppl",
+    "hydra",
+    "openssl and ssleay",
+    "unlicense",
+    "public domain",
+    "proprietary",
   ]
-  VCS_TYPES = @["git", "hg"]
 
 proc canFetchNimbleRepository(name: string, urlJson: JsonNode): bool =
-  # The fetch is a lie!
   # TODO: Make this check the actual repo url and check if there is a
   #       nimble file in it
   result = true
   var url: string
-  var client = newHttpClient(timeout=100000)
+  var client = newHttpClient(timeout = 100_000)
-  defer: client.close()
 
   if not urlJson.isNil:
     url = urlJson.str
@@ -64,79 +102,73 @@ proc canFetchNimbleRepository(name: string, urlJson: JsonNode): bool =
         client.headers = newHttpHeaders({"authorization": "Bearer " & getEnv("GITHUB_TOKEN")})
     try:
       discard client.getContent(url)
-    except HttpRequestError, TimeoutError:
+    except TimeoutError:
-      echo "W: ", name, ": unable to fetch repo ", url, " ",
+      echo "W: ", name, ": Timeout error fetching repo ", url, " ", getCurrentExceptionMsg()
-           getCurrentExceptionMsg()
+    except HttpRequestError:
-    except AssertionError:
+      echo "W: ", name, ": HTTP error fetching repo ", url, " ", getCurrentExceptionMsg()
-      echo "W: ", name, ": httpclient failed ", url, " ",
+    except AssertionDefect:
-           getCurrentExceptionMsg()
+      echo "W: ", name, ": httpclient error fetching repo ", url, " ", getCurrentExceptionMsg()
     except:
-      echo "W: Another error attempting to request: ", url
+      echo "W: Unkown error fetching repo ", url, " ", getCurrentExceptionMsg()
-      echo "  Error was: ", getCurrentExceptionMsg()
+    finally:
-
+      client.close()
-proc verifyAlias(pdata: JsonNode, result: var int) =
-  if not pdata.hasKey("name"):
-    echo "E: missing alias' package name"
-    result.inc()
 
+proc verifyAlias(pkg: JsonNode, result: var int) =
+  if not pkg.hasKey("name"):
+    echo "E: Missing alias' package name"
+    inc result
   # TODO: Verify that 'alias' points to a known package.
 
 proc check(): int =
   var name: string
-  echo ""
+  var names = initHashSet[string]()
-  let pkg_list = parseJson(readFile(getCurrentDir() / "packages.json"))
-  var names = initSet[string]()
 
-  for pdata in pkg_list:
+  for pkg in parseJson(readFile(getCurrentDir() / "packages.json")):
-    name = if pdata.hasKey("name"): pdata["name"].str else: ""
+    name = if pkg.hasKey("name"): pkg["name"].str else: ""
-
+    if pkg.hasKey("alias"):
-    if pdata.hasKey("alias"):
+      verifyAlias(pkg, result)
-      verifyAlias(pdata, result)
     else:
-      if name == "":
+      if name.len == 0:
         echo "E: missing package name"
-        result.inc()
+        inc result
-      elif not pdata.hasKey("method"):
+      elif not pkg.hasKey("method"):
         echo "E: ", name, " has no method"
-        result.inc()
+        inc result
-      elif not (pdata["method"].str in VCS_TYPES):
+      elif pkg["method"].str notin ["git", "hg"]:
-        echo "E: ", name, " has an unknown method: ", pdata["method"].str
+        echo "E: ", name, " has an unknown method: ", pkg["method"].str
-        result.inc()
+        inc result
-      elif not pdata.hasKey("url"):
+      elif not pkg.hasKey("url"):
         echo "E: ", name, " has no URL"
-        result.inc()
+        inc result
-      elif pdata.hasKey("web") and not canFetchNimbleRepository(name, pdata["web"]):
+      elif not pkg.hasKey("tags"):
-        result.inc()
-      elif not pdata.hasKey("tags"):
         echo "E: ", name, " has no tags"
-        result.inc()
+        inc result
-      elif not pdata.hasKey("description"):
+      elif not pkg.hasKey("description"):
         echo "E: ", name, " has no description"
-        result.inc()
+        inc result
-      elif pdata.hasKey("description") and pdata["description"].str == "":
+      elif pkg.hasKey("description") and pkg["description"].str == "":
         echo "E: ", name, " has empty description"
-        result.inc()
+        inc result
-      elif not pdata.hasKey("license"):
+      elif not pkg.hasKey("license"):
         echo "E: ", name, " has no license"
-        result.inc()
+        inc result
-      elif pdata["url"].str.normalize.startsWith("git://github.com/"):
+      elif pkg["url"].str.normalize.startsWith("git://github.com/"):
         echo "E: ", name, " has an insecure git:// URL instead of https://"
-        result.inc()
+        inc result
-      else:
+      elif pkg["license"].str.toLowerAscii notin licenses:
-        # Other warnings should go here
+        echo "E: ", name, " has an unexpected license: ", pkg["license"]
-        if not (pdata["license"].str in LICENSES):
+        inc result
-          echo "W: ", name, " has an unexpected license: ", pdata["license"]
+      elif pkg.hasKey("web") and not canFetchNimbleRepository(name, pkg["web"]):
+        echo "W: Failed to fetch source code repo for ", name
 
     if name.normalize notin names:
-      names.incl(name.normalize)
+      names.incl name.normalize
     else:
       echo("E: ", name, ": a package by that name already exists.")
-      result.inc()
+      inc result
 
-  echo ""
+  echo "\nProblematic packages count: ", result
-  echo "Problematic packages count: ", result
 
 
 when isMainModule:
   quit(check())
-