Merge pull request #36 from pmonks/dev

pmonks · Nov 20, 2023 · a7adc35 · a7adc35
2 parents 17c0e6d + 1c507e8
commit a7adc35
Show file tree

Hide file tree

Showing 13 changed files with 312 additions and 105 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,12 +16,16 @@ jobs:
     if: ${{ needs.skip_check.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        java-version: [ 11, 17, 21 ]   # Note: earliest supported JVM version is 11
+
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 21
+          java-version: ${{ matrix.java-version }}
       - uses: DeLaGuardo/[email protected]
         with:
           cli: latest

diff --git a/pbr.clj b/pbr.clj
@@ -31,4 +31,5 @@
                         :developers       [:developer {:id "pmonks" :name "Peter Monks" :email "[email protected]"}]
                         :scm              {:url "https://github.com/pmonks/lice-comb" :connection "scm:git:git://github.com/pmonks/lice-comb.git" :developer-connection "scm:git:ssh://[email protected]/pmonks/lice-comb.git"}
                         :issue-management {:system "github" :url "https://github.com/pmonks/lice-comb/issues"}}
-         :codox        {:namespaces ['lice-comb.deps 'lice-comb.files 'lice-comb.lein 'lice-comb.matching 'lice-comb.maven 'lice-comb.utils]}))
+         :codox        {:namespaces ['lice-comb.deps 'lice-comb.files 'lice-comb.lein 'lice-comb.matching 'lice-comb.maven 'lice-comb.utils]}
+         :eastwood     {:exclude-linters [:unused-ret-vals-in-try :no-ns-form-found]}))
diff --git a/src/lice_comb/deps.clj b/src/lice_comb/deps.clj
@@ -23,8 +23,8 @@
             [clojure.tools.logging           :as log]
             [lice-comb.maven                 :as lcmvn]
             [lice-comb.files                 :as lcf]
-            [lice-comb.impl.http             :as lcihttp]
-            [lice-comb.impl.expressions-info :as lciei]))
+            [lice-comb.impl.expressions-info :as lciei]
+            [lice-comb.impl.utils            :as lciu]))
 
 (defn- normalise-dep
   "Normalises a dep, by removing any classifier suffixes from the artifact-id
@@ -46,43 +46,29 @@
   [[ga info]]
   (str ga "@" (:git/sha info) (when-let [tag (:git/tag info)] (str "/" tag))))
 
-(defn dep->pom-uri
-  "Returns a java.net.URI that points to the pom for the given tools.dep dep (a
-  MapEntry or two-element vector of `['groupId/artifactId dep-info]`), or nil if
-  the dep is not a Maven dep, or a POM could not be found.  The returned URI is
-  guaranteed to be resolvable - either to a file that exists in the local Maven
-  cache, or to an HTTP-accessible resource on a remote Maven repository (i.e.
-  Maven Central or Clojars) that resolves."
-  [dep]
-  (when (and dep
-             (= :mvn (:deps/manifest (second dep))))
-    (let [[ga info]              (normalise-dep dep)
-          [group-id artifact-id] (s/split (str ga) #"/")
-          version                (:mvn/version info)]
-      (lcihttp/gav->pom-uri group-id artifact-id version))))
-
 (defn- expressions-from-dep
   "Find license expressions in the given dep, ignoring exceptions."
   [dep]
   (when dep
-    (let [info    (second dep)
-          pom-uri (dep->pom-uri dep)]
-      (if-let [pom-expressions (try
-                                 (lcmvn/pom->expressions-info pom-uri)
+    (let [[ga info]              (normalise-dep dep)
+          [group-id artifact-id] (s/split (str ga) #"/")
+          version                (:mvn/version info)]
+      (if-let [gav-expressions (try
+                                 (lcmvn/gav->expressions-info group-id artifact-id version)
                                  (catch javax.xml.stream.XMLStreamException xse
-                                   (log/warn (str "Failed to parse " pom-uri " - ignoring") xse)
+                                   (log/warn (str "Failed to parse POM for " group-id "/" artifact-id (when version (str "@" version)) " - ignoring") xse)
                                    nil))]
-        pom-expressions
+        gav-expressions
         ; If we didn't find any licenses in the dep's POM, check the dep's JAR(s)
-        (into {} (filter identity (pmap #(try
-                                           (lcf/zip->expressions-info %)
-                                           (catch javax.xml.stream.XMLStreamException xse
-                                             (log/warn (str "Failed to parse pom inside " % " - ignoring") xse)
-                                             nil)
-                                           (catch java.util.zip.ZipException ze
-                                             (log/warn (str "Failed to unzip " % " - ignoring") ze)
-                                             nil))
-                                        (:paths info))))))))
+        (into {} (filter identity (lciu/pmap* #(try
+                                                 (lcf/zip->expressions-info %)
+                                                 (catch javax.xml.stream.XMLStreamException xse
+                                                   (log/warn (str "Failed to parse pom inside " % " - ignoring") xse)
+                                                   nil)
+                                                 (catch java.util.zip.ZipException ze
+                                                   (log/warn (str "Failed to unzip " % " - ignoring") ze)
+                                                   nil))
+                                              (:paths info))))))))
 
 (defmulti dep->expressions-info
   "Returns an expressions-info map for the given tools.dep dep (a MapEntry or
@@ -93,22 +79,30 @@
 
 (defmethod dep->expressions-info :mvn
   [dep]
-  (when dep
-    (when-let [expressions (expressions-from-dep dep)]
-      (lciei/prepend-source (dep->string dep) expressions))))
+  (when-let [expressions (expressions-from-dep dep)]
+    (lciei/prepend-source (dep->string dep) expressions)))
 
 (defmethod dep->expressions-info :deps
   [dep]
-  (when dep
-    (let [[_ info] (normalise-dep dep)]
-      (lciei/prepend-source (dep->string dep) (lcf/dir->expressions-info (:deps/root info))))))
+  (let [[_ info] (normalise-dep dep)]
+    (lciei/prepend-source (dep->string dep) (lcf/dir->expressions-info (:deps/root info)))))
 
 (defmethod dep->expressions-info nil
-  [_])
+  [[ga _ :as dep]]
+  (let [[normalised-ga _]      (normalise-dep dep)
+        [group-id artifact-id] (s/split (str normalised-ga) #"/")
+        version                (lcmvn/ga-latest-version group-id artifact-id)]
+    (when version
+      (let [gav-expressions    (try
+                                 (lcmvn/gav->expressions-info group-id artifact-id version)
+                                 (catch javax.xml.stream.XMLStreamException xse
+                                   (log/warn (str "Failed to parse POM for " group-id "/" artifact-id (when version (str "@" version)) " - ignoring") xse)
+                                   nil))]
+        (lciei/prepend-source (str ga "@" version) gav-expressions)))))
 
 (defmethod dep->expressions-info :default
-  [dep]
-  (throw (ex-info (str "Unexpected manifest type '" (:deps/manifest (second dep)) "' for dependency " dep)
+  [[_ info :as dep]]
+  (throw (ex-info (str "Unexpected manifest type '" (:deps/manifest info) "' for dependency " dep)
                   {:dep dep})))
 
 (defn dep->expressions
@@ -134,6 +128,74 @@
                       %)
                    deps))))
 
+(defn dep->pom-uri
+  "Returns a java.net.URI that points to the pom for the given tools.dep dep (a
+  MapEntry or two-element vector of `['groupId/artifactId dep-info]`), or nil if
+  the dep is not a Maven dep, or a POM could not be found.  The returned URI is
+  guaranteed to be resolvable - either to a file that exists in the local Maven
+  cache, or to an HTTP-accessible resource on a remote Maven repository (i.e.
+  Maven Central or Clojars) that resolves."
+  [dep]
+  (when (and dep
+             (= :mvn (:deps/manifest (second dep))))
+    (let [[ga info]              (normalise-dep dep)
+          [group-id artifact-id] (s/split (str ga) #"/")
+          version                (:mvn/version info)]
+      (lcmvn/gav->pom-uri group-id artifact-id version))))
+
+(defmulti dep->locations
+  "Returns a sequence of Strings representing locations that may be searched
+  for license information for the given tools.dep dep (a MapEntry or two-element
+  vector of `['group-id/artifact-id dep-info]`),or nil if no locations were
+  found."
+  {:arglists '([[ga info]])}
+  (fn [[_ info]] (:deps/manifest info)))
+
+(defmethod dep->locations :mvn
+  [[ga info]]
+  (let [[group-id artifact-id] (s/split (str ga) #"/")
+         version               (:mvn/version info)]
+    (seq (filter identity (concat (list (lcmvn/gav->pom-uri group-id artifact-id version)) (:paths info))))))
+
+(defmethod dep->locations :deps
+  [[_ info]]
+  (seq (filter identity (list (:deps/root info)))))
+
+(defmethod dep->locations nil
+  [[ga _]]
+  (let [[group-id artifact-id] (s/split (str ga) #"/")]
+    (seq (filter identity (list (lcmvn/gav->pom-uri group-id artifact-id))))))
+
+(defmethod dep->locations :default
+  [[_ info :as dep]]
+  (throw (ex-info (str "Unexpected manifest type '" (:deps/manifest info) "' for dependency " dep)
+                  {:dep dep})))
+
+(defmulti dep->version
+  "Returns the version (as a String) for the given tools.dep dep (a MapEntry or
+  two-element vector of `['group-id/artifact-id dep-info]`),or nil if no version
+  was found."
+  {:arglists '([[ga info]])}
+  (fn [[_ info]] (:deps/manifest info)))
+
+(defmethod dep->version :mvn
+  [[_ info]]
+  (:mvn/version info))
+
+(defmethod dep->version :deps
+  [[_ info]]
+  (str (:git/sha info) (when-let [tag (:git/tag info)] (str "/" tag))))
+
+(defmethod dep->version nil
+  [[ga _]]
+  (let [[group-id artifact-id] (s/split (str ga) #"/")]
+    (lcmvn/ga-latest-version group-id artifact-id)))
+
+(defmethod dep->version :default
+  [[_ info :as dep]]
+  (throw (ex-info (str "Unexpected manifest type '" (:deps/manifest info) "' for dependency " dep)
+                  {:dep dep})))
+
 (defn init!
   "Initialises this namespace upon first call (and does nothing on subsequent
   calls), returning nil. Consumers of this namespace are not required to call

diff --git a/src/lice_comb/files.clj b/src/lice_comb/files.clj
@@ -145,19 +145,19 @@
   ([dir {:keys [include-zips?] :or {include-zips? false}}]
    (when (lciu/readable-dir? dir)
      (lciei/prepend-source (lciu/filepath dir)
-                           (let [file-expressions (into {} (filter identity (map #(try
-                                                                                    (file->expressions-info %)
-                                                                                    (catch Exception e
-                                                                                      (log/warn (str "Unexpected exception while processing " % " - ignoring") e)
-                                                                                      nil))
-                                                                                 (probable-license-files dir))))]
+                           (let [file-expressions (into {} (filter identity (lciu/pmap* #(try
+                                                                                           (file->expressions-info %)
+                                                                                           (catch Exception e
+                                                                                             (log/warn (str "Unexpected exception while processing " % " - ignoring") e)
+                                                                                             nil))
+                                                                                        (probable-license-files dir))))]
                              (if include-zips?
-                               (let [zip-expressions (into {} (filter identity (map #(try
-                                                                                       (zip->expressions-info %)
-                                                                                       (catch Exception e
-                                                                                         (log/warn (str "Unexpected exception while processing " % " - ignoring") e)
-                                                                                         nil))
-                                                                                    (zip-compressed-files dir))))]
+                               (let [zip-expressions (into {} (filter identity (lciu/pmap* #(try
+                                                                                              (zip->expressions-info %)
+                                                                                              (catch Exception e
+                                                                                                (log/warn (str "Unexpected exception while processing " % " - ignoring") e)
+                                                                                                nil))
+                                                                                           (zip-compressed-files dir))))]
                                  (merge file-expressions zip-expressions))
                                file-expressions))))))
 

diff --git a/src/lice_comb/impl/3rd_party.clj b/src/lice_comb/impl/3rd_party.clj
@@ -1,7 +1,7 @@
 ;;;; lice_comb.impl.3rd_party.clj
 ;;;
-;;; Code obtained from third party sources, but not available via standard
-;;; package-consumption mechanisms (i.e. as Maven artifacts)
+;;; Code obtained from third party sources, but not readily available as a
+;;; library via standard package-consumption mechanisms (i.e. a Maven artifact).
 ;;;
 ;;; Copyright and license information is on a per-code-snippet basis, and
 ;;; is communicated inline via further comments.

diff --git a/src/lice_comb/impl/http.clj b/src/lice_comb/impl/http.clj
@@ -21,7 +21,6 @@
   the public API of lice-comb and may change without notice."
   (:require [clojure.string       :as s]
             [clojure.java.io      :as io]
-            [clojure.java.shell   :as sh]
             [hato.client          :as hc]
             [lice-comb.impl.utils :as lciu]))
 
@@ -51,12 +50,12 @@
 
   If the given URI is not known, returns the input unchanged."
   [uri]
-  (if-let [^java.net.URL uri-obj (try (io/as-url uri) (catch Exception _ nil))]
-    (case (s/lower-case (.getHost uri-obj))
+  (if-let [^java.net.URL url-obj (try (io/as-url uri) (catch Exception _ nil))]
+    (case (s/lower-case (.getHost url-obj))
       "github.com" (-> uri
-                       (s/replace #"(?i)github\.com" "raw.githubusercontent.com")
-                       (s/replace "/blob/"          "/"))
-      uri)  ; Default case
+                       (s/replace-first #"(?i)github\.com" "raw.githubusercontent.com")
+                       (s/replace-first "/blob/"          "/"))
+      uri)
     uri))
 
 (defn get-text
@@ -75,47 +74,11 @@
       (catch Exception _
         nil))))
 
-(def ^:private local-maven-repo-d
-  (delay
-    (try
-      ; The command:
-      ;     mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout
-      ; determines where the local repository is located.
-      (let [sh-result (sh/sh "mvn" "help:evaluate" "-Dexpression=settings.localRepository" "-q" "-DforceStdout")]
-        (if (zero? (:exit sh-result))
-          (s/trim (:out sh-result))
-          (str (System/getProperty "user.home") "/.m2/repository")))
-      (catch java.io.IOException _
-        (str (System/getProperty "user.home") "/.m2/repository")))))
-
-; TODO: make this configurable
-(def ^:private remote-maven-repos #{"https://repo.maven.apache.org/maven2" "https://repo.clojars.org"})
-
-(defn gav->pom-uri
-  "Returns a java.net.URI pointing to the POM for the given GAV (a map), or nil
-  if one cannot be found.  The returned URI is guaranteed to be resolvable -
-  either to a file that exists in the local Maven cache, or to an HTTP-
-  accessible resource on a remote Maven repository (i.e. Maven Central or
-  Clojars) that resolves."
-  ([{:keys [group-id artifact-id version]}] (gav->pom-uri group-id artifact-id version))
-  ([group-id artifact-id version]
-   (when (and (not (s/blank? group-id))
-              (not (s/blank? artifact-id))
-              (not (s/blank? version)))
-     (let [gav-path  (str (s/replace group-id "." "/") "/" artifact-id "/" version "/" artifact-id "-" version ".pom")
-           local-pom (io/file (str @local-maven-repo-d "/" gav-path))]
-       (if (and (.exists local-pom)
-                (.isFile local-pom))
-         (.toURI local-pom)
-         (when-let [remote-uri (first (filter uri-resolves? (map #(str % "/" gav-path) remote-maven-repos)))]
-           (java.net.URI. remote-uri)))))))
-
 (defn init!
   "Initialises this namespace upon first call (and does nothing on subsequent
   calls), returning nil. Consumers of this namespace are not required to call
   this fn, as initialisation will occur implicitly anyway; it is provided to
   allow explicit control of the cost of initialisation to callers who need it."
   []
   @http-client-d
-  @local-maven-repo-d
   nil)
diff --git a/src/lice_comb/impl/non_vthread_pmap.clj b/src/lice_comb/impl/non_vthread_pmap.clj
@@ -0,0 +1,22 @@
+;
+; Copyright © 2023 Peter Monks
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+;     http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
+; SPDX-License-Identifier: Apache-2.0
+;
+
+(in-ns 'lice-comb.impl.utils)
+
+; Fallback on vanilla pmap
+(def pmap* pmap)
diff --git a/src/lice_comb/impl/regex_matching.clj b/src/lice_comb/impl/regex_matching.clj
@@ -383,7 +383,7 @@
   Results are in the order in which they appear in the string, and the function
   returns nil if there were no matches."
   [s]
-  (when-let [matches (seq (filter identity (pmap (partial match s) @license-name-matching-d)))]
+  (when-let [matches (seq (filter identity (lciu/pmap* (partial match s) @license-name-matching-d)))]
     (some->> matches
              (med/distinct-by :id)    ;####TODO: THINK ABOUT MERGING INSTEAD OF DROPPING
              (sort-by :start)

diff --git a/src/lice_comb/impl/utils.clj b/src/lice_comb/impl/utils.clj
@@ -257,3 +257,10 @@
       (if-not (s/blank? val)
         val
         default))))
+
+; Provides pmap*
+(try
+  (Class/forName "java.lang.VirtualThread")
+  (load "vthread_pmap")
+  (catch ClassNotFoundException _
+    (load "non_vthread_pmap")))