Merge pull request #1339 from llasram/uberjar-merge-with
Implement `:uberjar-merge-with` requested in issue #973.
This commit is contained in:
commit
2859d66a63
8 changed files with 142 additions and 46 deletions
|
@ -170,6 +170,10 @@
|
|||
:certificates ["clojars.pem"]
|
||||
:offline? (not (nil? (System/getenv "LEIN_OFFLINE")))
|
||||
:uberjar-exclusions [#"(?i)^META-INF/[^/]*\.(SF|RSA|DSA)$"]
|
||||
:uberjar-merge-with {"META-INF/plexus/components.xml"
|
||||
'leiningen.uberjar/components-merger,
|
||||
"data_readers.clj"
|
||||
'leiningen.uberjar/clj-map-merger}
|
||||
:global-vars {}})
|
||||
|
||||
(defn- dep-key
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
:url "http://www.eclipse.org/legal/epl-v10.html"}
|
||||
:dependencies [[leiningen-core "2.3.3-SNAPSHOT"]
|
||||
[org.clojure/data.xml "0.0.3"]
|
||||
[commons-io "2.4"]
|
||||
[bultitude "0.2.2"]
|
||||
[stencil "0.3.2"]
|
||||
[org.apache.maven.indexer/indexer-core "4.1.3"
|
||||
|
|
|
@ -332,6 +332,14 @@
|
|||
:jar-exclusions [#"(?:^|/).svn/"]
|
||||
;; Same thing, but for uberjars.
|
||||
:uberjar-exclusions [#"META-INF/DUMMY.SF"]
|
||||
;; Files to merge programmatically in uberjars when multiple same-named files
|
||||
;; exist across project and dependencies. Should be a map of filename strings
|
||||
;; or regular expressions to a sequence of three functions:
|
||||
;; 1. Takes an input stream; returns a parsed datum.
|
||||
;; 2. Takes a new datum and the current result datum; returns a merged datum.
|
||||
;; 3. Takes an output stream and a datum; writes the datum to the stream.
|
||||
;; Resolved in reverse dependency order, starting with project.
|
||||
:uberjar-merge-with {#"\.properties$" [slurp str spit]}
|
||||
;; Add arbitrary jar entries. Supports :path, :paths, :bytes, and :fn types.
|
||||
:filespecs [{:type :path :path "config/base.clj"}
|
||||
;; Directory paths are included recursively.
|
||||
|
|
|
@ -6,65 +6,112 @@
|
|||
[leiningen.core.classpath :as classpath]
|
||||
[leiningen.core.project :as project]
|
||||
[leiningen.core.main :as main]
|
||||
[leiningen.core.utils :as utils]
|
||||
[leiningen.jar :as jar])
|
||||
(:import (java.util.zip ZipFile ZipOutputStream ZipEntry)
|
||||
(java.io File FileOutputStream PrintWriter)))
|
||||
(:import (java.io File FileOutputStream PrintWriter)
|
||||
(java.util.regex Pattern)
|
||||
(java.util.zip ZipFile ZipOutputStream ZipEntry)
|
||||
(org.apache.commons.io.output CloseShieldOutputStream)))
|
||||
|
||||
(defn read-components [zipfile]
|
||||
(if-let [entry (.getEntry zipfile "META-INF/plexus/components.xml")]
|
||||
(->> (zip/xml-zip (xml/parse (.getInputStream zipfile entry)))
|
||||
zip/children
|
||||
(defn- components-read [ins]
|
||||
(->> ins xml/parse zip/xml-zip zip/children
|
||||
(filter #(= (:tag %) :components))
|
||||
first
|
||||
:content)))
|
||||
first :content))
|
||||
|
||||
;; We have to keep this separate from skip-set for performance reasons.
|
||||
(defn- make-skip-pred [project]
|
||||
(fn [filename]
|
||||
(some #(re-find % filename) (:uberjar-exclusions project))))
|
||||
|
||||
;; TODO: unify with copy-to-jar functionality in jar.clj (for 3.0?)
|
||||
(defn- copy-entries
|
||||
"Copies the entries of ZipFile in to the ZipOutputStream out, skipping
|
||||
the entries which satisfy skip-pred. Returns the names of the
|
||||
entries copied."
|
||||
[in out skip-set skip-pred]
|
||||
(for [file (enumeration-seq (.entries in))
|
||||
:let [filename (.getName file)]
|
||||
:when (not (or (skip-set filename) (skip-pred filename)))]
|
||||
(do
|
||||
(.setCompressedSize file -1) ; some jars report size incorrectly
|
||||
(.putNextEntry out file)
|
||||
(io/copy (.getInputStream in file) out)
|
||||
(.closeEntry out)
|
||||
(.getName file))))
|
||||
|
||||
;; we have to keep track of every entry we've copied so that we can
|
||||
;; skip duplicates. We also collect together all the plexus components so
|
||||
;; that we can merge them.
|
||||
(defn- include-dep [out skip-pred [skip-set components] dep]
|
||||
(main/debug "Including" (.getName dep))
|
||||
(with-open [zipfile (ZipFile. dep)]
|
||||
[(into skip-set (copy-entries zipfile out skip-set skip-pred))
|
||||
(concat components (read-components zipfile))]))
|
||||
|
||||
(defn write-components
|
||||
"Given a list of jarfiles, writes contents to a stream"
|
||||
[project jars out]
|
||||
(let [[_ components] (reduce (partial include-dep out
|
||||
(make-skip-pred project))
|
||||
[#{"META-INF/plexus/components.xml"} nil]
|
||||
jars)]
|
||||
(when-not (empty? components)
|
||||
(.putNextEntry out (ZipEntry. "META-INF/plexus/components.xml"))
|
||||
(defn- components-write [out components]
|
||||
(binding [*out* (PrintWriter. out)]
|
||||
(xml/emit {:tag :component-set
|
||||
:content
|
||||
[{:tag :components
|
||||
:content
|
||||
components}]})
|
||||
(.flush *out*))
|
||||
(.closeEntry out))))
|
||||
:content components}]})
|
||||
(.flush *out*)))
|
||||
|
||||
(def components-merger
|
||||
"Project `:uberjar-merge-with` merger for components.xml files."
|
||||
[components-read into components-write])
|
||||
|
||||
(def clj-map-merger
|
||||
"Project `:uberjar-merge-with` for files containing a single map
|
||||
read with `clojure.core/read`, such as data_readers.clj."
|
||||
[(comp read-string slurp) merge #(spit %1 (pr-str %2))])
|
||||
|
||||
(defn- merger-match? [[pattern] filename]
|
||||
(boolean
|
||||
(condp instance? pattern
|
||||
String (= pattern filename)
|
||||
Pattern (re-find pattern filename))))
|
||||
|
||||
(def ^:private skip-merger
|
||||
[(constantly ::skip)
|
||||
(constantly nil)])
|
||||
|
||||
(def ^:private default-merger
|
||||
[(fn [in out file prev]
|
||||
(when-not prev
|
||||
(.setCompressedSize file -1)
|
||||
(.putNextEntry out file)
|
||||
(io/copy (.getInputStream in file) out)
|
||||
(.closeEntry out))
|
||||
::skip)
|
||||
(constantly nil)])
|
||||
|
||||
(defn- make-merger [fns]
|
||||
{:pre [(sequential? fns) (= 3 (count fns)) (every? ifn? fns)]}
|
||||
(let [[read-fn merge-fn write-fn] fns]
|
||||
[(fn [in out file prev]
|
||||
(with-open [ins (.getInputStream in file)]
|
||||
(let [new (read-fn ins)]
|
||||
(if-not prev
|
||||
new
|
||||
(merge-fn new prev)))))
|
||||
(fn [out filename result]
|
||||
(.putNextEntry out (ZipEntry. filename))
|
||||
(write-fn (CloseShieldOutputStream. out) result)
|
||||
(.closeEntry out))]))
|
||||
|
||||
(defn- make-mergers [project]
|
||||
(into (utils/map-vals
|
||||
(:uberjar-merge-with project)
|
||||
(comp make-merger eval))
|
||||
(map #(-> [% skip-merger])
|
||||
(:uberjar-exclusions project))))
|
||||
|
||||
(defn- select-merger [mergers filename]
|
||||
(or (->> mergers (filter #(merger-match? % filename)) first second)
|
||||
default-merger))
|
||||
|
||||
;; TODO: unify with copy-to-jar functionality in jar.clj (for 3.0?)
|
||||
(defn- copy-entries
|
||||
"Read entries of ZipFile `in` and apply the filename-determined
|
||||
entry-merging logic captured in `mergers`. The default merger
|
||||
copies entry contents directly to the ZipOutputStream `out` and
|
||||
skips subsequent same-named files. Returns new `merged-map` merged
|
||||
entry map."
|
||||
[in out mergers merged-map]
|
||||
(reduce (fn [merged-map file]
|
||||
(let [filename (.getName file), prev (get merged-map filename)]
|
||||
(if (identical? ::skip prev)
|
||||
merged-map
|
||||
(let [[read-merge] (select-merger mergers filename)]
|
||||
(assoc merged-map
|
||||
filename (read-merge in out file prev))))))
|
||||
merged-map (enumeration-seq (.entries in))))
|
||||
|
||||
(defn- include-dep [out mergers merged-map dep]
|
||||
(main/debug "Including" (.getName dep))
|
||||
(with-open [zipfile (ZipFile. dep)]
|
||||
(copy-entries zipfile out mergers merged-map)))
|
||||
|
||||
(defn write-components
|
||||
"Given a list of jarfiles, writes contents to a stream"
|
||||
[project jars out]
|
||||
(let [mergers (make-mergers project)
|
||||
include-dep (partial include-dep out mergers)
|
||||
merged-map (reduce include-dep {} jars)]
|
||||
(doseq [[filename result] merged-map
|
||||
:when (not (identical? ::skip result))
|
||||
:let [[_ write] (select-merger mergers filename)]]
|
||||
(write out filename result))))
|
||||
|
||||
(defn uberjar
|
||||
"Package up the project files and all dependencies into a jar file.
|
||||
|
|
|
@ -34,6 +34,8 @@
|
|||
|
||||
(def provided-project (read-test-project "provided"))
|
||||
|
||||
(def uberjar-merging-project (read-test-project "uberjar-merging"))
|
||||
|
||||
(def overlapped-sourcepaths-project (read-test-project "overlapped-sourcepaths"))
|
||||
|
||||
(def more-gen-classes-project (read-test-project "more-gen-classes"))
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
[clojure.test :refer :all]
|
||||
[clojure.java.shell :refer [sh]]
|
||||
[leiningen.test.helper :refer [sample-no-aot-project
|
||||
uberjar-merging-project
|
||||
provided-project]])
|
||||
(:import (java.io File)
|
||||
(java.util.zip ZipFile)))
|
||||
|
@ -24,6 +25,22 @@
|
|||
(is (entries "org/codehaus/janino/Compiler$1.class"))
|
||||
(is (not (some #(re-find #"dummy" %) entries)))))))
|
||||
|
||||
(deftest test-uberjar-merge-with
|
||||
(uberjar uberjar-merging-project)
|
||||
(let [filename (str "test_projects/uberjar-merging/target/"
|
||||
"nomnomnom-0.5.0-SNAPSHOT-standalone.jar")
|
||||
uberjar-file (File. filename)]
|
||||
(is (= true (.exists uberjar-file)))
|
||||
(when (.exists uberjar-file)
|
||||
(.deleteOnExit uberjar-file)
|
||||
(with-open [zf (ZipFile. uberjar-file)]
|
||||
(is (= '{nomnomnom/identity clojure.core/identity
|
||||
mf/i nomnomnom/override
|
||||
mf/s method.fn/static}
|
||||
(->> (.getEntry zf "data_readers.clj")
|
||||
(.getInputStream zf)
|
||||
slurp read-string)))))))
|
||||
|
||||
;; TODO: this breaks on Java 6
|
||||
(deftest ^:disabled test-uberjar-provided
|
||||
(let [bootclasspath "-Xbootclasspath/a:leiningen-core/lib/clojure-1.4.0.jar"
|
||||
|
|
14
test_projects/uberjar-merging/project.clj
Normal file
14
test_projects/uberjar-merging/project.clj
Normal file
|
@ -0,0 +1,14 @@
|
|||
;; This project is used for leiningen's test suite, so don't change
|
||||
;; any of these values without updating the relevant tests. If you
|
||||
;; just want a basic project to work from, generate a new one with
|
||||
;; "lein new".
|
||||
|
||||
(defproject nomnomnom "0.5.0-SNAPSHOT"
|
||||
:dependencies [[org.clojure/clojure "1.5.1"]
|
||||
[janino "2.5.15"]
|
||||
[org.platypope/method-fn "0.1.0"]]
|
||||
:uberjar-exclusions [#"DUMMY"]
|
||||
:test-selectors {:default (fn [m] (not (:integration m)))
|
||||
:integration :integration
|
||||
:int2 :int2
|
||||
:no-custom (fn [m] (not (false? (:custom m))))})
|
3
test_projects/uberjar-merging/resources/data_readers.clj
Normal file
3
test_projects/uberjar-merging/resources/data_readers.clj
Normal file
|
@ -0,0 +1,3 @@
|
|||
{nomnomnom/identity clojure.core/identity,
|
||||
mf/i nomnomnom/override,
|
||||
}
|
Loading…
Reference in a new issue