diff --git a/biz.aQute.bndlib.tests/jar/jarA.jar b/biz.aQute.bndlib.tests/jar/jarA.jar new file mode 100644 index 0000000000..0f659c7657 Binary files /dev/null and b/biz.aQute.bndlib.tests/jar/jarA.jar differ diff --git a/biz.aQute.bndlib.tests/jar/jarB.jar b/biz.aQute.bndlib.tests/jar/jarB.jar new file mode 100644 index 0000000000..d4e324f4df Binary files /dev/null and b/biz.aQute.bndlib.tests/jar/jarB.jar differ diff --git a/biz.aQute.bndlib.tests/test/test/BuilderTest.java b/biz.aQute.bndlib.tests/test/test/BuilderTest.java index dd36dbff46..5d0903f7a6 100644 --- a/biz.aQute.bndlib.tests/test/test/BuilderTest.java +++ b/biz.aQute.bndlib.tests/test/test/BuilderTest.java @@ -381,6 +381,8 @@ public void test1017UsingPrivatePackagesVersion() throws Exception { B.setExportPackage("org.osgi.service.wireadmin"); B.setPrivatePackage("org.osgi.service.event"); B.setIncludeResource("org/osgi/service/event/packageinfo;literal='version 2.0.0'"); + B.setProperty("-fixupmessages.duplicates", + "includeresource.duplicates"); B.build(); assertTrue(B.check()); diff --git a/biz.aQute.bndlib.tests/test/test/IncludeResourceTest.java b/biz.aQute.bndlib.tests/test/test/IncludeResourceTest.java index 717154cd17..0083d9d92e 100644 --- a/biz.aQute.bndlib.tests/test/test/IncludeResourceTest.java +++ b/biz.aQute.bndlib.tests/test/test/IncludeResourceTest.java @@ -2,8 +2,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.File; import java.io.IOException; import java.net.URI; import java.util.HashSet; @@ -16,6 +18,7 @@ import aQute.bnd.osgi.Constants; import aQute.bnd.osgi.Jar; import aQute.bnd.osgi.Resource; +import aQute.bnd.test.jupiter.InjectTemporaryDirectory; import aQute.lib.io.IO; public class IncludeResourceTest { @@ -181,4 +184,365 @@ private String testPreprocessing(String ir, String resource, String... checks) t } } + + @Test + public void testIncludeResourceDuplicatesDefaultOverwrite() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource("@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*"); + Jar jar = a.build(); + assertFalse(a.check()); + assertEquals( + "includeresource.duplicates: Duplicate overwritten: META-INF/services/foo (Consider using the onduplicate: directive to handle duplicates.)", + a.getWarnings() + .get(0)); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + // default should be "overwrite" + assertEquals("b", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesDefaultOverwriteButNoWarningOnIdenticalFiles() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource("@jar/jarA.jar!/META-INF/services/*, @jar/jarA.jar!/META-INF/services/*"); + Jar jar = a.build(); + assertTrue(a.check()); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + // default should be "overwrite" + assertEquals("a", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesMerge() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:=MERGE"); + Jar jar = a.build(); + assertTrue(a.check()); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + assertEquals("a\nb", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceMixedMetaInfDuplicatesMerge() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource("@jar/jarA.jar!/META-INF/*, @jar/jarB.jar!/META-INF/*;onduplicate:=MERGE"); + Jar jar = a.build(); + assertTrue(a.check()); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resourceFoo = jar.getResource("META-INF/services/foo"); + assertEquals("a\nb", IO.collect(resourceFoo.openInputStream())); + + Resource resourceManifest = jar.getResource("META-INF/bar.txt"); + assertEquals("a", IO.collect(resourceManifest.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesMergeBlank() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + // dup_merge contains a blank value. should be ignored and use + // default 'overwrite' behavior + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:= "); + Jar jar = a.build(); + assertFalse(a.check()); + assertEquals("No value after '=' sign for attribute onduplicate:", a.getErrors() + .get(0)); + + } + } + + + @Test + public void testIncludeResourceDuplicatesError() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:=ERROR"); + Jar jar = a.build(); + assertFalse(a.check()); + assertEquals("includeresource.duplicates: duplicate found for path META-INF/services/foo", a.getErrors() + .get(0)); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + assertEquals("b", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesWarning() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:=WARN"); + Jar jar = a.build(); + assertFalse(a.check()); + assertEquals("includeresource.duplicates: duplicate found for path META-INF/services/foo", a.getWarnings() + .get(0)); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + assertEquals("b", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesOverwrite() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:=OVERWRITE"); + Jar jar = a.build(); + assertTrue(a.check()); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + assertEquals("b", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesSkip() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:=SKIP"); + Jar jar = a.build(); + assertTrue(a.check()); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + assertEquals("a", IO.collect(resource.openInputStream())); + + } + } + + + @Test + public void testIncludeResourceLiteralDuplicatesMerge(@InjectTemporaryDirectory + File tmp) throws Exception { + + try (Builder b = new Builder()) { + b.setIncludeResource("/a/a.txt;literal='a', /a/a.txt;literal='b';onduplicate:=MERGE"); + b.build(); + assertTrue(b.check()); + + b.getJar() + .writeFolder(tmp); + + assertEquals("a", IO.collect(IO.getFile(tmp, "a/a.txt"))); + } + } + + @Test + public void testIncludeResourceLiteralMetaInfServicesDuplicatesMerge(@InjectTemporaryDirectory + File tmp) throws Exception { + + try (Builder b = new Builder()) { + b.setIncludeResource( + "META-INF/services/a.txt;literal='a', META-INF/services/a.txt;literal='b';onduplicate:=MERGE"); + b.build(); + assertTrue(b.check()); + + b.getJar() + .writeFolder(tmp); + + assertEquals("a\nb", IO.collect(IO.getFile(tmp, "META-INF/services/a.txt"))); + } + } + + @Test + public void testIncludeResourceLiteralDuplicatesError(@InjectTemporaryDirectory + File tmp) throws Exception { + + try (Builder b = new Builder()) { + b.setIncludeResource("/a/a.txt;literal='a', /a/a.txt;literal='b';onduplicate:=ERROR"); + b.build(); + assertFalse(b.check()); + assertEquals("includeresource.duplicates: duplicate found for path /a/a.txt", b.getErrors() + .get(0)); + + b.getJar() + .writeFolder(tmp); + + assertEquals("b", IO.collect(IO.getFile(tmp, "a/a.txt"))); + } + } + + @Test + public void testIncludeResourceDuplicatesMergeWithTag() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:='MERGE,metainfservices'"); + Jar jar = a.build(); + assertTrue(a.check()); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + assertEquals("a\nb", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesMergeWithoutPlugin() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:='MERGE,nonexistingtag'"); + Jar jar = a.build(); + assertFalse(a.check()); + + assertEquals("includeresource.duplicates: no plugins found for tags: [nonexistingtag]", a.getErrors() + .get(0)); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + // we expect nothing because there is no plugin with the tag + // 'nonexistingtag' + // which means we have nothing which can merge META-INF/services + // files. + // so we keep the existing file + assertEquals("a", IO.collect(resource.openInputStream())); + + } + + } + + @Test + public void testIncludeResourceDuplicatesTagWithoutStrategyEnumButExistingTag() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:='metainfservices'"); + Jar jar = a.build(); + assertTrue(a.check()); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + assertEquals("a\nb", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesTagWithoutStrategyEnumButNonExistingTag() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:='nonexistingtag'"); + Jar jar = a.build(); + assertFalse(a.check()); + + assertEquals("includeresource.duplicates: no plugins found for tags: [nonexistingtag]", a.getErrors() + .get(0)); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + // we expect nothing because there is no plugin with the tag + // 'nonexistingtag' + // which means we have nothing which can merge META-INF/services + // files. + // so we keep the existing file + assertEquals("a", IO.collect(resource.openInputStream())); + + } + } + + @Test + public void testIncludeResourceDuplicatesMergeWithTagAndWarn() throws Exception { + + try (Builder a = new Builder();) { + a.addClasspath(new File("jar/jarA.jar")); + a.addClasspath(a.getFile("jar/jarB.jar")); + a.setIncludeResource( + "@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;onduplicate:='MERGE,metainfservices,WARN'"); + Jar jar = a.build(); + assertFalse(a.check()); + assertEquals("includeresource.duplicates: duplicate found for path META-INF/services/foo", a.getWarnings() + .get(0)); + + assertTrue(jar.getDirectories() + .containsKey("META-INF/services")); + + Resource resource = jar.getResource("META-INF/services/foo"); + assertEquals("a\nb", IO.collect(resource.openInputStream())); + + } + } + } diff --git a/biz.aQute.bndlib/src/aQute/bnd/osgi/Builder.java b/biz.aQute.bndlib/src/aQute/bnd/osgi/Builder.java index 98f08888ba..ed70ff6e90 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/osgi/Builder.java +++ b/biz.aQute.bndlib/src/aQute/bnd/osgi/Builder.java @@ -15,19 +15,23 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; import java.util.function.Function; import java.util.jar.JarFile; import java.util.jar.Manifest; @@ -55,6 +59,7 @@ import aQute.bnd.metatype.MetatypeAnnotations; import aQute.bnd.osgi.Descriptors.PackageRef; import aQute.bnd.osgi.Descriptors.TypeRef; +import aQute.bnd.osgi.metainf.MetaInfServiceMerger; import aQute.bnd.osgi.metainf.MetaInfServiceParser; import aQute.bnd.plugin.jpms.JPMSAnnotations; import aQute.bnd.plugin.jpms.JPMSModuleInfoPlugin; @@ -65,8 +70,10 @@ import aQute.bnd.service.diff.Diff; import aQute.bnd.service.diff.Tree; import aQute.bnd.service.diff.Type; +import aQute.bnd.service.merge.MergeResources; import aQute.bnd.service.specifications.BuilderSpecification; import aQute.bnd.stream.MapStream; +import aQute.bnd.unmodifiable.Maps; import aQute.bnd.version.Version; import aQute.lib.collections.Logic; import aQute.lib.collections.MultiMap; @@ -1348,7 +1355,7 @@ private void extractFromJar(Jar jar, String source, String destination, boolean } for (Jar j : sub) - addAll(jar, j, instr, destination, nameMapper); + addAll(jar, j, instr, destination, nameMapper, extra); } } @@ -1369,10 +1376,14 @@ public boolean addAll(Jar to, Jar sub, Instruction filter) { * @param filter a pattern that should match the resoures in sub to be added */ public boolean addAll(Jar to, Jar sub, Instruction filter, String destination) { - return addAll(to, sub, filter, destination, Function.identity()); + return addAll(to, sub, filter, destination, Function.identity(), Maps.of()); } - private boolean addAll(Jar to, Jar sub, Instruction filter, String destination, Function modifier) { + private boolean addAll(Jar to, Jar sub, Instruction filter, String destination, Function modifier, + Map extra) { + + Function> dupStrategy = parseDupStrategy(extra); + boolean dupl = false; for (String name : sub.getResources() .keySet()) { @@ -1382,9 +1393,25 @@ private boolean addAll(Jar to, Jar sub, Instruction filter, String destination, if (doNotCopy(Strings.getLastSegment(name, '/'))) continue; - if (filter == null || filter.matches(name) ^ filter.isNegated()) - dupl |= to.putResource(Processor.appendPath(destination, modifier.apply(name)), sub.getResource(name), - true); + if (filter == null || filter.matches(name) ^ filter.isNegated()) { + + String path = Processor.appendPath(destination, modifier.apply(name)); + Resource resource = sub.getResource(name); + Resource existing = to.getResource(path); + boolean duplicate = existing != null; + + if (!duplicate) { + dupl |= to.putResource(path, resource, true); + } else { + Optional maybeMerged = dupStrategy.apply(new Duplication(path, existing, resource)); + // Resource maybeMerged = dupStrategy.onDuplicate(path, + // existing, resource, this); + if (maybeMerged.isPresent()) { + dupl |= to.putResource(path, maybeMerged.get()); + } + } + + } } return dupl; } @@ -1426,7 +1453,19 @@ private void copy(Jar jar, String path, File from, Instructions preprocess, Map< } private void copy(Jar jar, String path, Resource resource, Map extra) { - jar.putResource(path, resource); + Resource existing = jar.getResource(path); + + boolean duplicate = existing != null; + if (!duplicate) { + jar.putResource(path, resource, true); + } else { + Function> dupStrategy = parseDupStrategy(extra); + Optional maybeMerged = dupStrategy.apply(new Duplication(path, existing, resource)); + if (maybeMerged.isPresent()) { + jar.putResource(path, maybeMerged.get(), true); + } + } + if (isTrue(extra.get(LIB_DIRECTIVE))) { setProperty(BUNDLE_CLASSPATH, append(getProperty(BUNDLE_CLASSPATH, "."), path)); } @@ -1766,6 +1805,7 @@ public Pattern getDoNotCopy() { static SPIDescriptorGenerator spiDescriptorGenerator = new SPIDescriptorGenerator(); static JPMSMultiReleasePlugin jpmsReleasePlugin = new JPMSMultiReleasePlugin(); static MetaInfServiceParser metaInfoServiceParser = new MetaInfServiceParser(); + static MetaInfServiceMerger metaInfServiceMerger = new MetaInfServiceMerger(); @Override protected void setTypeSpecificPlugins(PluginsContainer pluginsContainer) { @@ -1780,6 +1820,7 @@ protected void setTypeSpecificPlugins(PluginsContainer pluginsContainer) { pluginsContainer.add(spiDescriptorGenerator); pluginsContainer.add(jpmsReleasePlugin); pluginsContainer.add(metaInfoServiceParser); + pluginsContainer.add(metaInfServiceMerger); super.setTypeSpecificPlugins(pluginsContainer); } @@ -2086,4 +2127,146 @@ public String system(boolean allowFail, String command, String input) throws IOE return cachedSystemCalls.computeIfAbsent(key, asFunction(k -> super.system(allowFail, command, input))); } + private Function> parseDupStrategy(Map extra) { + String onduplicate = extra.get(DUP_STRATEGY); + return Duplication.doDuplicate(onduplicate, this); + } + + + /** + * Handles how duplicate resources are handled in -includeresource + * instruction. + */ + record Duplication(String path, Resource existing, Resource candidate) { + + private static final String DUP_MSG_DEFAULT_OVERWRITE = "includeresource.duplicates: Duplicate overwritten: %s (Consider using the %s directive to handle duplicates.)"; + + enum OnDuplicateCommand { + OVERWRITE, + SKIP, + MERGE, + WARN, + ERROR; + } + + public static Function> doDuplicate(String onduplicate, Processor processor) { + + if (onduplicate == null || onduplicate.isBlank()) { + Consumer warn = dupl -> { + // but only warn if resources are not identical + if (!isIdentical(dupl.existing, dupl.candidate)) { + processor.warning(DUP_MSG_DEFAULT_OVERWRITE, dupl.path, Constants.DUP_STRATEGY); + } + }; + return dupl -> { + + warn.accept(dupl); + return Optional.of(dupl.candidate); + }; + } + + Set commands = new LinkedHashSet<>(); + List tags = new ArrayList(); + + Strings.split(onduplicate) + .forEach(string -> { + try { + commands.add(OnDuplicateCommand.valueOf(string)); + } catch (Exception e) { + tags.add(string); + } + }); + + Consumer error = commands.remove(OnDuplicateCommand.ERROR) + ? dupl -> processor.error("includeresource.duplicates: duplicate found for path %s", dupl.path) + : d -> {}; + Consumer warn = commands.remove(OnDuplicateCommand.WARN) + ? dupl -> processor.warning("includeresource.duplicates: duplicate found for path %s", dupl.path) + : d -> {}; + + String[] tags2 = tags.toArray(new String[0]); + Function> result; + + int what = tags.isEmpty() ? 0 : 1; + if (!commands.isEmpty()) + what += 2; + + result = switch (what) { + // OVERWRITE + case 0 -> dupl -> Optional.of(dupl.candidate); + // try MERGE + case 1 -> { + List mergers = mergePlugins(tags2, processor); + yield dupl -> merge(dupl, mergers); + } + // commands + case 2, 3 -> getCommand(commands, tags2, processor); + default -> throw new UnsupportedOperationException(); + }; + + return dupl -> { + error.accept(dupl); + warn.accept(dupl); + return result.apply(dupl); + }; + } + + private static Function> getCommand(Set commands, + String[] tags, Processor processor) { + + if (commands.size() != 1) { + processor.error("includeresource.duplicates: specifies multiple strategies to handle duplicates: %s", + commands); + return dupl -> Optional.empty(); + } + + if (commands.contains(OnDuplicateCommand.OVERWRITE)) { + return dupl -> Optional.of(dupl.candidate); + } else if (commands.contains(OnDuplicateCommand.SKIP)) { + return dupl -> Optional.empty(); + } else if (commands.contains(OnDuplicateCommand.MERGE)) { + return dupl -> merge(dupl, mergePlugins(tags, processor)); + } else { + throw new UnsupportedOperationException("missed an enum value? " + commands); + } + } + + private static List mergePlugins(String[] tags, Processor processor) { + + List plugins = processor.getPlugins(MergeResources.class, tags); + if (tags.length > 0 && plugins.isEmpty()) { + processor.error("includeresource.duplicates: no plugins found for tags: %s", Arrays.toString(tags)); + } + return plugins; + } + + private static Optional merge(Duplication dupl, List list) { + for (MergeResources mr : list) { + Optional merged = mr.tryMerge(dupl.path, dupl.existing, dupl.candidate); + if (merged.isPresent()) + return merged; + + } + return Optional.empty(); + } + + private static boolean isIdentical(Resource a, Resource b) { + try { + ByteBuffer buffer1 = a.buffer(); + ByteBuffer buffer2 = b.buffer(); + + if (buffer1.remaining() != buffer2.remaining()) { + return false; + } + + return buffer1.equals(buffer2); + + } catch (Exception e) { + return false; + } + } + } + + + } diff --git a/biz.aQute.bndlib/src/aQute/bnd/osgi/Constants.java b/biz.aQute.bndlib/src/aQute/bnd/osgi/Constants.java index 2e5e102f7a..2d45a7ca60 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/osgi/Constants.java +++ b/biz.aQute.bndlib/src/aQute/bnd/osgi/Constants.java @@ -416,6 +416,11 @@ public interface Constants { String EEPROFILE_AUTO_ATTRIBUTE = "auto"; String NONE = "none"; + /* + * -includeresource directives for duplicate handling strategy + */ + String DUP_STRATEGY = "onduplicate:"; + Set directives = Sets.of(SPLIT_PACKAGE_DIRECTIVE, NO_IMPORT_DIRECTIVE, IMPORT_DIRECTIVE, RESOLUTION_DIRECTIVE, INCLUDE_DIRECTIVE, USES_DIRECTIVE, EXCLUDE_DIRECTIVE, KEYSTORE_LOCATION_DIRECTIVE, KEYSTORE_PROVIDER_DIRECTIVE, KEYSTORE_PASSWORD_DIRECTIVE, SIGN_PASSWORD_DIRECTIVE, @@ -423,7 +428,7 @@ public interface Constants { EFFECTIVE_DIRECTIVE, FILTER_DIRECTIVE, FIXUPMESSAGES_RESTRICT_DIRECTIVE, FIXUPMESSAGES_REPLACE_DIRECTIVE, FIXUPMESSAGES_IS_DIRECTIVE, BNDDRIVER_GRADLE, BNDDRIVER_GRADLE_NATIVE, BNDDRIVER_ANT, BNDDRIVER_ECLIPSE, BNDDRIVER_MAVEN, BNDDRIVER_INTELLIJ, BNDDRIVER_SBT, BNDDRIVER_OSMORC, AUGMENT_CAPABILITY_DIRECTIVE, - AUGMENT_REQUIREMENT_DIRECTIVE); + AUGMENT_REQUIREMENT_DIRECTIVE, DUP_STRATEGY); String USES_USES = "<>"; String CURRENT_USES = "@uses"; diff --git a/biz.aQute.bndlib/src/aQute/bnd/osgi/metainf/MetaInfServiceMerger.java b/biz.aQute.bndlib/src/aQute/bnd/osgi/metainf/MetaInfServiceMerger.java new file mode 100644 index 0000000000..2383bcddbb --- /dev/null +++ b/biz.aQute.bndlib/src/aQute/bnd/osgi/metainf/MetaInfServiceMerger.java @@ -0,0 +1,52 @@ +package aQute.bnd.osgi.metainf; + +import java.io.ByteArrayInputStream; +import java.io.SequenceInputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; + +import aQute.bnd.exceptions.Exceptions; +import aQute.bnd.osgi.EmbeddedResource; +import aQute.bnd.osgi.Resource; +import aQute.bnd.service.merge.MergeResources; +import aQute.bnd.service.tags.Tagged; +import aQute.bnd.service.tags.Tags; + +/** + * Knows how to "merge" duplicate files in META-INF/services, by concatenating + * them with a linebreak in between. + */ +public class MetaInfServiceMerger implements MergeResources, Tagged { + + private final static Tags META_INF_SERVICES = Tags.of("metainfservices"); + + + @Override + public Optional tryMerge(String path, Resource a, Resource b) { + + if (!path.startsWith("META-INF/services/")) { + return Optional.empty(); + } + + // do something with a and b + try (SequenceInputStream in = new SequenceInputStream(Collections.enumeration( + Arrays.asList(a.openInputStream(), new ByteArrayInputStream("\n".getBytes()), b.openInputStream())));) { + + long lastModified = Math.max(a.lastModified(), b.lastModified()); + Resource r = new EmbeddedResource(ByteBuffer.wrap(in.readAllBytes()), lastModified); + + return Optional.of(r); + } catch (Exception e) { + throw Exceptions.duck(e); + } + + } + + @Override + public Tags getTags() { + return META_INF_SERVICES; + } + +} diff --git a/biz.aQute.bndlib/src/aQute/bnd/service/merge/MergeResources.java b/biz.aQute.bndlib/src/aQute/bnd/service/merge/MergeResources.java new file mode 100644 index 0000000000..1dad3995cf --- /dev/null +++ b/biz.aQute.bndlib/src/aQute/bnd/service/merge/MergeResources.java @@ -0,0 +1,20 @@ +package aQute.bnd.service.merge; + +import java.util.Optional; + +import aQute.bnd.osgi.Resource; + +/** + * For plugins knowing how to merge two resources. + */ +public interface MergeResources { + /** + * @param path a path (used for validation if the path segment is supported + * for merge) + * @param a first resource + * @param b second resource to be merged with a + * @return the merged resource if possible or an empty optional if merging + * was not possible. + */ + Optional tryMerge(String path, Resource a, Resource b); +} diff --git a/biz.aQute.bndlib/src/aQute/bnd/service/merge/package-info.java b/biz.aQute.bndlib/src/aQute/bnd/service/merge/package-info.java new file mode 100644 index 0000000000..b4399c1b87 --- /dev/null +++ b/biz.aQute.bndlib/src/aQute/bnd/service/merge/package-info.java @@ -0,0 +1,2 @@ +@org.osgi.annotation.versioning.Version("1.0.0") +package aQute.bnd.service.merge; diff --git a/docs/_instructions/includeresource.md b/docs/_instructions/includeresource.md index f1c1fb2d9e..604b57b967 100644 --- a/docs/_instructions/includeresource.md +++ b/docs/_instructions/includeresource.md @@ -1,11 +1,11 @@ --- layout: default class: Builder & Executable -title: -includeresource iclause +title: -includeresource iclause summary: Include resources from the file system --- -The purpose of `-includeresource` is to fill the JAR with non-class resources. In general these come from the file system. For example, today it is very common to have these type of resources in `src/main/resources`. This pattern can easily be simulated by bnd with the `-includeresource` instruction. However, since in OSGi the packaging is so important the `-includeresource` contains a number of options to minimize files on disk and speed up things. +The purpose of `-includeresource` is to fill the JAR with non-class resources. In general these come from the file system. For example, today it is very common to have these type of resources in `src/main/resources`. This pattern can easily be simulated by bnd with the `-includeresource` instruction. However, since in OSGi the packaging is so important the `-includeresource` contains a number of options to minimize files on disk and speed up things. The syntax of the `-includeresource` has become quite complex over time: @@ -18,56 +18,56 @@ The syntax of the `-includeresource` has become quite complex over time: unroll ::= '@' (PATH | URL) ( '!/' SELECTOR )? parameters ::= 'flatten:' | 'recursive:' | 'filter:' | `-preprocessmatchers` -In the case of `assignment` or `source`, the PATH parameter can point to a file or directory. It is also possible to use the name.ext path of a JAR file on the classpath, that is, ignoring the directory. The `source` form will place the resource in the target JAR with only the file name, therefore without any path components. That is, including `src/a/b.c` will result in a resource `b.c` in the root of the target JAR. +In the case of `assignment` or `source`, the PATH parameter can point to a file or directory. It is also possible to use the name.ext path of a JAR file on the classpath, that is, ignoring the directory. The `source` form will place the resource in the target JAR with only the file name, therefore without any path components. That is, including `src/a/b.c` will result in a resource `b.c` in the root of the target JAR. -If the PATH points to a directory, the directory name itself is not used in the target JAR path. If the resource must be placed in a sub directory of the target jar, use the `assignment` form. If the file is not found, bnd will traverse the classpath to see of any entry on the classpath matches the given file name (without the directory) and use that when it matches. The `inline` requires a ZIP or JAR file, which will be completely expanded in the target JAR (except the manifest), unless followed with a file specification. The file specification can be a specific file in the jar or a directory followed by ** or *. The ** indicates recursively and the * indicates one level. If just a directory name is given, it will mean **. +If the PATH points to a directory, the directory name itself is not used in the target JAR path. If the resource must be placed in a sub directory of the target jar, use the `assignment` form. If the file is not found, bnd will traverse the classpath to see of any entry on the classpath matches the given file name (without the directory) and use that when it matches. The `inline` requires a ZIP or JAR file, which will be completely expanded in the target JAR (except the manifest), unless followed with a file specification. The file specification can be a specific file in the jar or a directory followed by **or *. The** indicates recursively and the* indicates one level. If just a directory name is given, it will mean **. The `filter:` directive is an optional filter on the resources. This uses the same format as the instructions. Only the file name is verified against this instruction. Include-Resource: @osgi.jar,[=\ =] {LICENSE.txt},[=\ =] acme/Merge.class=src/acme/Merge.class - + The `-includeresources` instruction will be merged with all properties that starts with `-includeresources*`. ## Preprocessing -A clause contained in curly braces (`{` `}`) are _preprocessed_. While copying the files are run through the macro processor with the builder providing the properties. In the workspace model, all macros of the project are then available. Well known binary resources (as decided by their extension) are ignored. You can override the extension list with the `-preprocessmatchers` instruction. This must be a a selector that takes the source file name as the input. The clause can also specify a local `-preprocessmatchers`. This selector is _prepended_ to the either the default pre process matchers or the set pre process matchers. This allows for the selection or rejection of specific files and/or extensions. +A clause contained in curly braces (`{` `}`) are *preprocessed*. While copying the files are run through the macro processor with the builder providing the properties. In the workspace model, all macros of the project are then available. Well known binary resources (as decided by their extension) are ignored. You can override the extension list with the `-preprocessmatchers` instruction. This must be a a selector that takes the source file name as the input. The clause can also specify a local `-preprocessmatchers`. This selector is *prepended* to the either the default pre process matchers or the set pre process matchers. This allows for the selection or rejection of specific files and/or extensions. -includeresource: {src/main/resources}, {legal=contracts} ## Ignoring Missing Sources -A _source_ in the clause starting with a `-` sign will not generare an error when the source in the clause cannot be located. This is very convenient if you specify an global `-includeresource` instruction in `build.bnd`. For example, `-includeresource.all = -src/main/resources` will not complain when a project does not have a `src/main/resources` directory. Note that the minus sign must be on the _source_. E.g. +A *source* in the clause starting with a `-` sign will not generare an error when the source in the clause cannot be located. This is very convenient if you specify an global `-includeresource` instruction in `build.bnd`. For example, `-includeresource.all = -src/main/resources` will not complain when a project does not have a `src/main/resources` directory. Note that the minus sign must be on the *source*. E.g. `-includeresource.all = {foo=-bar}`, -foo.txt ## Rolling -There are two variants of the rolling _operator_ `@`. It can be used to _roll up_ a directory as a zip or jar file, or it can be used to unroll a jar file into its constituents. +There are two variants of the rolling *operator* `@`. It can be used to *roll up* a directory as a zip or jar file, or it can be used to unroll a jar file into its constituents. If the destination is a path of a `jar` or `zip` file, like `foo/bar/icons.zip` and the source points to a directory in the file system, then the directory will be wrapped up in a Jar and stored as a single entry in the receiving jar file. -includeresource foo/bar/icons.zip=@icons/ -_Unrolling_ is getting the content from another JAR. It is activated by starting the source with an at sign (`@`). The at sign signals that it is not the actual file that should be copied, but the contents of that file should be placed in the destination. +*Unrolling* is getting the content from another JAR. It is activated by starting the source with an at sign (`@`). The at sign signals that it is not the actual file that should be copied, but the contents of that file should be placed in the destination. -includeresource tmp=@jar/foo.jar -The part that follows the at sign (`@`) is either a file path or a URL. Without any extra parameters it will copy all resources except the ones in the `-donotcopy` list and the `META-INF/MANIFEST`. +The part that follows the at sign (`@`) is either a file path or a URL. Without any extra parameters it will copy all resources except the ones in the `-donotcopy` list and the `META-INF/MANIFEST`. -includeresource @jar/foo.jar This is an ideal way to wrap a bundle since it is a full copy. After that one can add additional resources or use `-exportcontents` to export the contained packages in the normal way. In this way, bnd will calculate all imports. -The unrolling can also be restricted with a single _selector_. The syntax for the selector must start with a `!/` marker, which is commonly used for this purpose. After the `!/` the normal selector operators and patterns can be used. For example, if we want to get just the `LICENSE` from a bundle then we can do: +The unrolling can also be restricted with a single *selector*. The syntax for the selector must start with a `!/` marker, which is commonly used for this purpose. After the `!/` the normal selector operators and patterns can be used. For example, if we want to get just the `LICENSE` from a bundle then we can do: -includeresource @jar/foo.jar!/LICENSE However, since selectors can also negate, it is also possible to do the reverse: -includeresource "@jar/foo.jar!/!LICENSE" - + This is a single selector, it is therefore not possible to specify a chain with rejections and selections. However, also a single selector can match multiple file paths: -includeresource @jar/osgi.jar!/!(LICENSE|about.html|org/*) @@ -86,6 +86,69 @@ Wrapping often requires access to a JAR from the repository. It is therefore com -includeresource new.package=@jar/cxf-rt-rs-sse-3.2.5.jar!/(META-INF)/(c*f)/(*);rename:=$2/$1/$3.copy +`onduplicate` - controls duplicate file handling for files with the same path and filename. See **Handling duplicates** below. + +### Handling duplicates + +When unrolling multiple jar files into your target jar then duplicates can occur when multiple files share the same path and filename. By default duplicates overwrite existing files (last wins). +With the `onduplicate` directive you can control this behavior. For example there is the command `onduplicate:=MERGE` which by default is able to merge (append) services files in `/META-INF/services/`. + +**Examples:** + +- `onduplicate:=OVERWRITE` - (default) duplicates overwrite existing files (in other words: last wins) +- `onduplicate:=MERGE` - tries to merge duplicate files under `/META-INF/services/` by default. Other paths are skipped. +- `onduplicate:='MERGE,metainfservices'` - same as MERGE. `metainfservices` is a tag which pulls in Plugins with this tag. Currently there is one default Plugin for handling files under `/META-INF/services/` +- `onduplicate:='sometag'` - tries to merge with Plugins tagged with `sometag` +- `onduplicate:=SKIP` - duplicates are skipped (in other words: first wins) +- `onduplicate:=WARN` - output a warning if there are duplicates +- `onduplicate:=ERROR` output an error if there are duplicates + +`WARN` and `ERROR` can be combined with other commands, while OVERWRITE, MERGE, SKIP are mutually exclusive. +So combinations are possible, e.g. + +- `onduplicate:=WARN,MERGE` - this outputs are warning if duplicates occur, but also tries to merge files under `META-INF/services`. + +#### Example - Handling duplicates + +Let's take Apache FOP as an example. This library comes with 4 jars which are not OSGi bundles. +To use them in OSGi you could wrap them with bnd. Because of some classloading issues related to the ServiceLoader mechanism, one way to do it is to combine them into a single bundle. +One challenge is its extension mechanism which uses ServiceLoaders under `/META-INF/services/`. + +For example the bundle `fop-core` contains a file `META-INF/services/org.apache.xmlgraphics.image.loader.spi.ImagePreloader` with the content: + +``` +org.apache.fop.image.loader.batik.PreloaderWMF +org.apache.fop.image.loader.batik.PreloaderSVG +``` + +`xmlgraphics-commons` contains the same file `META-INF/services/org.apache.xmlgraphics.image.loader.spi.ImagePreloader` with content: + +``` +org.apache.xmlgraphics.image.loader.impl.PreloaderTIFF +org.apache.xmlgraphics.image.loader.impl.PreloaderGIF +org.apache.xmlgraphics.image.loader.impl.PreloaderJPEG +org.apache.xmlgraphics.image.loader.impl.PreloaderBMP +org.apache.xmlgraphics.image.loader.impl.PreloaderEMF +org.apache.xmlgraphics.image.loader.impl.PreloaderEPS +org.apache.xmlgraphics.image.loader.impl.imageio.PreloaderImageIO +org.apache.xmlgraphics.image.loader.impl.PreloaderRawPNG +``` + +If you combine these two jars into a single target jar you want to ensure that both files do not overwrite each other but are merged / appended instead, in order to be a valid ServiceLoader file. + +This can be achieved by the following instructions: + +``` +@${repo;org.apache.xmlgraphics:fop-core;latest}!/*,\ +@${repo;org.apache.xmlgraphics:xmlgraphics-commons;latest}!/*;onduplicate:=MERGE,\ + +``` + +The instructions above can be read like this: + +- the first line `fop-core` can be considered the parent which is unrolled without any special handling. +- the second line is the interesting one: The `onduplicate:=MERGE` directive tells bnd to try merging files. By default bnd is only able to merge files under `META-INF/services`. So bnd will append the duplicate file to the existing file with a line break. + ## Literals For testing purposes it is often necessary to have tiny resources in the bundle. These could of course be placed on the file system but bnd can also generate these on the fly. Since these are defined in the bnd files, the content has full access to the macros. This is done by specifying a `literal` attribute on the clause. @@ -96,8 +159,7 @@ The previous example will create a resource with the given content. ## Flattening & Recurse -When a directory is specified bnd will by default recurse the source and create a similar hierarchy on the destination. - +When a directory is specified bnd will by default recurse the source and create a similar hierarchy on the destination. The recursion and the hierarchy can be controlled with directives. @@ -111,10 +173,9 @@ In this case, only the `hierarchy` directory itself will be copied to the `targe -includeresource target/=hierarchy/;flatten:=true - -## Sample usages: +## Sample usages -### Simple form: +### Simple form | Instruction | Explanation | | --- | --- | @@ -123,7 +184,7 @@ In this case, only the `hierarchy` directory itself will be copied to the `targe | `-includeresource: ${workspace}/LICENSE, {readme.md}` | Copy the LICENSE file residing in the bnd workspace folder (above the project directory) as well as the pre-processed readme.md file (allowing for e.g. variable substitution) in the project folder into the target JAR | | `-includeresource: ${repo;com.acme:foo;latest}` | Copy the com.acme.foo bundle JAR in highest version number found in the bnd workspace repository into the root of the target JAR | -### Assignment form: +### Assignment form | Instruction | Explanation | | --- | --- | @@ -137,9 +198,9 @@ In this case, only the `hierarchy` directory itself will be copied to the `targe | `-includeresource: bsn.txt;literal='${bsn}'` | Create a file named bsn.txt containing the bundle symbolic name (bsn) of this project in the root folder of the target JAR | | `-includeresource: libraries/=lib/;filter:=fancylibrary-*.jar;recursive:=false;lib:=true` or
`-includeresource: libraries/=lib/fancylibrary-*.jar;lib:=true` (as of bndtools 4.2) | Copy a wildcarded library from lib/ into libraries and add it to the bundle classpath | -### Inline form: +### Inline form | Instruction | Explanation | | --- | --- | | `-includeresource: @lib/fancylibrary-3.12.jar!/*` | Extract the contents of lib/fancylibrary-3.12.jar into the root folder of the target JAR, preserving relative paths | -| `-includeresource: @${repo;com.acme.foo;latest}!/!META-INF/*` | Extract the contents of the highest found com.acme.foo version in the bnd workspace repository into the root folder of the target JAR, preserving relative paths, excluding the META-INF/ folder | \ No newline at end of file +| `-includeresource: @${repo;com.acme.foo;latest}!/!META-INF/*` | Extract the contents of the highest found com.acme.foo version in the bnd workspace repository into the root folder of the target JAR, preserving relative paths, excluding the META-INF/ folder |