From 7e3c86905f9d5f1fe8c8922810980a169dd953c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Harabie=C5=84?= Date: Sat, 24 Aug 2024 04:22:47 +0200 Subject: [PATCH] Update avro to 1.11.3 (backport to 2.18 branch) (#512) * Update avro to 1.11.3 Namespace for nested classes no longer ends with '$'. This is how avro library generates schema since version 1.9. See: AVRO-2143 Please note that resolution of nested classes without '$' was implemented long ago in c5705492. Fixes #167 * Remove '$' from Avro namespaces when Java class has multiple nesting levels Avro before 1.11 was generating schemas with '$' in namespace if class had multiple nesting levels. To fix compatibility with avro 1.11+ make sure all dollar characters are replaced by dots. Related: AVRO-2757 --- avro/pom.xml | 2 +- .../avro/schema/AvroSchemaHelper.java | 32 ++++++++++---- .../avro/annotation/AvroNamespaceTest.java | 42 ++++++++++++++++++- .../interop/annotations/AvroAliasTest.java | 4 +- release-notes/CREDITS-2.x | 4 ++ release-notes/VERSION-2.x | 3 ++ 6 files changed, 74 insertions(+), 13 deletions(-) diff --git a/avro/pom.xml b/avro/pom.xml index 71b14bad8..37eb21c15 100644 --- a/avro/pom.xml +++ b/avro/pom.xml @@ -47,7 +47,7 @@ abstractions. org.apache.avro avro - 1.8.2 + 1.11.3 diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java index 007e2efc6..00f0592f6 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java @@ -108,7 +108,9 @@ protected static String getNamespace(Class cls) { // NOTE: was reverted in 2.8.8, but is enabled for Jackson 2.9. Class enclosing = cls.getEnclosingClass(); if (enclosing != null) { - return enclosing.getName() + "$"; + // 23-Aug-2024: Changed as per [dataformats-binary#167] + // Enclosing class may also be nested + return enclosing.getName().replace('$', '.'); } Package pkg = cls.getPackage(); return (pkg == null) ? "" : pkg.getName(); @@ -351,6 +353,8 @@ public static String getFullName(Schema schema) { if (namespace == null) { return name; } + // 23-Aug-2024: [dataformats-binary#167] Still needed for backwards-compatibility + // with schemas that use dollar sign for nested classes (Apache Avro before 1.9) final int len = namespace.length(); if (namespace.charAt(len-1) == '$') { return namespace + name; @@ -441,13 +445,25 @@ private static String _resolve(FullNameKey key) { // Check if this is a nested class // 19-Sep-2020, tatu: This is a horrible, horribly inefficient and all-around // wrong mechanism. To be abolished if possible. - final String nestedClassName = key.nameWithSeparator('$'); - try { - Class.forName(nestedClassName); - return nestedClassName; - } catch (ClassNotFoundException e) { - // Could not find a nested class, must be a regular class - return key.nameWithSeparator('.'); + // 23-Aug-2024:[dataformats-binary#167] Based on SpecificData::getClass + // from Apache Avro. Initially assume that namespace is a Java package + StringBuilder sb = new StringBuilder(key.nameWithSeparator('.')); + int lastDot = sb.length(); + while (true) { + try { + // Try to resolve the class + String className = sb.toString(); + Class.forName(className); + return className; + } catch (ClassNotFoundException e) { + // Class does not exist - perhaps last dot is actually a nested class + lastDot = sb.lastIndexOf(".", lastDot); + if (lastDot == -1) { + // No more dots so we are unable to resolve, should we throw an exception? + return key.nameWithSeparator('.'); + } + sb.setCharAt(lastDot, '$'); + } } } } diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/annotation/AvroNamespaceTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/annotation/AvroNamespaceTest.java index fe4639f40..ce3160ccb 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/annotation/AvroNamespaceTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/annotation/AvroNamespaceTest.java @@ -23,6 +23,15 @@ enum EnumWithoutAvroNamespaceAnnotation {FOO, BAR;} @AvroNamespace("EnumWithAvroNamespaceAnnotation.namespace") enum EnumWithAvroNamespaceAnnotation {FOO, BAR;} + static class Foo { + static class Bar { + static class ClassWithMultipleNestingLevels { + } + + enum EnumWithMultipleNestingLevels {FOO, BAR;} + } + } + @Test public void class_without_AvroNamespace_test() throws Exception { // GIVEN @@ -35,7 +44,7 @@ public void class_without_AvroNamespace_test() throws Exception { // THEN assertThat(actualSchema.getNamespace()) - .isEqualTo("com.fasterxml.jackson.dataformat.avro.annotation.AvroNamespaceTest$"); + .isEqualTo("com.fasterxml.jackson.dataformat.avro.annotation.AvroNamespaceTest"); } @Test @@ -53,6 +62,21 @@ public void class_with_AvroNamespace_test() throws Exception { .isEqualTo("ClassWithAvroNamespaceAnnotation.namespace"); } + @Test + public void class_with_multiple_nesting_levels_test() throws Exception { + // GIVEN + AvroMapper mapper = new AvroMapper(); + AvroSchemaGenerator gen = new AvroSchemaGenerator(); + + // WHEN + mapper.acceptJsonFormatVisitor(Foo.Bar.ClassWithMultipleNestingLevels.class, gen); + Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + // THEN + assertThat(actualSchema.getNamespace()) + .isEqualTo("com.fasterxml.jackson.dataformat.avro.annotation.AvroNamespaceTest.Foo.Bar"); + } + @Test public void enum_without_AvroNamespace_test() throws Exception { // GIVEN @@ -65,7 +89,7 @@ public void enum_without_AvroNamespace_test() throws Exception { // THEN assertThat(actualSchema.getNamespace()) - .isEqualTo("com.fasterxml.jackson.dataformat.avro.annotation.AvroNamespaceTest$"); + .isEqualTo("com.fasterxml.jackson.dataformat.avro.annotation.AvroNamespaceTest"); } @Test @@ -83,4 +107,18 @@ public void enum_with_AvroNamespace_test() throws Exception { .isEqualTo("EnumWithAvroNamespaceAnnotation.namespace"); } + @Test + public void enum_with_multiple_nesting_levels_test() throws Exception { + // GIVEN + AvroMapper mapper = new AvroMapper(); + AvroSchemaGenerator gen = new AvroSchemaGenerator(); + + // WHEN + mapper.acceptJsonFormatVisitor(Foo.Bar.EnumWithMultipleNestingLevels.class, gen); + Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + // THEN + assertThat(actualSchema.getNamespace()) + .isEqualTo("com.fasterxml.jackson.dataformat.avro.annotation.AvroNamespaceTest.Foo.Bar"); + } } diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/interop/annotations/AvroAliasTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/interop/annotations/AvroAliasTest.java index 2ce2e43c6..357f56410 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/interop/annotations/AvroAliasTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/interop/annotations/AvroAliasTest.java @@ -15,7 +15,7 @@ public class AvroAliasTest extends InteropTestBase { - @AvroAlias(alias = "Employee", space = "com.fasterxml.jackson.dataformat.avro.AvroTestBase$") + @AvroAlias(alias = "Employee", space = "com.fasterxml.jackson.dataformat.avro.AvroTestBase") public static class NewEmployee { public String name; @@ -40,7 +40,7 @@ public static class AliasedNameEmployee { public AliasedNameEmployee boss; } - @AvroAlias(alias = "Size", space = "com.fasterxml.jackson.dataformat.avro.AvroTestBase$") + @AvroAlias(alias = "Size", space = "com.fasterxml.jackson.dataformat.avro.AvroTestBase") public enum NewSize { SMALL, LARGE; diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 8430cd469..d88ef9453 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -328,6 +328,10 @@ Yoann Vernageau (@yvrng) when source is an empty `InputStream` (2.17.1) +Rafał Harabień (@rafalh) + * Contributed fix for #167: (avro) Incompatibility with Avro >=1.9.0 (upgrade to Avro 1.11.3) + (2.18.0) + PJ Fanning (@pjfanning) * Contributed #484: (protobuf) Rework synchronization in `ProtobufMapper` (2.18.0) diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index 3d2686002..614765bb1 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -16,6 +16,9 @@ Active maintainers: 2.18.0 (not yet released) +#167: (avro) Incompatibility with Avro >=1.9.0 (upgrade to Avro 1.11.3) + (reported by @Sage-Pierce) + (fix contributed by Rafał H) #484: (protobuf) Rework synchronization in `ProtobufMapper` (contributed by @pjfanning) #494: (avro) Avro Schema generation: allow mapping Java Enum properties to