Fast avro compilation

.gitignore

@@ -26,4 +26,7 @@ project/plugins/project/
 *metals.sbt
 
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
-hs_err_pid*
+hs_err_pid*
+
+.ivy2_credentials
+credentials.sbt

.scalafmt.conf

@@ -0,0 +1,33 @@
+version = 3.5.9
+runner.dialect = scala213
+preset = IntelliJ
+maxColumn = 140
+lineEndings = unix
+align.preset = more
+align.tokens = [
+  {
+    code = "="
+    owners = [{
+      regex = "Term.Assign"
+    },{
+      regex = "Defn.Def"
+    },{
+      regex = "Defn.Val"
+    }]
+  },
+  "%", "%%", ":", "=>", "<-", "~>", "<~"
+]
+align.arrowEnumeratorGenerator = true
+align.openParenCallSite = false
+align.openParenDefnSite = false
+rewrite.rules = [RedundantBraces, RedundantParens, SortImports, PreferCurlyFors, SortModifiers]
+rewriteTokens = {
+  "⇒": "=>"
+  "→": "->"
+  "←": "<-"
+}
+newlines.sometimesBeforeColonInMethodReturnType = true
+spaces.inImportCurlyBraces = true
+danglingParentheses.preset = true
+includeCurlyBraceInSelectChains = true
+project.git = true

Jenkinsfile.gcp

@@ -4,7 +4,7 @@ pipelineEportalLibrary {
     GITHUB_REPO = 'eportal-sbtavrohugger'
     USE_FEATURE_RELEASE_PROCESS = true
     RUN_SBT_TEST = true
-    RUN_SBT_IT_TEST = true
-    HAS_SBT_IT_TEST = true
+    RUN_SBT_IT_TEST = false
+    HAS_SBT_IT_TEST = false
     SBT_CROSS_BUILD_ENABLED = true
 }

build.sbt

@@ -1,50 +1,47 @@
-ThisBuild / organization := "com.julianpeeters"
-ThisBuild / description := "Sbt plugin for compiling Avro to Scala"
-ThisBuild / version := "2.8.4"
-ThisBuild / versionScheme := Some("semver-spec")
-
-enablePlugins(SbtPlugin)
 
 (Global / run / fork) := true
 (Global / run / connectInput) := true
 (Global / run / outputStrategy) := Some(StdoutOutput)
 
-ThisBuild / scalaVersion := "2.12.20"
-ThisBuild / crossSbtVersions := Seq(sbtVersion.value)
-ThisBuild / scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Ywarn-value-discard")
-
-ThisBuild / libraryDependencies ++= Seq(
-  "com.julianpeeters" %% "avrohugger-core" % "2.8.4",
-  "com.julianpeeters" %% "avrohugger-filesorter" % "2.8.4",
-  "io.spray" %% "spray-json" % "1.3.6",
-  "org.specs2" %% "specs2-core" % "4.20.2" % "test")
-
-ThisBuild / publishMavenStyle := true
-Test / publishArtifact := false
-ThisBuild / publishTo := {
-  val nexus = "https://oss.sonatype.org/"
-  if (version.value.trim.endsWith("SNAPSHOT"))
-    Some("snapshots" at nexus + "content/repositories/snapshots")
-  else
-    Some("releases" at nexus + "service/local/staging/deploy/maven2")
-}
-ThisBuild / pomIncludeRepository := { _ => false }
-ThisBuild / licenses := Seq("Apache 2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0"))
-ThisBuild / homepage := Some(url(s"https://github.com/julianpeeters/${name.value}"))
-ThisBuild / pomExtra := (
-  <scm>
-    <url>git://github.com/julianpeeters/sbt-avrohugger.git</url>
-    <connection>scm:git://github.com/julianpeeters/sbt-avrohugger.git</connection>
-  </scm>
-  <developers>
-    <developer>
-      <id>julianpeeters</id>
-      <name>Julian Peeters</name>
-      <url>http://github.com/julianpeeters</url>
-    </developer>
-  </developers>)
+lazy val avrohuggerVersion = "3.0.11-1-g32e8dd1"
 
-ThisBuild / scriptedLaunchOpts := { scriptedLaunchOpts.value ++
-  Seq("-Xmx1024M", "-Dplugin.version=" + version.value)
-}
-ThisBuild / scriptedBufferLog := false
+lazy val sbtavrohugger = (project in file("."))
+  .enablePlugins(SbtPlugin, EPortalSbtSettings)
+  .settings(
+    organization := "com.julianpeeters",
+    description := "Sbt plugin for compiling Avro to Scala",
+    versionScheme := Some("semver-spec"),
+    scalaVersion := "2.12.20",
+    crossSbtVersions := Seq(sbtVersion.value),
+    scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Ywarn-value-discard"),
+    libraryDependencies ++= Seq(
+      "com.julianpeeters" %% "avrohugger-core" % avrohuggerVersion,
+      "com.julianpeeters" %% "avrohugger-filesorter" % avrohuggerVersion,
+      "io.spray" %% "spray-json" % "1.3.6",
+      "org.specs2" %% "specs2-core" % "4.20.2" % "test"),
+    Test / publishArtifact := false,
+    (Global / run / fork) := true,
+    (Global / run / connectInput) := true,
+    (Global / run / outputStrategy) := Some(StdoutOutput),
+    artifactType := ArtifactType.JarLibrary,
+    Release.parentReleaseSettings(Seq.empty, "sbtavrohugger", buildRootProject = true),
+    pomIncludeRepository := { _ => false },
+    licenses := Seq("Apache 2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0")),
+    homepage := Some(url(s"https://github.com/julianpeeters/${name.value}")),
+    pomExtra := (
+      <scm>
+        <url>git://github.com/julianpeeters/sbt-avrohugger.git</url>
+        <connection>scm:git://github.com/julianpeeters/sbt-avrohugger.git</connection>
+      </scm>
+        <developers>
+          <developer>
+            <id>julianpeeters</id>
+            <name>Julian Peeters</name>
+            <url>http://github.com/julianpeeters</url>
+          </developer>
+        </developers>),
+    scriptedLaunchOpts := { scriptedLaunchOpts.value ++
+      Seq("-Xmx1024M", "-Dplugin.version=" + version.value)
+    },
+    scriptedBufferLog := false,
+  )

project/eportal.sbt

@@ -0,0 +1,7 @@
+resolvers += Resolver.url(
+  "eportal artifactory",
+  url(s"""${sys.env.getOrElse("ARTIFACTORY_HOST", "https://eportal-artifacts.fpprod.corp/artifactory")}/${sys.env
+    .getOrElse("ARTIFACTORY_REPO_SBT", "eportal-sbt")}""")
+)(Resolver.defaultIvyPatterns)
+
+addSbtPlugin("com.leonteq.eportal"   % "eportal-sbt-settings" % "1.4.7")

src/main/scala/sbtavrohugger/FileWriter.scala

@@ -1,47 +1,48 @@
 package sbtavrohugger;
 
-import avrohugger.filesorter.{AvdlFileSorter, AvscFileSorter}
+import avrohugger.filesorter.{ AvdlFileSorter, AvscFileSorter }
 import avrohugger.Generator
 import java.io.File
 
 import sbt.Keys._
-import sbt.{Logger, globFilter, singleFileFinder}
+import sbt.{ globFilter, singleFileFinder, Logger }
 import sbt.Path._
 
 object FileWriter {
 
-  private[sbtavrohugger] def generateCaseClasses(
-    generator: Generator,
-    srcDirs: Seq[File],
-    target: File,
-    log: Logger): Set[java.io.File] = {
+  private[sbtavrohugger] def generateCaseClasses(generator: Generator, srcDirs: Seq[File], target: File, log: Logger): Set[java.io.File] = {
     log.info("Considering source directories %s".format(srcDirs.mkString(",")))
     def getSrcFiles(dirs: Seq[File], fileExtension: String) = for {
-      srcDir <- dirs
+      srcDir  <- dirs
       srcFile <- (srcDir ** s"*.$fileExtension").get
     } yield srcFile
-
+
+    generator.clear()
     for (inFile <- AvscFileSorter.sortSchemaFiles(getSrcFiles(srcDirs, "avsc"))) {
       log.info("Compiling AVSC %s to %s".format(inFile, target.getPath))
       generator.fileToFile(inFile, target.getPath)
     }
 
-   for (idlFile <- AvdlFileSorter.sortSchemaFiles(getSrcFiles(srcDirs, "avdl"))) {
+    generator.clear()
+    for (idlFile <- AvdlFileSorter.sortSchemaFiles(getSrcFiles(srcDirs, "avdl"))) {
       log.info("Compiling Avro IDL %s".format(idlFile))
       generator.fileToFile(idlFile, target.getPath)
     }
 
+    generator.clear()
     for (inFile <- getSrcFiles(srcDirs, "avro")) {
       log.info("Compiling Avro datafile %s".format(inFile))
       generator.fileToFile(inFile, target.getPath)
     }
 
+    generator.clear()
     for (protocol <- getSrcFiles(srcDirs, "avpr")) {
       log.info("Compiling Avro protocol %s".format(protocol))
       generator.fileToFile(protocol, target.getPath)
     }
 
-    (target ** ("*.java"|"*.scala")).get.toSet
+    generator.clear()
+    (target ** ("*.java" | "*.scala")).get.toSet
   }
 
 }

src/main/scala/sbtavrohugger/SbtAvrohugger.scala

@@ -1,7 +1,7 @@
 package sbtavrohugger
 
 import avrohugger.Generator
-import avrohugger.format.{SpecificRecord, Standard}
+import avrohugger.format.{ SpecificRecord, Standard }
 import avrohugger.types.AvroScalaTypes
 import java.io.File
 
@@ -11,13 +11,12 @@ import sbt.internal.inc.classpath.ClasspathUtilities
 import java.net.URLClassLoader
 import java.net.URL
 
-/**
- * Simple plugin for generating the Scala sources for Avro IDL, schemas and protocols.
- */
+/** Simple plugin for generating the Scala sources for Avro IDL, schemas and protocols.
+  */
 object SbtAvrohugger extends AutoPlugin {
-  
+
   object autoImport {
-    
+
     // sbt tasks:
     lazy val avroScalaGenerateSpecific = taskKey[Seq[File]]("Generate Scala sources implementing SpecificRecord")
     lazy val avroScalaGenerate         = taskKey[Seq[File]]("Generate Scala sources from avro files")
@@ -34,88 +33,80 @@ object SbtAvrohugger extends AutoPlugin {
     lazy val avroScalaCustomTypes     = settingKey[AvroScalaTypes]("Custom Scala types of generated Scala code")
     lazy val avroScalaCustomNamespace = settingKey[Map[String, String]]("Custom namespace of generated Scala code")
   }
-    
+
   import autoImport._
   override def requires = plugins.JvmPlugin
-  override def trigger = allRequirements
-  
-  lazy val baseSettings = 
+  override def trigger  = allRequirements
+
+  lazy val baseSettings =
     avroSettings ++
-    specificAvroSettings
-    
+      specificAvroSettings
+
   override lazy val projectSettings: Seq[Def.Setting[_]] =
     inConfig(Compile)(baseSettings) ++
-    inConfig(Test)(baseSettings)
+      inConfig(Test)(baseSettings)
 
   val majMinV = """(\d+.\d+).*""".r
 
   // Standard Format
   lazy val avroSettings: Seq[Def.Setting[_]] = Seq(
-    avroScalaSource               := sourceManaged.value / "compiled_avro",
-    avroSourceDirectories         := Seq(sourceDirectory.value / "avro"),
-    avroScalaCustomNamespace      := Map.empty[String, String],
-    avroScalaCustomTypes          := Standard.defaultTypes,
-    avroScalaGenerate / logLevel := (logLevel?? Level.Info).value,
+    avroScalaSource := sourceManaged.value / "compiled_avro",
+    avroSourceDirectories := Seq(sourceDirectory.value / "avro"),
+    avroScalaCustomNamespace := Map.empty[String, String],
+    avroScalaCustomTypes := Standard.defaultTypes,
+    avroScalaGenerate / logLevel := (logLevel ?? Level.Info).value,
     avroScalaGenerate := {
-      val cache = crossTarget.value
-      val srcDirs = avroSourceDirectories.value
-      val targetDir = avroScalaSource.value
-      val out = streams.value
-      val majMinV(scalaV) = scalaVersion.value
-      val customTypes = avroScalaCustomTypes.value
-      val customNamespace = avroScalaCustomNamespace.value
-      val res = (Compile / resourceDirectory).value
-      val old = (Compile/ scalaInstance).value
-      val classLoader = new java.net.URLClassLoader(Array(res.toURI().toURL()), old.loader)
+      val cache                      = crossTarget.value
+      val srcDirs                    = avroSourceDirectories.value
+      val targetDir                  = avroScalaSource.value
+      val out                        = streams.value
+      val majMinV(scalaV)            = scalaVersion.value
+      val customTypes                = avroScalaCustomTypes.value
+      val customNamespace            = avroScalaCustomNamespace.value
+      val res                        = (Compile / resourceDirectory).value
+      val old                        = (Compile / scalaInstance).value
+      val classLoader                = new java.net.URLClassLoader(Array(res.toURI().toURL()), old.loader)
       val isNumberOfFieldsRestricted = scalaV == "2.10"
       val gen = new Generator(
-        format = Standard,
-        avroScalaCustomTypes = Some(customTypes),
+        format                   = Standard,
+        avroScalaCustomTypes     = Some(customTypes),
         avroScalaCustomNamespace = customNamespace,
-        restrictedFieldNumber = isNumberOfFieldsRestricted,
+        restrictedFieldNumber    = isNumberOfFieldsRestricted,
         classLoader,
-        scalaV)
-      val cachedCompile = FileFunction.cached(
-        cache / "avro",
-        inStyle = FilesInfo.lastModified,
-        outStyle = FilesInfo.exists
-      ) { (in: Set[File]) => FileWriter.generateCaseClasses(gen, srcDirs, targetDir, out.log) }
-
+        scalaV
+      )
+      val cachedCompile = FileFunction.cached(cache / "avro", inStyle = FilesInfo.lastModified, outStyle = FilesInfo.exists) {
+        (in: Set[File]) => FileWriter.generateCaseClasses(gen, srcDirs, targetDir, out.log)
+      }
+
       cachedCompile((srcDirs ** "*.av*").get.toSet).toSeq
     }
   )
-  
+
   // SpecificRecord Format
   lazy val specificAvroSettings: Seq[Def.Setting[_]] = Seq(
     avroSpecificScalaSource := sourceManaged.value / "compiled_avro",
     avroSpecificSourceDirectories := Seq(sourceDirectory.value / "avro"),
     avroScalaSpecificCustomTypes := SpecificRecord.defaultTypes,
     avroScalaSpecificCustomNamespace := Map.empty[String, String],
-    avroScalaGenerateSpecific / logLevel := (logLevel?? Level.Info).value,
+    avroScalaGenerateSpecific / logLevel := (logLevel ?? Level.Info).value,
     avroScalaGenerateSpecific := {
-      val cache = crossTarget.value
-      val srcDirs = avroSpecificSourceDirectories.value
-      val targetDir = avroSpecificScalaSource.value
-      val out = streams.value
-      val majMinV(scalaV) = scalaVersion.value
-      val specificCustomTypes = avroScalaSpecificCustomTypes.value
-      val specificCustomNamespace = avroScalaSpecificCustomNamespace.value
-      val res = (Compile / resourceDirectory).value
-      val old = (Compile / scalaInstance).value
-      val classLoader = new java.net.URLClassLoader(Array(res.toURI().toURL()), old.loader)
+      val cache                      = crossTarget.value
+      val srcDirs                    = avroSpecificSourceDirectories.value
+      val targetDir                  = avroSpecificScalaSource.value
+      val out                        = streams.value
+      val majMinV(scalaV)            = scalaVersion.value
+      val specificCustomTypes        = avroScalaSpecificCustomTypes.value
+      val specificCustomNamespace    = avroScalaSpecificCustomNamespace.value
+      val res                        = (Compile / resourceDirectory).value
+      val old                        = (Compile / scalaInstance).value
+      val classLoader                = new java.net.URLClassLoader(Array(res.toURI().toURL()), old.loader)
       val isNumberOfFieldsRestricted = scalaV == "2.10"
-      val gen = new Generator(
-        SpecificRecord,
-        Some(specificCustomTypes),
-        specificCustomNamespace,
-        isNumberOfFieldsRestricted,
-        classLoader,
-        scalaV)
-      val cachedCompile = FileFunction.cached(
-        cache / "avro",
-        inStyle = FilesInfo.lastModified,
-        outStyle = FilesInfo.exists
-      ) { (in: Set[File]) => FileWriter.generateCaseClasses(gen, srcDirs, targetDir, out.log) }
+      val gen =
+        new Generator(SpecificRecord, Some(specificCustomTypes), specificCustomNamespace, isNumberOfFieldsRestricted, classLoader, scalaV)
+      val cachedCompile = FileFunction.cached(cache / "avro", inStyle = FilesInfo.lastModified, outStyle = FilesInfo.exists) {
+        (in: Set[File]) => FileWriter.generateCaseClasses(gen, srcDirs, targetDir, out.log)
+      }
       cachedCompile((srcDirs ** "*.av*").get.toSet).toSeq
     }
   )