Merge pull request sbt#1479 from Friendseeker/reproducible-toggleable

[1.x] Make reproducibility toggleable for `ConsistentAnalysisFormat`

internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala

@@ -56,7 +56,7 @@ class AnalysisFormatBenchmark {
     )
     writeAll(
       "-ref-cbin-nosort",
-      ConsistentFileAnalysisStore.binary(_, ReadWriteMappers.getEmptyMappers, sort = false),
+      ConsistentFileAnalysisStore.binary(_, ReadWriteMappers.getEmptyMappers, reproducible = false),
       cached
     )
     println("Sizes:")
@@ -94,7 +94,11 @@ class AnalysisFormatBenchmark {
     bh.consume(
       writeAll(
         "-test-cbin-nosort",
-        ConsistentFileAnalysisStore.binary(_, ReadWriteMappers.getEmptyMappers, sort = false),
+        ConsistentFileAnalysisStore.binary(
+          _,
+          ReadWriteMappers.getEmptyMappers,
+          reproducible = false
+        ),
         cached
       )
     )
@@ -104,7 +108,7 @@ class AnalysisFormatBenchmark {
     cached.foreach {
       case (s, a) =>
         val ser = new NullSerializer
-        val af = new ConsistentAnalysisFormat(ReadWriteMappers.getEmptyMappers, sort = true)
+        val af = new ConsistentAnalysisFormat(ReadWriteMappers.getEmptyMappers, reproducible = true)
         af.write(ser, a.getAnalysis, a.getMiniSetup)
         bh.consume(ser.count)
     }
@@ -115,7 +119,8 @@ class AnalysisFormatBenchmark {
     cached.foreach {
       case (s, a) =>
         val ser = new NullSerializer
-        val af = new ConsistentAnalysisFormat(ReadWriteMappers.getEmptyMappers, sort = false)
+        val af =
+          new ConsistentAnalysisFormat(ReadWriteMappers.getEmptyMappers, reproducible = false)
         af.write(ser, a.getAnalysis, a.getMiniSetup)
         bh.consume(ser.count)
     }

internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala

@@ -36,7 +36,7 @@ import Compat._
  * - Faster serialization and deserialization than the existing binary format.
  * - Smaller implementation than either of the existing formats.
  */
-class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) {
+class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, reproducible: Boolean) {
   import ConsistentAnalysisFormat._
 
   private[this] final val VERSION = 1100029
@@ -52,6 +52,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) {
     writeAPIs(out, analysis0.apis, setup.storeApis())
     writeSourceInfos(out, analysis0.infos)
     // we do not read or write the Compilations
+    // as zinc does not use Compilations from deserialized analysis
     out.int(VERSION)
     out.end()
   }
@@ -64,6 +65,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) {
     val apis = readAPIs(in, setup.storeApis())
     val infos = readSourceInfos(in)
     // we do not read or write the Compilations
+    // as zinc does not use Compilations from deserialized analysis
     val compilations = Compilations.of(Nil)
     readVersion(in)
     in.end()
@@ -76,9 +78,12 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) {
       name: String,
       map: scala.collection.Iterable[(String, V)],
       perEntry: Int = 1
-  )(f: V => Unit): Unit =
-    if (sort) out.writeSortedStringMap(name, map, perEntry)(f)
+  )(f: V => Unit): Unit = {
+    // For reproducible output, need to write strings in sorted order
+    // otherwise strings may be written in different order resulting in different output
+    if (reproducible) out.writeSortedStringMap(name, map, perEntry)(f)
     else out.writeColl(name, map, perEntry + 1) { kv => out.string(kv._1); f(kv._2) }
+  }
 
   private[this] def readVersion(in: Deserializer): Unit = {
     val ver = in.int()
@@ -158,7 +163,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) {
       out.string(ac.provenance())
       out.int(ac.extraHash())
       val nh0 = ac.nameHashes()
-      val nh = if (nh0.length > 1 && sort) {
+      val nh = if (nh0.length > 1 && reproducible) {
         val nh = nh0.clone()
         Arrays.sort(nh, nameHashComparator)
         nh
@@ -199,14 +204,17 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) {
   }
 
   private[this] def writeAPIs(out: Serializer, apis: APIs, storeApis: Boolean): Unit = {
-    def write(n: String, m: Map[String, AnalyzedClass]): Unit =
+    def write(n: String, m: Map[String, AnalyzedClass]): Unit = {
       writeMaybeSortedStringMap(
         out,
         n,
-        m.mapValues(_.withCompilationTimestamp(DefaultCompilationTimestamp))
+        if (reproducible) m.mapValues(_.withCompilationTimestamp(DefaultCompilationTimestamp))
+        else m
       ) { ac =>
         writeAnalyzedClass(out, ac, storeApis)
       }
+    }
+
     write("internal", apis.internal)
     write("external", apis.external)
   }
@@ -353,7 +361,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) {
         rel.forwardMap.view.map { case (k, vs) => kf(k) -> vs }
       ) { vs =>
         val a = vs.iterator.map(vf).toArray
-        if (sort) Arrays.sort(a, implicitly[Ordering[String]])
+        if (reproducible) Arrays.sort(a, implicitly[Ordering[String]])
         out.writeColl("item", a)(out.string)
       }
     def wrS(name: String, rel: Relation[String, String]): Unit =
@@ -428,11 +436,11 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) {
         if (sc.contains(UseScope.PatMatTarget)) i += 4
         (un.name, i.toByte)
       }.toArray.groupBy(_._2)
-      val groups = if (sort) groups0.toVector.sortBy(_._1) else groups0
+      val groups = if (reproducible) groups0.toVector.sortBy(_._1) else groups0
       out.writeColl("groups", groups, 2) { case (g, gNames) =>
         out.byte(g)
         val names = gNames.map(_._1)
-        if (sort) Arrays.sort(names, implicitly[Ordering[String]])
+        if (reproducible) Arrays.sort(names, implicitly[Ordering[String]])
         out.writeStringColl("names", names)
       }
     }

internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentFileAnalysisStore.scala

@@ -35,12 +35,12 @@ object ConsistentFileAnalysisStore {
   def text(
       file: File,
       mappers: ReadWriteMappers,
-      sort: Boolean = true,
+      reproducible: Boolean = true,
       parallelism: Int = Runtime.getRuntime.availableProcessors()
   ): XAnalysisStore =
     new AStore(
       file,
-      new ConsistentAnalysisFormat(mappers, sort),
+      new ConsistentAnalysisFormat(mappers, reproducible),
       SerializerFactory.text,
       parallelism
     )
@@ -49,7 +49,7 @@ object ConsistentFileAnalysisStore {
     binary(
       file,
       mappers = ReadWriteMappers.getEmptyMappers(),
-      sort = true,
+      reproducible = true,
     )
 
   def binary(
@@ -59,18 +59,18 @@ object ConsistentFileAnalysisStore {
     binary(
       file,
       mappers,
-      sort = true,
+      reproducible = true,
     )
 
   def binary(
       file: File,
       mappers: ReadWriteMappers,
-      sort: Boolean,
+      reproducible: Boolean,
       parallelism: Int = Runtime.getRuntime.availableProcessors()
   ): XAnalysisStore =
     new AStore(
       file,
-      new ConsistentAnalysisFormat(mappers, sort),
+      new ConsistentAnalysisFormat(mappers, reproducible),
       SerializerFactory.binary,
       parallelism
     )

zinc/src/main/scala/sbt/internal/inc/MixedAnalyzingCompiler.scala

@@ -507,7 +507,7 @@ object MixedAnalyzingCompiler {
       useTextAnalysis = useTextAnalysis,
       useConsistent = false,
       mappers = ReadWriteMappers.getEmptyMappers(),
-      sort = true,
+      reproducible = true,
       parallelism = Runtime.getRuntime.availableProcessors(),
     )
 
@@ -516,7 +516,7 @@ object MixedAnalyzingCompiler {
       useTextAnalysis: Boolean,
       useConsistent: Boolean,
       mappers: ReadWriteMappers,
-      sort: Boolean,
+      reproducible: Boolean,
       parallelism: Int,
   ): AnalysisStore = {
     val fileStore = (useTextAnalysis, useConsistent) match {
@@ -526,7 +526,7 @@ object MixedAnalyzingCompiler {
         ConsistentFileAnalysisStore.binary(
           file = analysisFile.toFile,
           mappers = mappers,
-          sort = sort,
+          reproducible = reproducible,
           parallelism = parallelism,
         )
       case (true, false) =>
@@ -535,7 +535,7 @@ object MixedAnalyzingCompiler {
         ConsistentFileAnalysisStore.text(
           file = analysisFile.toFile,
           mappers = mappers,
-          sort = sort,
+          reproducible = reproducible,
           parallelism = parallelism,
         )
     }