diff --git a/internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala b/internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala index 53f768423..eb54d69c2 100644 --- a/internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala +++ b/internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala @@ -56,7 +56,7 @@ class AnalysisFormatBenchmark { ) writeAll( "-ref-cbin-nosort", - ConsistentFileAnalysisStore.binary(_, ReadWriteMappers.getEmptyMappers, sort = false), + ConsistentFileAnalysisStore.binary(_, ReadWriteMappers.getEmptyMappers, reproducible = false), cached ) println("Sizes:") @@ -94,7 +94,11 @@ class AnalysisFormatBenchmark { bh.consume( writeAll( "-test-cbin-nosort", - ConsistentFileAnalysisStore.binary(_, ReadWriteMappers.getEmptyMappers, sort = false), + ConsistentFileAnalysisStore.binary( + _, + ReadWriteMappers.getEmptyMappers, + reproducible = false + ), cached ) ) @@ -104,7 +108,7 @@ class AnalysisFormatBenchmark { cached.foreach { case (s, a) => val ser = new NullSerializer - val af = new ConsistentAnalysisFormat(ReadWriteMappers.getEmptyMappers, sort = true) + val af = new ConsistentAnalysisFormat(ReadWriteMappers.getEmptyMappers, reproducible = true) af.write(ser, a.getAnalysis, a.getMiniSetup) bh.consume(ser.count) } @@ -115,7 +119,8 @@ class AnalysisFormatBenchmark { cached.foreach { case (s, a) => val ser = new NullSerializer - val af = new ConsistentAnalysisFormat(ReadWriteMappers.getEmptyMappers, sort = false) + val af = + new ConsistentAnalysisFormat(ReadWriteMappers.getEmptyMappers, reproducible = false) af.write(ser, a.getAnalysis, a.getMiniSetup) bh.consume(ser.count) } diff --git a/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala b/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala index 78aea1106..271de3492 100644 --- a/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala +++ b/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala @@ -36,7 +36,7 @@ import Compat._ * - Faster serialization and deserialization than the existing binary format. * - Smaller implementation than either of the existing formats. */ -class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { +class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, reproducible: Boolean) { import ConsistentAnalysisFormat._ private[this] final val VERSION = 1100029 @@ -52,6 +52,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { writeAPIs(out, analysis0.apis, setup.storeApis()) writeSourceInfos(out, analysis0.infos) // we do not read or write the Compilations + // as zinc does not use Compilations from deserialized analysis out.int(VERSION) out.end() } @@ -64,6 +65,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { val apis = readAPIs(in, setup.storeApis()) val infos = readSourceInfos(in) // we do not read or write the Compilations + // as zinc does not use Compilations from deserialized analysis val compilations = Compilations.of(Nil) readVersion(in) in.end() @@ -76,9 +78,12 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { name: String, map: scala.collection.Iterable[(String, V)], perEntry: Int = 1 - )(f: V => Unit): Unit = - if (sort) out.writeSortedStringMap(name, map, perEntry)(f) + )(f: V => Unit): Unit = { + // For reproducible output, need to write strings in sorted order + // otherwise strings may be written in different order resulting in different output + if (reproducible) out.writeSortedStringMap(name, map, perEntry)(f) else out.writeColl(name, map, perEntry + 1) { kv => out.string(kv._1); f(kv._2) } + } private[this] def readVersion(in: Deserializer): Unit = { val ver = in.int() @@ -158,7 +163,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { out.string(ac.provenance()) out.int(ac.extraHash()) val nh0 = ac.nameHashes() - val nh = if (nh0.length > 1 && sort) { + val nh = if (nh0.length > 1 && reproducible) { val nh = nh0.clone() Arrays.sort(nh, nameHashComparator) nh @@ -199,14 +204,17 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { } private[this] def writeAPIs(out: Serializer, apis: APIs, storeApis: Boolean): Unit = { - def write(n: String, m: Map[String, AnalyzedClass]): Unit = + def write(n: String, m: Map[String, AnalyzedClass]): Unit = { writeMaybeSortedStringMap( out, n, - m.mapValues(_.withCompilationTimestamp(DefaultCompilationTimestamp)) + if (reproducible) m.mapValues(_.withCompilationTimestamp(DefaultCompilationTimestamp)) + else m ) { ac => writeAnalyzedClass(out, ac, storeApis) } + } + write("internal", apis.internal) write("external", apis.external) } @@ -353,7 +361,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { rel.forwardMap.view.map { case (k, vs) => kf(k) -> vs } ) { vs => val a = vs.iterator.map(vf).toArray - if (sort) Arrays.sort(a, implicitly[Ordering[String]]) + if (reproducible) Arrays.sort(a, implicitly[Ordering[String]]) out.writeColl("item", a)(out.string) } def wrS(name: String, rel: Relation[String, String]): Unit = @@ -428,11 +436,11 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { if (sc.contains(UseScope.PatMatTarget)) i += 4 (un.name, i.toByte) }.toArray.groupBy(_._2) - val groups = if (sort) groups0.toVector.sortBy(_._1) else groups0 + val groups = if (reproducible) groups0.toVector.sortBy(_._1) else groups0 out.writeColl("groups", groups, 2) { case (g, gNames) => out.byte(g) val names = gNames.map(_._1) - if (sort) Arrays.sort(names, implicitly[Ordering[String]]) + if (reproducible) Arrays.sort(names, implicitly[Ordering[String]]) out.writeStringColl("names", names) } } diff --git a/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentFileAnalysisStore.scala b/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentFileAnalysisStore.scala index 2cec0baa3..469c3a72c 100644 --- a/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentFileAnalysisStore.scala +++ b/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentFileAnalysisStore.scala @@ -35,12 +35,12 @@ object ConsistentFileAnalysisStore { def text( file: File, mappers: ReadWriteMappers, - sort: Boolean = true, + reproducible: Boolean = true, parallelism: Int = Runtime.getRuntime.availableProcessors() ): XAnalysisStore = new AStore( file, - new ConsistentAnalysisFormat(mappers, sort), + new ConsistentAnalysisFormat(mappers, reproducible), SerializerFactory.text, parallelism ) @@ -49,7 +49,7 @@ object ConsistentFileAnalysisStore { binary( file, mappers = ReadWriteMappers.getEmptyMappers(), - sort = true, + reproducible = true, ) def binary( @@ -59,18 +59,18 @@ object ConsistentFileAnalysisStore { binary( file, mappers, - sort = true, + reproducible = true, ) def binary( file: File, mappers: ReadWriteMappers, - sort: Boolean, + reproducible: Boolean, parallelism: Int = Runtime.getRuntime.availableProcessors() ): XAnalysisStore = new AStore( file, - new ConsistentAnalysisFormat(mappers, sort), + new ConsistentAnalysisFormat(mappers, reproducible), SerializerFactory.binary, parallelism ) diff --git a/zinc/src/main/scala/sbt/internal/inc/MixedAnalyzingCompiler.scala b/zinc/src/main/scala/sbt/internal/inc/MixedAnalyzingCompiler.scala index b08b699d1..5a934a91e 100644 --- a/zinc/src/main/scala/sbt/internal/inc/MixedAnalyzingCompiler.scala +++ b/zinc/src/main/scala/sbt/internal/inc/MixedAnalyzingCompiler.scala @@ -507,7 +507,7 @@ object MixedAnalyzingCompiler { useTextAnalysis = useTextAnalysis, useConsistent = false, mappers = ReadWriteMappers.getEmptyMappers(), - sort = true, + reproducible = true, parallelism = Runtime.getRuntime.availableProcessors(), ) @@ -516,7 +516,7 @@ object MixedAnalyzingCompiler { useTextAnalysis: Boolean, useConsistent: Boolean, mappers: ReadWriteMappers, - sort: Boolean, + reproducible: Boolean, parallelism: Int, ): AnalysisStore = { val fileStore = (useTextAnalysis, useConsistent) match { @@ -526,7 +526,7 @@ object MixedAnalyzingCompiler { ConsistentFileAnalysisStore.binary( file = analysisFile.toFile, mappers = mappers, - sort = sort, + reproducible = reproducible, parallelism = parallelism, ) case (true, false) => @@ -535,7 +535,7 @@ object MixedAnalyzingCompiler { ConsistentFileAnalysisStore.text( file = analysisFile.toFile, mappers = mappers, - sort = sort, + reproducible = reproducible, parallelism = parallelism, ) }