From b3a83c2a1526364bf154e9da9322e844be0acc94 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 6 Sep 2024 01:18:32 -0400 Subject: [PATCH 01/18] Avoid using ZipError --- .../src/main/java/sbt/internal/inc/zip/ZipCentralDir.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/zinc-classfile/src/main/java/sbt/internal/inc/zip/ZipCentralDir.java b/internal/zinc-classfile/src/main/java/sbt/internal/inc/zip/ZipCentralDir.java index 20fe9e1c9..c0ca6d8aa 100644 --- a/internal/zinc-classfile/src/main/java/sbt/internal/inc/zip/ZipCentralDir.java +++ b/internal/zinc-classfile/src/main/java/sbt/internal/inc/zip/ZipCentralDir.java @@ -302,8 +302,8 @@ public int hashCode() { IndexNode child; // 1st child } - private static void zerror(String msg) { - throw new ZipError(msg); + private static void zerror(String msg) throws ZipException { + throw new ZipException(msg); } private void buildNodeTree() { From ac211dcae45c6e52973c5c524c6e0fd180c8dbdc Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 6 Sep 2024 23:04:33 -0400 Subject: [PATCH 02/18] Fix build --- .github/workflows/ci.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3af480be..48653ce17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,20 +64,19 @@ jobs: shell: bash run: | sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks" - - name: Checkout Develop Branch (4, 5) + - name: Checkout 1.10.x Branch (4, 5) if: ${{ github.event_name == 'pull_request' && (matrix.jobtype == 4 || matrix.jobtype == 5) }} uses: actions/checkout@v4 with: clean: false - ref: develop - - name: Benchmark (Scalac) against Develop Branch (4) + ref: 1.10.x + - name: Benchmark (Scalac) against 1.10.x Branch (4) if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 4 }} shell: bash run: | - sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "zincBenchmarks/jmh:clean" "runBenchmarks" - - name: Benchmark (Shapeless) against Develop Branch (5) + sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" + - name: Benchmark (Shapeless) against 1.10.x Branch (5) if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 5 }} shell: bash run: | - sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "zincBenchmarks/jmh:clean" "runBenchmarks" - + sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" From ed57d0e51fa21b7b046e2cd84096caa86be1636c Mon Sep 17 00:00:00 2001 From: rochala Date: Wed, 11 Sep 2024 18:47:04 +0200 Subject: [PATCH 03/18] Copy bytes directly instead of using scala.reflect.io.Streamable suboptimal methods --- .../src/main/scala/xsbt/AbstractZincFile.scala | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/compiler-bridge/src/main/scala/xsbt/AbstractZincFile.scala b/internal/compiler-bridge/src/main/scala/xsbt/AbstractZincFile.scala index 2398339a2..b3352a8fd 100644 --- a/internal/compiler-bridge/src/main/scala/xsbt/AbstractZincFile.scala +++ b/internal/compiler-bridge/src/main/scala/xsbt/AbstractZincFile.scala @@ -12,7 +12,6 @@ package xsbt import xsbti.{ PathBasedFile, VirtualFile } -import scala.reflect.io.Streamable private trait AbstractZincFile extends scala.reflect.io.AbstractFile { def underlying: VirtualFile @@ -25,7 +24,22 @@ private final class ZincPlainFile private[xsbt] (val underlying: PathBasedFile) private final class ZincVirtualFile private[xsbt] (val underlying: VirtualFile) extends scala.reflect.io.VirtualFile(underlying.name, underlying.id) with AbstractZincFile { - Streamable.closing(output)(_.write(Streamable.bytes(underlying.input))) // fill in the content + val buffer = new Array[Byte](4096) + + val in = underlying.input() + val output0 = output + + try { + var readBytes = in.read(buffer) + + while (readBytes != -1) { + output0.write(buffer, 0, readBytes) + readBytes = in.read(buffer) + } + } finally { + in.close() + output0.close() + } } private object AbstractZincFile { From 0929cf689403b0702f8921540b335aaf424ed630 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Tue, 17 Sep 2024 00:16:27 -0700 Subject: [PATCH 04/18] early return in transitive invalidation if possible --- .../sbt/internal/inc/IncrementalCommon.scala | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala index 492b0e63f..1548fb3cf 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala @@ -464,16 +464,10 @@ private[inc] abstract class IncrementalCommon( val dependsOnClass = findClassDependencies(_, relations) val firstClassInvalidation: Set[String] = { val invalidated = - if (invalidateTransitively) { - // Invalidate by brute force (normally happens when we've done more than 3 incremental runs) - IncrementalCommon.transitiveDeps(initial, log)(dependsOnClass) - } else { - changes.apiChanges.flatMap(invalidateClassesInternally(relations, _, isScalaClass)).toSet - } - val included = includeTransitiveInitialInvalidations(initial, invalidated, dependsOnClass) - log.debug("Final step, transitive dependencies:\n\t" + included) - included + changes.apiChanges.flatMap(invalidateClassesInternally(relations, _, isScalaClass)).toSet + includeTransitiveInitialInvalidations(initial, invalidated, dependsOnClass) } + log.debug("Final step, transitive dependencies:\n\t" + firstClassInvalidation) // Invalidate classes linked with a class file that is produced by more than one source file val secondClassInvalidation = IncrementalCommon.invalidateNamesProducingSameClassFile(relations) @@ -506,7 +500,13 @@ private[inc] abstract class IncrementalCommon( Set.empty } else { if (invalidateTransitively) { - newInvalidations ++ recompiledClasses + val firstClassTransitiveInvalidation = includeTransitiveInitialInvalidations( + initial, + IncrementalCommon.transitiveDeps(initial, log)(dependsOnClass), + dependsOnClass + ) + log.debug("Invalidate by brute force:\n\t" + firstClassTransitiveInvalidation) + firstClassTransitiveInvalidation ++ secondClassInvalidation ++ thirdClassInvalidation } else { firstClassInvalidation ++ secondClassInvalidation ++ thirdClassInvalidation } From 39aa5135d83b9457bdaafd7daac989a03880648b Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Wed, 18 Sep 2024 21:32:49 -0700 Subject: [PATCH 05/18] Add back Carston's transitive invalidation logic Probably not needed anymore but better safe than sorry --- .../src/main/scala/sbt/internal/inc/IncrementalCommon.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala index 1548fb3cf..4bdef942d 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala @@ -506,7 +506,7 @@ private[inc] abstract class IncrementalCommon( dependsOnClass ) log.debug("Invalidate by brute force:\n\t" + firstClassTransitiveInvalidation) - firstClassTransitiveInvalidation ++ secondClassInvalidation ++ thirdClassInvalidation + firstClassTransitiveInvalidation ++ secondClassInvalidation ++ thirdClassInvalidation ++ recompiledClasses } else { firstClassInvalidation ++ secondClassInvalidation ++ thirdClassInvalidation } From 77e7a2c9ca8a764249135a6109468c1135147f1b Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 20 Sep 2024 19:54:48 +0000 Subject: [PATCH 06/18] Update sbt, util-control, util-interface, ... to 1.10.2 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index ee4c672cd..0b699c305 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.10.1 +sbt.version=1.10.2 From cdc924c3ac274a72f3e029190cb3da2249718bcc Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Fri, 20 Sep 2024 19:55:01 +0000 Subject: [PATCH 07/18] Update scalacheck to 1.18.1 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 80aa604ec..065d7351d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -75,7 +75,7 @@ object Dependencies { val parserCombinator = "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2" val sbinary = "org.scala-sbt" %% "sbinary" % "0.5.1" val scalaXml = "org.scala-lang.modules" %% "scala-xml" % "2.3.0" - val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.18.0" + val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.18.1" val scalatest = "org.scalatest" %% "scalatest" % "3.2.19" val verify = "com.eed3si9n.verify" %% "verify" % "2.0.1" val sjsonnew = Def.setting { From 34aa51af3c10e4b0b0dbc7325e4dda503ce5048b Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Sun, 22 Sep 2024 20:54:45 -0700 Subject: [PATCH 08/18] Add unit test to demonstrate the 1 cycle saving --- .../source-dependencies/transitive-early-stopping/A.scala | 3 +++ .../source-dependencies/transitive-early-stopping/B.scala | 3 +++ .../source-dependencies/transitive-early-stopping/C.scala | 4 ++++ .../source-dependencies/transitive-early-stopping/D.scala | 3 +++ .../transitive-early-stopping/changes/A.scala | 3 +++ .../source-dependencies/transitive-early-stopping/test | 5 +++++ 6 files changed, 21 insertions(+) create mode 100644 zinc/src/sbt-test/source-dependencies/transitive-early-stopping/A.scala create mode 100644 zinc/src/sbt-test/source-dependencies/transitive-early-stopping/B.scala create mode 100644 zinc/src/sbt-test/source-dependencies/transitive-early-stopping/C.scala create mode 100644 zinc/src/sbt-test/source-dependencies/transitive-early-stopping/D.scala create mode 100644 zinc/src/sbt-test/source-dependencies/transitive-early-stopping/changes/A.scala create mode 100644 zinc/src/sbt-test/source-dependencies/transitive-early-stopping/test diff --git a/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/A.scala b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/A.scala new file mode 100644 index 000000000..d26475934 --- /dev/null +++ b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/A.scala @@ -0,0 +1,3 @@ +object A { + val a: Int = 0 +} \ No newline at end of file diff --git a/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/B.scala b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/B.scala new file mode 100644 index 000000000..e2eb7968d --- /dev/null +++ b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/B.scala @@ -0,0 +1,3 @@ +object B { + val b = A.a +} \ No newline at end of file diff --git a/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/C.scala b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/C.scala new file mode 100644 index 000000000..75b2d7c07 --- /dev/null +++ b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/C.scala @@ -0,0 +1,4 @@ +object C { + val c1 = B.b + val c2: Int = 0; +} \ No newline at end of file diff --git a/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/D.scala b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/D.scala new file mode 100644 index 000000000..497b75205 --- /dev/null +++ b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/D.scala @@ -0,0 +1,3 @@ +object D { + val d: Int = C.c2 +} \ No newline at end of file diff --git a/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/changes/A.scala b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/changes/A.scala new file mode 100644 index 000000000..822a53e8f --- /dev/null +++ b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/changes/A.scala @@ -0,0 +1,3 @@ +object A { + val a: String = "" +} \ No newline at end of file diff --git a/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/test b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/test new file mode 100644 index 000000000..10867af1a --- /dev/null +++ b/zinc/src/sbt-test/source-dependencies/transitive-early-stopping/test @@ -0,0 +1,5 @@ +> compile + +$ copy-file changes/A.scala A.scala +> compile +> checkIterations 4 \ No newline at end of file From f2a5923abb94c2d76220043a7c369d6f7cc8fd92 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Sun, 22 Sep 2024 21:04:05 -0700 Subject: [PATCH 09/18] Replace XOR with hashCode --- .../zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala index 37e93decb..b16e6b6d5 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala @@ -886,7 +886,7 @@ private final class AnalysisCallback( val externalParentsAPI = externalParents.map(analysis.apis.externalAPI) val internalParentsAPI = internalParents.map(analysis.apis.internalAPI) val parentsHashes = (externalParentsAPI ++ internalParentsAPI).map(_.extraHash()) - parentsHashes.fold(currentExtraHash)(_ ^ _) + parentsHashes.hashCode case None => currentExtraHash } } From 6cbcb28b7b2d9e48e8d5f0c7b62ff59ec5c84c75 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Sun, 22 Sep 2024 21:28:52 -0700 Subject: [PATCH 10/18] Add currentExtraHash to be part of hashCode() --- .../zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala index b16e6b6d5..5db24c6db 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala @@ -886,7 +886,7 @@ private final class AnalysisCallback( val externalParentsAPI = externalParents.map(analysis.apis.externalAPI) val internalParentsAPI = internalParents.map(analysis.apis.internalAPI) val parentsHashes = (externalParentsAPI ++ internalParentsAPI).map(_.extraHash()) - parentsHashes.hashCode + (parentsHashes + currentExtraHash).hashCode() case None => currentExtraHash } } From 698f3994e4ab41fc5a0ebf67937bbd3dfad97e55 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sun, 22 Sep 2024 17:10:13 +0000 Subject: [PATCH 11/18] Update util-control, util-interface, ... to 1.10.2 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 065d7351d..7387f699d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -16,7 +16,7 @@ object Dependencies { val scala212_213 = Seq(defaultScalaVersion, scala213) private val ioVersion = nightlyVersion.getOrElse("1.10.0") - private val utilVersion = nightlyVersion.getOrElse("1.10.1") + private val utilVersion = nightlyVersion.getOrElse("1.10.2") private val sbtIO = "org.scala-sbt" %% "io" % ioVersion From 95edec7b77cc85fd073a9d4eb140f6ecf5faa357 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Tue, 24 Sep 2024 00:10:37 -0700 Subject: [PATCH 12/18] Run AnalysisFormatBenchmark during CI --- .github/workflows/ci.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48653ce17..86b9faeec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: - os: ubuntu-latest java: 21 jobtype: 5 + - os: ubuntu-latest + java: 21 + jobtype: 6 runs-on: ${{ matrix.os }} steps: - name: Checkout @@ -64,19 +67,29 @@ jobs: shell: bash run: | sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks" - - name: Checkout 1.10.x Branch (4, 5) + - name: Benchmark (AnalysisFormatBenchmark) (6) + if: ${{ matrix.jobtype == 6 }} + shell: bash + run: | + sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" + - name: Checkout Target Branch (4, 5) if: ${{ github.event_name == 'pull_request' && (matrix.jobtype == 4 || matrix.jobtype == 5) }} uses: actions/checkout@v4 with: clean: false - ref: 1.10.x - - name: Benchmark (Scalac) against 1.10.x Branch (4) + ref: ${{ github.event.pull_request.base.ref }} + - name: Benchmark (Scalac) against Target Branch (4) if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 4 }} shell: bash run: | sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" - - name: Benchmark (Shapeless) against 1.10.x Branch (5) + - name: Benchmark (Shapeless) against Target Branch (5) if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 5 }} shell: bash run: | sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" + - name: Benchmark (AnalysisFormatBenchmark) against Target Branch (6) + if: ${{ matrix.jobtype == 6 }} + shell: bash + run: | + sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" From 4ecd7edee320d9314dd567f9ce681a9c1b587134 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Thu, 26 Sep 2024 00:23:33 -0700 Subject: [PATCH 13/18] Remove text & protobuf format from benchmarking --- .github/workflows/ci.yml | 13 ++++++------- .../test/scala/xsbt/AnalysisFormatBenchmark.scala | 14 -------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86b9faeec..43f5dc7c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,25 +71,24 @@ jobs: if: ${{ matrix.jobtype == 6 }} shell: bash run: | - sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" - - name: Checkout Target Branch (4, 5) - if: ${{ github.event_name == 'pull_request' && (matrix.jobtype == 4 || matrix.jobtype == 5) }} + sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks" + - name: Checkout Target Branch (4-6) + if: ${{ github.event_name == 'pull_request' && (matrix.jobtype >= 4 || matrix.jobtype <= 6) }} uses: actions/checkout@v4 with: - clean: false ref: ${{ github.event.pull_request.base.ref }} - name: Benchmark (Scalac) against Target Branch (4) if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 4 }} shell: bash run: | - sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" + sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "runBenchmarks" - name: Benchmark (Shapeless) against Target Branch (5) if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 5 }} shell: bash run: | - sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" + sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks" - name: Benchmark (AnalysisFormatBenchmark) against Target Branch (6) if: ${{ matrix.jobtype == 6 }} shell: bash run: | - sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "zincBenchmarks/Jmh/clean" "runBenchmarks" + sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks" diff --git a/internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala b/internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala index 353a52eb3..2c219c79c 100644 --- a/internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala +++ b/internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala @@ -62,26 +62,12 @@ class AnalysisFormatBenchmark { if (temp != null) IO.delete(temp) } - @Benchmark - def readBinary(bh: Blackhole): Unit = bh.consume(readAll("", FileAnalysisStore.binary(_))) - - @Benchmark - def readText(bh: Blackhole): Unit = bh.consume(readAll("-ref-text", FileAnalysisStore.text(_))) - @Benchmark def readConsistentBinary(bh: Blackhole): Unit = bh.consume( readAll("-ref-cbin", ConsistentFileAnalysisStore.binary(_, ReadWriteMappers.getEmptyMappers)) ) - @Benchmark - def writeBinary(bh: Blackhole): Unit = - bh.consume(writeAll("-test-bin", FileAnalysisStore.binary(_), cached)) - - @Benchmark - def writeText(bh: Blackhole): Unit = - bh.consume(writeAll("-test-text", FileAnalysisStore.text(_), cached)) - @Benchmark def writeConsistentBinary(bh: Blackhole): Unit = bh.consume( From 1e41c1a6bf36bbaf8c40de0a271905062bdc0c15 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Thu, 26 Sep 2024 23:39:38 -0700 Subject: [PATCH 14/18] Add missing condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43f5dc7c7..b936e63be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: run: | sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks" - name: Benchmark (AnalysisFormatBenchmark) against Target Branch (6) - if: ${{ matrix.jobtype == 6 }} + if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 6 }} shell: bash run: | sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks" From 2855609eed404da0956ce514901f9a53a5eaad27 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:26:42 -0800 Subject: [PATCH 15/18] Emits SourceInfos when incremental compilation fails Zinc does not provide a way to surface list of all problems present in a codebase. This PR builds SourceInfos in Analysis Callback and in `handleErrors`, the compiler bridge then throws an exception containing the SourceInfos so build tools like sbt can access the informations. Closes #932 --- .../src/main/scala/xsbt/CompilerBridge.scala | 29 +++++++++++++++---- .../main/java/xsbti/AnalysisCallback3.java | 21 ++++++++++++++ .../src/main/java/xsbti/CompileFailed2.java | 22 ++++++++++++++ .../scala/sbt/internal/inc/RawCompiler.scala | 4 ++- .../scala/sbt/internal/inc/Incremental.scala | 21 +++++++++++++- .../src/main/scala/xsbti/TestCallback.scala | 8 +++-- .../src/main/scala/xsbti/TestSourceInfo.scala | 23 +++++++++++++++ .../main/scala/xsbti/TestSourceInfos.scala | 24 +++++++++++++++ .../inc/IncrementalCompilerImpl.scala | 8 +++++ .../sbt/inc/IncrementalCompilerSpec.scala | 21 ++++++++++++++ 10 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 internal/compiler-interface/src/main/java/xsbti/AnalysisCallback3.java create mode 100644 internal/compiler-interface/src/main/java/xsbti/CompileFailed2.java create mode 100644 internal/zinc-testing/src/main/scala/xsbti/TestSourceInfo.scala create mode 100644 internal/zinc-testing/src/main/scala/xsbti/TestSourceInfos.scala diff --git a/internal/compiler-bridge/src/main/scala/xsbt/CompilerBridge.scala b/internal/compiler-bridge/src/main/scala/xsbt/CompilerBridge.scala index b491b0e71..77260e7ac 100644 --- a/internal/compiler-bridge/src/main/scala/xsbt/CompilerBridge.scala +++ b/internal/compiler-bridge/src/main/scala/xsbt/CompilerBridge.scala @@ -18,6 +18,8 @@ import scala.collection.mutable import scala.reflect.io.AbstractFile import scala.tools.nsc.CompilerCommand import Log.debug +import xsbti.compile.analysis.ReadSourceInfos + import java.io.File /** @@ -49,6 +51,16 @@ class InterfaceCompileFailed( override val toString: String ) extends xsbti.CompileFailed +class InterfaceCompileFailed2( + val arguments: Array[String], + val sourceInfos: ReadSourceInfos, + override val toString: String +) extends xsbti.CompileFailed2 { + import scala.collection.JavaConverters._ + val problems: Array[Problem] = + sourceInfos.getAllSourceInfos.values().asScala.flatMap(_.getReportedProblems).toArray +} + class InterfaceCompileCancelled(val arguments: Array[String], override val toString: String) extends xsbti.CompileCancelled @@ -149,8 +161,8 @@ private final class CachedCompiler0( underlyingReporter: DelegatingReporter, compileProgress: CompileProgress ): Unit = { - // cast down to AnalysisCallback2 - val callback2 = callback.asInstanceOf[xsbti.AnalysisCallback2] + // cast down to AnalysisCallback3 + val callback3 = callback.asInstanceOf[xsbti.AnalysisCallback3] compiler.set(callback, underlyingReporter) if (command.shouldStopWithInfo) { @@ -165,7 +177,7 @@ private final class CachedCompiler0( run.compileFiles(sources) processUnreportedWarnings(run) underlyingReporter.problems.foreach(p => - callback2.problem2( + callback3.problem2( p.category, p.position, p.message, @@ -180,8 +192,10 @@ private final class CachedCompiler0( } underlyingReporter.printSummary() - if (!noErrors(underlyingReporter)) - handleErrors(underlyingReporter, log) + if (!noErrors(underlyingReporter)) { + val infos = callback3.getSourceInfos + handleErrors(infos, log) + } // the case where we cancelled compilation _after_ some compilation errors got reported // will be handled by line above so errors still will be reported properly just potentially not @@ -195,6 +209,11 @@ private final class CachedCompiler0( throw new InterfaceCompileFailed(args, dreporter.problems, "Compilation failed") } + def handleErrors(sourceInfos: ReadSourceInfos, log: Logger): Nothing = { + debug(log, "Compilation failed (CompilerInterface)") + throw new InterfaceCompileFailed2(args, sourceInfos, "Compilation failed") + } + def handleCompilationCancellation(dreporter: DelegatingReporter, log: Logger): Nothing = { assert(dreporter.cancelled, "We should get here only if when compilation got cancelled") debug(log, "Compilation cancelled (CompilerInterface)") diff --git a/internal/compiler-interface/src/main/java/xsbti/AnalysisCallback3.java b/internal/compiler-interface/src/main/java/xsbti/AnalysisCallback3.java new file mode 100644 index 000000000..649ad752e --- /dev/null +++ b/internal/compiler-interface/src/main/java/xsbti/AnalysisCallback3.java @@ -0,0 +1,21 @@ +/* + * Zinc - The incremental compiler for Scala. + * Copyright Scala Center, Lightbend, and Mark Harrah + * + * Licensed under Apache License 2.0 + * SPDX-License-Identifier: Apache-2.0 + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package xsbti; +import xsbti.compile.analysis.ReadSourceInfos; + +/** + * Extension to {@link AnalysisCallback2}. + * Similar to {@link AnalysisCallback2}, it serves as compatibility layer for Scala compilers. + */ +public interface AnalysisCallback3 extends AnalysisCallback2 { + ReadSourceInfos getSourceInfos(); +} diff --git a/internal/compiler-interface/src/main/java/xsbti/CompileFailed2.java b/internal/compiler-interface/src/main/java/xsbti/CompileFailed2.java new file mode 100644 index 000000000..e5d4260a3 --- /dev/null +++ b/internal/compiler-interface/src/main/java/xsbti/CompileFailed2.java @@ -0,0 +1,22 @@ +/* + * Zinc - The incremental compiler for Scala. + * Copyright Scala Center, Lightbend, and Mark Harrah + * + * Licensed under Apache License 2.0 + * SPDX-License-Identifier: Apache-2.0 + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package xsbti; + +import xsbti.compile.analysis.ReadSourceInfos; + +public abstract class CompileFailed2 extends CompileFailed { + /** + * Returns SourceInfos containing problems for each file. + * This includes problems found by most recent compilation run. + */ + public abstract ReadSourceInfos sourceInfos(); +} diff --git a/internal/zinc-compile-core/src/main/scala/sbt/internal/inc/RawCompiler.scala b/internal/zinc-compile-core/src/main/scala/sbt/internal/inc/RawCompiler.scala index c90f8eda0..b22165f43 100644 --- a/internal/zinc-compile-core/src/main/scala/sbt/internal/inc/RawCompiler.scala +++ b/internal/zinc-compile-core/src/main/scala/sbt/internal/inc/RawCompiler.scala @@ -15,6 +15,7 @@ package inc import java.nio.file.Path import sbt.internal.util.FeedbackProvidedException +import xsbti.compile.analysis.ReadSourceInfos import xsbti.compile.{ ClasspathOptions, ScalaInstance => XScalaInstance } /** @@ -98,11 +99,12 @@ class CompileFailed( val arguments: Array[String], override val toString: String, val problems: Array[xsbti.Problem], + val sourceInfosOption: Option[ReadSourceInfos], cause: Throwable ) extends xsbti.CompileFailed(cause) with FeedbackProvidedException { def this(arguments: Array[String], toString: String, problems: Array[xsbti.Problem]) = { - this(arguments, toString, problems, null) + this(arguments, toString, problems, None, null) } } diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala index 37e93decb..508af7f52 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala @@ -590,7 +590,7 @@ private final class AnalysisCallback( progress: Option[CompileProgress], incHandlerOpt: Option[Incremental.IncrementalCallback], log: Logger -) extends xsbti.AnalysisCallback2 { +) extends xsbti.AnalysisCallback3 { import Incremental.CompileCycleResult // This must have a unique value per AnalysisCallback @@ -1067,6 +1067,25 @@ private final class AnalysisCallback( } } + def getSourceInfos: SourceInfos = { + // Collect Source Info from current run + val sources = reporteds.keySet ++ unreporteds.keySet ++ mainClasses.keySet + val sourceToInfo = sources.map { source => + val info = SourceInfos.makeInfo( + getOrNil(reporteds.iterator.map { case (k, v) => k -> v.asScala.toSeq }.toMap, source), + getOrNil(unreporteds.iterator.map { case (k, v) => k -> v.asScala.toSeq }.toMap, source), + getOrNil(mainClasses.iterator.map { case (k, v) => k -> v.asScala.toSeq }.toMap, source) + ) + (source, info) + }.toMap + val sourceInfoFromCurrentRun = SourceInfos.of(sourceToInfo) + // Collect reported problems from previous run + incHandlerOpt.map(_.previousAnalysisPruned) match { + case Some(prevAnalysis) => prevAnalysis.infos ++ sourceInfoFromCurrentRun + case None => sourceInfoFromCurrentRun + } + } + override def apiPhaseCompleted(): Unit = { // If we know we're done with cycles (presumably because all sources were invalidated) we can store early analysis // and picke data now. Otherwise, we need to wait for dependency information to decide if there are more cycles. diff --git a/internal/zinc-testing/src/main/scala/xsbti/TestCallback.scala b/internal/zinc-testing/src/main/scala/xsbti/TestCallback.scala index 94161317f..9426b8c6b 100644 --- a/internal/zinc-testing/src/main/scala/xsbti/TestCallback.scala +++ b/internal/zinc-testing/src/main/scala/xsbti/TestCallback.scala @@ -15,12 +15,12 @@ import java.io.File import java.nio.file.Path import java.{ util => ju } import ju.Optional - -import xsbti.api.{ DependencyContext, ClassLike } +import xsbti.api.{ ClassLike, DependencyContext } +import xsbti.compile.analysis.ReadSourceInfos import scala.collection.mutable.ArrayBuffer -class TestCallback extends AnalysisCallback2 { +class TestCallback extends AnalysisCallback3 { case class TestUsedName(name: String, scopes: ju.EnumSet[UseScope]) val classDependencies = new ArrayBuffer[(String, String, DependencyContext)] @@ -153,6 +153,8 @@ class TestCallback extends AnalysisCallback2 { override def isPickleJava: Boolean = false override def getPickleJarPair = Optional.empty() + + override def getSourceInfos: ReadSourceInfos = new TestSourceInfos } object TestCallback { diff --git a/internal/zinc-testing/src/main/scala/xsbti/TestSourceInfo.scala b/internal/zinc-testing/src/main/scala/xsbti/TestSourceInfo.scala new file mode 100644 index 000000000..6ded26142 --- /dev/null +++ b/internal/zinc-testing/src/main/scala/xsbti/TestSourceInfo.scala @@ -0,0 +1,23 @@ +/* + * Zinc - The incremental compiler for Scala. + * Copyright Scala Center, Lightbend, and Mark Harrah + * + * Licensed under Apache License 2.0 + * SPDX-License-Identifier: Apache-2.0 + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package xsbti + +import xsbti.compile.analysis.SourceInfo + +class TestSourceInfo extends SourceInfo { + + override def getReportedProblems: Array[Problem] = Array.empty[Problem] + + override def getUnreportedProblems: Array[Problem] = Array.empty[Problem] + + override def getMainClasses: Array[String] = Array.empty[String] +} diff --git a/internal/zinc-testing/src/main/scala/xsbti/TestSourceInfos.scala b/internal/zinc-testing/src/main/scala/xsbti/TestSourceInfos.scala new file mode 100644 index 000000000..060eec1c7 --- /dev/null +++ b/internal/zinc-testing/src/main/scala/xsbti/TestSourceInfos.scala @@ -0,0 +1,24 @@ +/* + * Zinc - The incremental compiler for Scala. + * Copyright Scala Center, Lightbend, and Mark Harrah + * + * Licensed under Apache License 2.0 + * SPDX-License-Identifier: Apache-2.0 + * + * See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + */ + +package xsbti + +import xsbti.compile.analysis.{ ReadSourceInfos, SourceInfo } + +import java.util + +class TestSourceInfos extends ReadSourceInfos { + + override def get(sourceFile: VirtualFileRef): SourceInfo = new TestSourceInfo + + override def getAllSourceInfos: util.Map[VirtualFileRef, SourceInfo] = + new util.HashMap[VirtualFileRef, SourceInfo]() +} diff --git a/zinc/src/main/scala/sbt/internal/inc/IncrementalCompilerImpl.scala b/zinc/src/main/scala/sbt/internal/inc/IncrementalCompilerImpl.scala index 889400427..b954db27c 100644 --- a/zinc/src/main/scala/sbt/internal/inc/IncrementalCompilerImpl.scala +++ b/zinc/src/main/scala/sbt/internal/inc/IncrementalCompilerImpl.scala @@ -331,11 +331,19 @@ class IncrementalCompilerImpl extends IncrementalCompiler { try { compilerRun } catch { + case e: xsbti.CompileFailed2 => throw new sbt.internal.inc.CompileFailed( + e.arguments, + e.toString, + e.problems, + Some(e.sourceInfos()), + e, + ) // just ignore case e: xsbti.CompileFailed => throw new sbt.internal.inc.CompileFailed( e.arguments, e.toString, e.problems, + None, e ) // just ignore case e: CompileFailed => throw e // just ignore diff --git a/zinc/src/test/scala/sbt/inc/IncrementalCompilerSpec.scala b/zinc/src/test/scala/sbt/inc/IncrementalCompilerSpec.scala index c812e6898..ff1a4c7b3 100644 --- a/zinc/src/test/scala/sbt/inc/IncrementalCompilerSpec.scala +++ b/zinc/src/test/scala/sbt/inc/IncrementalCompilerSpec.scala @@ -240,4 +240,25 @@ class IncrementalCompilerSpec extends BaseCompilerSpec { } } finally comp.close() } + + it should "emit SourceInfos when incremental compilation fails" in withTmpDir { + tmp => + val project = VirtualSubproject(tmp.toPath / "p1") + val comp = project.setup.createCompiler() + val s1 = "object A { final val i = 1" + val f1 = StringVirtualFile("A.scala", s1) + try { + val exception = intercept[CompileFailed] { + comp.compile(f1) + } + exception.sourceInfosOption match { + case Some(sourceInfos) => + assert( + !sourceInfos.getAllSourceInfos.isEmpty, + "Expected non-empty source infos" + ) + case None => fail("Expected sourceInfos") + } + } finally comp.close() + } } From a996c7392b13830a6cb5d01db00ccdd1dc61f8be Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Sun, 29 Sep 2024 20:10:08 -0700 Subject: [PATCH 16/18] Add log for third class invalidation --- .../src/main/scala/sbt/internal/inc/IncrementalCommon.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala index 4bdef942d..dfe6a869f 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala @@ -492,6 +492,7 @@ private[inc] abstract class IncrementalCommon( val transitive = IncrementalCommon.transitiveDeps(recompiledClasses, log)(dependsOnClass) (transitive -- recompiledClasses).filter(analysis.apis.internalAPI(_).hasMacro) } + log.debug(s"Invalidated macros due to upstream dependencies change: ${thirdClassInvalidation}") val newInvalidations = (firstClassInvalidation -- recompiledClasses) ++ secondClassInvalidation ++ thirdClassInvalidation From ae2bf27c95f857c4104816acccb90e368053603d Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Tue, 1 Oct 2024 00:04:50 -0700 Subject: [PATCH 17/18] tell Scala Steward to ignore jgit update --- .scala-steward.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.scala-steward.conf b/.scala-steward.conf index fd3928446..00e3eba3a 100644 --- a/.scala-steward.conf +++ b/.scala-steward.conf @@ -5,5 +5,6 @@ pullRequests.frequency = "14 days" updates.ignore = [ # as per discussion on sbt/zinc#1236, this is # "if it ain't broke don't fix it" territory - { groupId = "com.google.protobuf" } + { groupId = "com.google.protobuf" }, + { groupId = "org.eclipse.jgit" } ] From 5a014fc8a959f17ef4eee82c9b44461032676688 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Wed, 2 Oct 2024 00:14:23 -0700 Subject: [PATCH 18/18] Make zinc-scripted show up in IntelliJ --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index 4afb5cd89..dc91d3e53 100644 --- a/build.sbt +++ b/build.sbt @@ -686,7 +686,6 @@ lazy val zincScripted = (projectMatrix in internalPath / "zinc-scripted") .enablePlugins(BuildInfoPlugin) .settings( baseSettings, - ideSkipProject := true, // otherwise IntelliJ complains publish / skip := true, name := "zinc Scripted", Compile / buildInfo := Nil, // Only generate build info for tests