Merge branch '1.10.x' into merge-1-x

.github/workflows/ci.yml

@@ -30,6 +30,9 @@ jobs:
           - os: ubuntu-latest
             java: 21
             jobtype: 5
+          - os: ubuntu-latest
+            java: 21
+            jobtype: 6
     runs-on: ${{ matrix.os }}
     steps:
     - name: Checkout
@@ -48,7 +51,7 @@ jobs:
       if: ${{ matrix.jobtype == 2 }}
       shell: bash
       run: |
-        sbt -v -Dfile.encoding=UTF-8 -Dsbt.supershell=never "Test/compile" "crossTestBridges" "zincRoot/test" 
+        sbt -v -Dfile.encoding=UTF-8 -Dsbt.supershell=never "Test/compile" "crossTestBridges" "zincRoot/test"
       # "zincScripted/Test/run"
     - name: Build and test (3)
       if: ${{ matrix.jobtype == 3 }}
@@ -65,8 +68,13 @@ jobs:
       shell: bash
       run: |
         sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks"
-    - name: Checkout Target Branch (4, 5)
-      if: ${{ github.event_name == 'pull_request' && (matrix.jobtype == 4 || matrix.jobtype == 5) }}
+    - name: Benchmark (AnalysisFormatBenchmark) (6)
+      if: ${{ matrix.jobtype == 6 }}
+      shell: bash
+      run: |
+        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks"
+    - name: Checkout Target Branch (4-6)
+      if: ${{ github.event_name == 'pull_request' && (matrix.jobtype >= 4 && matrix.jobtype <= 6) }}
       uses: actions/checkout@v4
       with:
         clean: false
@@ -75,10 +83,14 @@ jobs:
       if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 4 }}
       shell: bash
       run: |
-        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "zincBenchmarks3/Jmh/clean" "runBenchmarks"
+        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "runBenchmarks"
     - name: Benchmark (Shapeless) against Target Branch (5)
       if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 5 }}
       shell: bash
       run: |
-        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "zincBenchmarks3/Jmh/clean" "runBenchmarks"
-        
+        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks"
+    - name: Benchmark (AnalysisFormatBenchmark) against Target Branch (6)
+      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 6 }}
+      shell: bash
+      run: |
+        sbt -v -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks"

.scala-steward.conf

@@ -5,5 +5,6 @@ pullRequests.frequency = "14 days"
 updates.ignore = [
   # as per discussion on sbt/zinc#1236, this is
   # "if it ain't broke don't fix it" territory
-  { groupId = "com.google.protobuf" }
+  { groupId = "com.google.protobuf" },
+  { groupId = "org.eclipse.jgit" }
 ]

build.sbt

@@ -645,7 +645,6 @@ lazy val zincScripted = (projectMatrix in internalPath / "zinc-scripted")
   .enablePlugins(BuildInfoPlugin)
   .settings(
     baseSettings,
-    ideSkipProject := true, // otherwise IntelliJ complains
     publish / skip := true,
     name := "zinc Scripted",
     Compile / buildInfo := Nil, // Only generate build info for tests

internal/compiler-bridge/src/main/scala/xsbt/AbstractZincFile.scala

@@ -12,7 +12,6 @@
 package xsbt
 
 import xsbti.{ PathBasedFile, VirtualFile }
-import scala.reflect.io.Streamable
 
 private trait AbstractZincFile extends scala.reflect.io.AbstractFile {
   def underlying: VirtualFile
@@ -25,7 +24,22 @@ private final class ZincPlainFile private[xsbt] (val underlying: PathBasedFile)
 private final class ZincVirtualFile private[xsbt] (val underlying: VirtualFile)
     extends scala.reflect.io.VirtualFile(underlying.name, underlying.id)
     with AbstractZincFile {
-  Streamable.closing(output)(_.write(Streamable.bytes(underlying.input))) // fill in the content
+  val buffer = new Array[Byte](4096)
+
+  val in = underlying.input()
+  val output0 = output
+
+  try {
+    var readBytes = in.read(buffer)
+
+    while (readBytes != -1) {
+      output0.write(buffer, 0, readBytes)
+      readBytes = in.read(buffer)
+    }
+  } finally {
+    in.close()
+    output0.close()
+  }
 }
 
 private object AbstractZincFile {

internal/compiler-bridge/src/main/scala/xsbt/CompilerBridge.scala

@@ -18,6 +18,8 @@ import scala.collection.mutable
 import scala.reflect.io.AbstractFile
 import scala.tools.nsc.CompilerCommand
 import Log.debug
+import xsbti.compile.analysis.ReadSourceInfos
+
 import java.io.File
 
 /**
@@ -49,6 +51,16 @@ class InterfaceCompileFailed(
     override val toString: String
 ) extends xsbti.CompileFailed
 
+class InterfaceCompileFailed2(
+    val arguments: Array[String],
+    val sourceInfos: ReadSourceInfos,
+    override val toString: String
+) extends xsbti.CompileFailed2 {
+  import scala.collection.JavaConverters._
+  val problems: Array[Problem] =
+    sourceInfos.getAllSourceInfos.values().asScala.flatMap(_.getReportedProblems).toArray
+}
+
 class InterfaceCompileCancelled(val arguments: Array[String], override val toString: String)
     extends xsbti.CompileCancelled
 
@@ -149,8 +161,8 @@ private final class CachedCompiler0(
       underlyingReporter: DelegatingReporter,
       compileProgress: CompileProgress
   ): Unit = {
-    // cast down to AnalysisCallback2
-    val callback2 = callback.asInstanceOf[xsbti.AnalysisCallback2]
+    // cast down to AnalysisCallback3
+    val callback3 = callback.asInstanceOf[xsbti.AnalysisCallback3]
 
     compiler.set(callback, underlyingReporter)
     if (command.shouldStopWithInfo) {
@@ -165,7 +177,7 @@ private final class CachedCompiler0(
       run.compileFiles(sources)
       processUnreportedWarnings(run)
       underlyingReporter.problems.foreach(p =>
-        callback2.problem2(
+        callback3.problem2(
           p.category,
           p.position,
           p.message,
@@ -180,8 +192,10 @@ private final class CachedCompiler0(
     }
 
     underlyingReporter.printSummary()
-    if (!noErrors(underlyingReporter))
-      handleErrors(underlyingReporter, log)
+    if (!noErrors(underlyingReporter)) {
+      val infos = callback3.getSourceInfos
+      handleErrors(infos, log)
+    }
 
     // the case where we cancelled compilation _after_ some compilation errors got reported
     // will be handled by line above so errors still will be reported properly just potentially not
@@ -195,6 +209,11 @@ private final class CachedCompiler0(
     throw new InterfaceCompileFailed(args, dreporter.problems, "Compilation failed")
   }
 
+  def handleErrors(sourceInfos: ReadSourceInfos, log: Logger): Nothing = {
+    debug(log, "Compilation failed (CompilerInterface)")
+    throw new InterfaceCompileFailed2(args, sourceInfos, "Compilation failed")
+  }
+
   def handleCompilationCancellation(dreporter: DelegatingReporter, log: Logger): Nothing = {
     assert(dreporter.cancelled, "We should get here only if when compilation got cancelled")
     debug(log, "Compilation cancelled (CompilerInterface)")

internal/compiler-interface/src/main/java/xsbti/AnalysisCallback3.java

@@ -0,0 +1,21 @@
+/*
+ * Zinc - The incremental compiler for Scala.
+ * Copyright Scala Center, Lightbend, and Mark Harrah
+ *
+ * Licensed under Apache License 2.0
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package xsbti;
+import xsbti.compile.analysis.ReadSourceInfos;
+
+/**
+ * Extension to {@link AnalysisCallback2}.
+ * Similar to {@link AnalysisCallback2}, it serves as compatibility layer for Scala compilers.
+ */
+public interface AnalysisCallback3 extends AnalysisCallback2 {
+    ReadSourceInfos getSourceInfos();
+}

internal/compiler-interface/src/main/java/xsbti/CompileFailed2.java

@@ -0,0 +1,22 @@
+/*
+ * Zinc - The incremental compiler for Scala.
+ * Copyright Scala Center, Lightbend, and Mark Harrah
+ *
+ * Licensed under Apache License 2.0
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package xsbti;
+
+import xsbti.compile.analysis.ReadSourceInfos;
+
+public abstract class CompileFailed2 extends CompileFailed {
+    /**
+     * Returns SourceInfos containing problems for each file.
+     * This includes problems found by most recent compilation run.
+     */
+    public abstract ReadSourceInfos sourceInfos();
+}

internal/zinc-benchmarks/src/test/scala/xsbt/AnalysisFormatBenchmark.scala

@@ -35,7 +35,7 @@ class AnalysisFormatBenchmark {
       IO.copyFile(f, f2)
       assert(f2.exists())
     }
-    this.cached = readAll("", FileAnalysisStore.binary(_))
+    this.cached = readAll("", FileAnalysisStore.text(_))
     writeAll("-ref-text", FileAnalysisStore.text(_), cached)
     // writeAll("-ref-ctext", ConsistentFileAnalysisStore.text(_, ReadWriteMappers.getEmptyMappers), cached)
     writeAll(
@@ -62,26 +62,12 @@ class AnalysisFormatBenchmark {
     if (temp != null) IO.delete(temp)
   }
 
-  @Benchmark
-  def readBinary(bh: Blackhole): Unit = bh.consume(readAll("", FileAnalysisStore.binary(_)))
-
-  @Benchmark
-  def readText(bh: Blackhole): Unit = bh.consume(readAll("-ref-text", FileAnalysisStore.text(_)))
-
   @Benchmark
   def readConsistentBinary(bh: Blackhole): Unit =
     bh.consume(
       readAll("-ref-cbin", ConsistentFileAnalysisStore.binary(_, ReadWriteMappers.getEmptyMappers))
     )
 
-  @Benchmark
-  def writeBinary(bh: Blackhole): Unit =
-    bh.consume(writeAll("-test-bin", FileAnalysisStore.binary(_), cached))
-
-  @Benchmark
-  def writeText(bh: Blackhole): Unit =
-    bh.consume(writeAll("-test-text", FileAnalysisStore.text(_), cached))
-
   @Benchmark
   def writeConsistentBinary(bh: Blackhole): Unit =
     bh.consume(

internal/zinc-compile-core/src/main/scala/sbt/internal/inc/RawCompiler.scala

@@ -15,6 +15,7 @@ package inc
 
 import java.nio.file.Path
 import sbt.internal.util.FeedbackProvidedException
+import xsbti.compile.analysis.ReadSourceInfos
 import xsbti.compile.{ ClasspathOptions, ScalaInstance => XScalaInstance }
 
 /**
@@ -98,11 +99,12 @@ class CompileFailed(
     val arguments: Array[String],
     override val toString: String,
     val problems: Array[xsbti.Problem],
+    val sourceInfosOption: Option[ReadSourceInfos],
     cause: Throwable
 ) extends xsbti.CompileFailed(cause)
     with FeedbackProvidedException {
 
   def this(arguments: Array[String], toString: String, problems: Array[xsbti.Problem]) = {
-    this(arguments, toString, problems, null)
+    this(arguments, toString, problems, None, null)
   }
 }

internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala

@@ -590,7 +590,7 @@ private final class AnalysisCallback(
     progress: Option[CompileProgress],
     incHandlerOpt: Option[Incremental.IncrementalCallback],
     log: Logger
-) extends xsbti.AnalysisCallback2 {
+) extends xsbti.AnalysisCallback3 {
   import Incremental.CompileCycleResult
 
   // This must have a unique value per AnalysisCallback
@@ -886,7 +886,7 @@ private final class AnalysisCallback(
                 val externalParentsAPI = externalParents.map(analysis.apis.externalAPI)
                 val internalParentsAPI = internalParents.map(analysis.apis.internalAPI)
                 val parentsHashes = (externalParentsAPI ++ internalParentsAPI).map(_.extraHash())
-                parentsHashes.fold(currentExtraHash)(_ ^ _)
+                (parentsHashes + currentExtraHash).hashCode()
               case None => currentExtraHash
             }
           }
@@ -1067,6 +1067,25 @@ private final class AnalysisCallback(
     }
   }
 
+  def getSourceInfos: SourceInfos = {
+    // Collect Source Info from current run
+    val sources = reporteds.keySet ++ unreporteds.keySet ++ mainClasses.keySet
+    val sourceToInfo = sources.map { source =>
+      val info = SourceInfos.makeInfo(
+        getOrNil(reporteds.iterator.map { case (k, v) => k -> v.asScala.toSeq }.toMap, source),
+        getOrNil(unreporteds.iterator.map { case (k, v) => k -> v.asScala.toSeq }.toMap, source),
+        getOrNil(mainClasses.iterator.map { case (k, v) => k -> v.asScala.toSeq }.toMap, source)
+      )
+      (source, info)
+    }.toMap
+    val sourceInfoFromCurrentRun = SourceInfos.of(sourceToInfo)
+    // Collect reported problems from previous run
+    incHandlerOpt.map(_.previousAnalysisPruned) match {
+      case Some(prevAnalysis) => prevAnalysis.infos ++ sourceInfoFromCurrentRun
+      case None               => sourceInfoFromCurrentRun
+    }
+  }
+
   override def apiPhaseCompleted(): Unit = {
     // If we know we're done with cycles (presumably because all sources were invalidated) we can store early analysis
     // and picke data now.  Otherwise, we need to wait for dependency information to decide if there are more cycles.

internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala

@@ -472,16 +472,10 @@ private[inc] abstract class IncrementalCommon(
     val dependsOnClass = findClassDependencies(_, relations)
     val firstClassInvalidation: Set[String] = {
       val invalidated =
-        if (invalidateTransitively) {
-          // Invalidate by brute force (normally happens when we've done more than 3 incremental runs)
-          IncrementalCommon.transitiveDeps(initial, log)(dependsOnClass)
-        } else {
-          changes.apiChanges.flatMap(invalidateClassesInternally(relations, _, isScalaClass)).toSet
-        }
-      val included = includeTransitiveInitialInvalidations(initial, invalidated, dependsOnClass)
-      log.debug("Final step, transitive dependencies:\n\t" + included)
-      included
+        changes.apiChanges.flatMap(invalidateClassesInternally(relations, _, isScalaClass)).toSet
+      includeTransitiveInitialInvalidations(initial, invalidated, dependsOnClass)
     }
+    log.debug("Final step, transitive dependencies:\n\t" + firstClassInvalidation)
 
     // Invalidate classes linked with a class file that is produced by more than one source file
     val secondClassInvalidation = IncrementalCommon.invalidateNamesProducingSameClassFile(relations)
@@ -506,6 +500,7 @@ private[inc] abstract class IncrementalCommon(
       val transitive = IncrementalCommon.transitiveDeps(recompiledClasses, log)(dependsOnClass)
       (transitive -- recompiledClasses).filter(analysis.apis.internalAPI(_).hasMacro)
     }
+    log.debug(s"Invalidated macros due to upstream dependencies change: ${thirdClassInvalidation}")
 
     val newInvalidations =
       (firstClassInvalidation -- recompiledClasses) ++ secondClassInvalidation ++ thirdClassInvalidation
@@ -514,7 +509,13 @@ private[inc] abstract class IncrementalCommon(
       Set.empty
     } else {
       if (invalidateTransitively) {
-        newInvalidations ++ recompiledClasses
+        val firstClassTransitiveInvalidation = includeTransitiveInitialInvalidations(
+          initial,
+          IncrementalCommon.transitiveDeps(initial, log)(dependsOnClass),
+          dependsOnClass
+        )
+        log.debug("Invalidate by brute force:\n\t" + firstClassTransitiveInvalidation)
+        firstClassTransitiveInvalidation ++ secondClassInvalidation ++ thirdClassInvalidation ++ recompiledClasses
       } else {
         firstClassInvalidation ++ secondClassInvalidation ++ thirdClassInvalidation
       }

internal/zinc-testing/src/main/scala/xsbti/TestCallback.scala

@@ -15,12 +15,12 @@ import java.io.File
 import java.nio.file.Path
 import java.{ util => ju }
 import ju.Optional
-
-import xsbti.api.{ DependencyContext, ClassLike }
+import xsbti.api.{ ClassLike, DependencyContext }
+import xsbti.compile.analysis.ReadSourceInfos
 
 import scala.collection.mutable.ArrayBuffer
 
-class TestCallback extends AnalysisCallback2 {
+class TestCallback extends AnalysisCallback3 {
   case class TestUsedName(name: String, scopes: ju.EnumSet[UseScope])
 
   val classDependencies = new ArrayBuffer[(String, String, DependencyContext)]
@@ -153,6 +153,8 @@ class TestCallback extends AnalysisCallback2 {
   override def isPickleJava: Boolean = false
 
   override def getPickleJarPair: Optional[T2[Path, Path]] = Optional.empty()
+
+  override def getSourceInfos: ReadSourceInfos = new TestSourceInfos
 }
 
 object TestCallback {