Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve SBT caching during code generation #1499

Merged
merged 13 commits into from
May 18, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import smithy4s.codegen.CodegenArgs
import smithy4s.codegen.FileType

import Options._
import smithy4s.codegen.PathRef

object CodegenCommand {

Expand Down Expand Up @@ -116,7 +117,7 @@ object CodegenCommand {
defaultDependencies ++ dependencies.getOrElse(List.empty)
}
CodegenArgs(
specsArgs,
specsArgs.map(PathRef(_)),
kubukoz marked this conversation as resolved.
Show resolved Hide resolved
output.getOrElse(os.pwd),
resourseOutput.getOrElse(os.pwd),
skip,
Expand All @@ -126,7 +127,7 @@ object CodegenCommand {
repositories.getOrElse(List.empty),
dependenciesWithDefaults,
transformers.getOrElse(List.empty),
localJars.getOrElse(List.empty)
localJars.getOrElse(List.empty).map(PathRef(_))
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import smithy4s.codegen.FileType
import weaver._

import Defaults.defaultDependencies
import smithy4s.codegen.PathRef

object CommandParsingSpec extends FunSuite {

Expand Down Expand Up @@ -81,7 +82,7 @@ object CommandParsingSpec extends FunSuite {
specs = List(
os.pwd / "sampleSpecs" / "pizza.smithy",
os.pwd / "sampleSpecs" / "example.smithy"
),
).map(PathRef(_)),
output = os.pwd / "target",
resourceOutput = os.pwd / "target" / "openapi",
skip = Set(FileType.Openapi, FileType.Scala),
Expand All @@ -94,7 +95,7 @@ object CommandParsingSpec extends FunSuite {
localJars = List(
os.pwd / "lib1.jar",
os.pwd / "lib2.jar"
)
).map(PathRef(_))
)
)
)
Expand Down
24 changes: 17 additions & 7 deletions modules/codegen-plugin/src/smithy4s/codegen/JsonConverters.scala
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,30 @@ private[smithy4s] object JsonConverters {
// This serialises a path by providing a hash of the content it points to.
// Because the hash is part of the Json, this allows SBT to detect when a file
// changes and invalidate its relevant caches, leading to a call to Smithy4s' code generator.
implicit val pathFormat: JsonFormat[os.Path] =
BasicJsonProtocol.projectFormat[os.Path, HashFileInfo](
implicit val pathRefFormat: JsonFormat[PathRef] =
BasicJsonProtocol.projectFormat[PathRef, HashFileInfo](
p => {
if (os.isFile(p)) FileInfo.hash(p.toIO)
if (os.isFile(p.underlying)) FileInfo.hash(p.underlying.toIO)
else
// If the path is a directory, we get the hashes of all files
// then hash the concatenation of the hash's bytes.
FileInfo.hash(
p.toIO,
p.underlying.toIO,
Hash(
os.walk(p)
os.walk(p.underlying)
.map(_.toIO)
.map(Hash(_))
.foldLeft(Array.emptyByteArray)(_ ++ _)
)
)
},
hash => os.Path(hash.file)
hash => PathRef(os.Path(hash.file))
)

implicit val pathFormat: JsonFormat[os.Path] =
BasicJsonProtocol.projectFormat[os.Path, String](
p => p.toString,
str => os.Path(str)
)

implicit val fileTypeFormat: JsonFormat[FileType] =
Expand All @@ -61,8 +67,12 @@ private[smithy4s] object JsonConverters {
)

// format: off
type GenTarget = List[os.Path] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[os.Path] :*: LNil
type GenTarget = List[PathRef] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[PathRef] :*: LNil
// format: on

// `output` and `resourceOutput` are intentionally serialized as strings
// instead paths. This is to avoid hashing the directories that are generated by smithy4s anyway
// See https://github.com/disneystreaming/smithy4s/issues/1495 for reference on this decision
implicit val codegenArgsIso = LList.iso[CodegenArgs, GenTarget](
{ ca: CodegenArgs =>
("specs", ca.specs) :*:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -403,28 +403,30 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
(inputDirs ++ generatedFiles)
.filter(_.exists())
.toList
kubukoz marked this conversation as resolved.
Show resolved Hide resolved
val outputPath = (conf / smithy4sOutputDir).value
val resourceOutputPath = (conf / smithy4sResourceDir).value
val outputPath = (conf / smithy4sOutputDir).value / "smithy4s"
val resourceOutputPath = (conf / smithy4sResourceDir).value / "smithy4s"
kubukoz marked this conversation as resolved.
Show resolved Hide resolved
val allowedNamespaces =
(conf / smithy4sAllowedNamespaces).?.value.map(_.toSet)
val excludedNamespaces =
(conf / smithy4sExcludedNamespaces).?.value.map(_.toSet)
val localJars =
(conf / smithy4sAllDependenciesAsJars).value.map(os.Path(_)).toList
(conf / smithy4sAllDependenciesAsJars).value.toList.sorted
.map(p => PathRef(os.Path(p)))
val res =
(conf / resolvers).value.toList.collect { case m: MavenRepository =>
m.root
}
val transforms = (conf / smithy4sModelTransformers).value
val transforms = (conf / smithy4sModelTransformers).value.sorted
kubukoz marked this conversation as resolved.
Show resolved Hide resolved
val s = (conf / streams).value
val skipResources: Set[FileType] =
if ((conf / smithy4sSmithyLibrary).value) Set.empty
else Set(FileType.Resource)
val skipSet = skipResources

val filePaths = inputFiles.map(_.getAbsolutePath())
val specs = filePaths.sorted.map(p => PathRef(os.Path(p))).toList
val codegenArgs = CodegenArgs(
filePaths.map(os.Path(_)).toList,
specs,
output = os.Path(outputPath),
resourceOutput = os.Path(resourceOutputPath),
skip = skipSet,
Expand All @@ -443,14 +445,16 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
) {
Function.untupled {
Tracked.lastOutput[(Boolean, CodegenArgs), Seq[File]](
s.cacheStoreFactory.make("output")
s.cacheDirectory / "smithy4s-output"
) { case ((inputChanged, args), outputs) =>
if (inputChanged || outputs.isEmpty) {
s.log.debug("Regenerating managed sources")
kubukoz marked this conversation as resolved.
Show resolved Hide resolved
val resPaths = smithy4s.codegen.Codegen
.generateToDisk(args)
.toList
resPaths.map(path => new File(path.toString))
} else {
s.log.debug("Using cached version of outputs")
outputs.getOrElse(Seq.empty)
}
}
Expand Down
5 changes: 3 additions & 2 deletions modules/codegen/src/smithy4s/codegen/CodegenArgs.scala
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import cats.data.ValidatedNel
import cats.syntax.all._

final case class CodegenArgs(
specs: List[os.Path],
specs: List[PathRef],
output: os.Path,
resourceOutput: os.Path,
skip: Set[FileType],
Expand All @@ -30,12 +30,13 @@ final case class CodegenArgs(
repositories: List[String],
dependencies: List[String],
transformers: List[String],
localJars: List[os.Path]
localJars: List[PathRef]
) {
def skipScala: Boolean = skip(FileType.Scala)
def skipOpenapi: Boolean = skip(FileType.Openapi)
def skipResources: Boolean = skip(FileType.Resource)
def skipProto: Boolean = skip(FileType.Proto)

}

sealed abstract class FileType(val name: String)
Expand Down
3 changes: 3 additions & 0 deletions modules/codegen/src/smithy4s/codegen/PathRef.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package smithy4s.codegen

final case class PathRef(underlying: os.Path)
kubukoz marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@ private[codegen] object CodegenImpl { self =>

def generate(args: CodegenArgs): CodegenResult = {
val (classloader, model): (ClassLoader, Model) = internals.ModelLoader.load(
args.specs.map(_.toIO).toSet,
args.specs.map(_.underlying.toIO).toSet,
args.dependencies,
args.repositories,
withBuiltinTransformers(args.transformers),
args.discoverModels,
args.localJars
args.localJars.map(_.underlying)
)

val (scalaFiles, smithyResources) = if (!args.skipScala) {
Expand All @@ -56,7 +56,7 @@ private[codegen] object CodegenImpl { self =>
val resources = if (!skipResource) {
SmithyResources.produce(
args.resourceOutput,
args.specs,
args.specs.map(_.underlying),
generatedNamespaces
)
} else List.empty[CodegenEntry]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import mill.api.JarManifest
import mill.scalalib.CrossVersion.Binary
import mill.scalalib.CrossVersion.Constant
import mill.scalalib.CrossVersion.Full
import smithy4s.codegen

trait Smithy4sModule extends ScalaModule {

Expand Down Expand Up @@ -190,6 +191,8 @@ trait Smithy4sModule extends ScalaModule {
val specFiles = (smithy4sGeneratedSmithyFiles() ++ smithy4sInputDirs())
.map(_.path)
.filter(os.exists(_))
.toList
.map(codegen.PathRef(_))

val scalaOutput = smithy4sOutputDir().path
val resourcesOutput = smithy4sResourceOutputDir().path
Expand All @@ -205,10 +208,14 @@ trait Smithy4sModule extends ScalaModule {
val skipSet = skipResources ++ skipOpenApi

val allLocalJars =
smithy4sAllDependenciesAsJars().map(_.path).iterator.to(List)
smithy4sAllDependenciesAsJars()
.map(_.path)
.iterator
.to(List)
.map(codegen.PathRef(_))

val args = CodegenArgs(
specs = specFiles.toList,
specs = specFiles,
output = scalaOutput,
resourceOutput = resourcesOutput,
skip = skipSet,
Expand Down
Loading