-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Reorganised the code for generating indexes.
- Loading branch information
Showing
6 changed files
with
188 additions
and
135 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
76 changes: 76 additions & 0 deletions
76
src/main/scala/com/virtuslab/shared_indexes/core/WorkPlan.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package com.virtuslab.shared_indexes.core | ||
|
|
||
| import com.intellij.indexing.shared.cdn.S3_uploadKt | ||
| import com.intellij.indexing.shared.cdn.{CdnUpdatePlan, S3, S3Kt} | ||
| import com.intellij.indexing.shared.local.Local_uploadKt | ||
| import com.virtuslab.shared_indexes.config.MainConfig | ||
|
|
||
| case class WorkPlan( | ||
| intelliJ: IntelliJ, | ||
| workspace: Workspace, | ||
| projectRoot: Option[os.Path], | ||
| artifactPaths: Seq[os.Path], | ||
| commit: Option[String], | ||
| indexBaseUrl: Option[String] = None, | ||
| indexExportingStrategy: Option[CdnUpdatePlan => Unit] = None, | ||
| s3: Option[S3] = None | ||
| ) | ||
|
|
||
| object WorkPlan { | ||
| def apply(config: MainConfig): WorkPlan = { | ||
| withServerUploadPlan(createBaseWorkPlan(config), config) | ||
| } | ||
|
|
||
| private def createBaseWorkPlan(config: MainConfig): WorkPlan = { | ||
| import config._ | ||
| val intelliJ = { | ||
| import generatorConfig._ | ||
| new IntelliJ(ideaBinary, ideaCacheDir) | ||
| } | ||
| val workspace = new Workspace(generatorConfig.workDir) | ||
| WorkPlan(intelliJ, workspace, indexInputConfig.projectRoot, indexInputConfig.artifactPaths, indexInputConfig.commit) | ||
| } | ||
|
|
||
| private def withServerUploadPlan(basePlan: WorkPlan, config: MainConfig): WorkPlan = { | ||
| import basePlan.workspace | ||
| import config._ | ||
| val indexBaseUrl = indexStorageConfig.indexServerUrl match { | ||
| case Some(url) => | ||
| s3Config.bucketName match { | ||
| case Some(bucket) => url.replaceAll("/$", "") + "/" + bucket | ||
| case None => url | ||
| } | ||
| // If the server URL is not provided, use the workspace path instead. | ||
| // For example, if the workspace is located at /var/foo/bar/baz, | ||
| // the IntelliJ registry should look like this: | ||
| // shared.indexes.jdk.download.url=file:///var/foo/bar/baz/jdk | ||
| case None => "file://" + workspace.cdnPath | ||
| } | ||
|
|
||
| val (s3Opt, indexExportingStrategy): (Option[S3], CdnUpdatePlan => Unit) = { | ||
| s3Config.bucketName match { | ||
| case Some(bucketName) => | ||
| // To upload to S3, you need to provide access credentials. | ||
| // The easiest way is setting up these environment variables: | ||
| // AWS_ACCESS_KEY_ID=xxxxx123456789xxxxx | ||
| // AWS_SECRET_ACCESS_KEY=xxxxx123456789xxxxx | ||
| // (replace the values with valid credentials to your S3 service) | ||
| // For other ways to supply credentials, see | ||
| // com.amazonaws.auth.DefaultAWSCredentialsProviderChain | ||
|
|
||
| // You must make sure the S3 API is accessible and the bucket exists | ||
| val s3ApiUrl = indexStorageConfig.indexServerUrl | ||
| .getOrElse(throw new IllegalStateException("Must provide the server address for uploading to S3")) | ||
| val s3 = S3Kt.S3(s"$s3ApiUrl/$bucketName", bucketName, s3ApiUrl, "/", 10) | ||
| (Some(s3), S3_uploadKt.updateS3Indexes(s3, _)) | ||
| case None => | ||
| // updateLocalIndexes executes the update plan in given basePath, which is the location that | ||
| // the server should host | ||
| val basePath = workspace.cdnPath.toNIO | ||
| (None, Local_uploadKt.updateLocalIndexes(basePath, _)) | ||
| } | ||
| } | ||
|
|
||
| basePlan.copy(indexBaseUrl = Some(indexBaseUrl), indexExportingStrategy = Some(indexExportingStrategy), s3 = s3Opt) | ||
| } | ||
| } |
21 changes: 21 additions & 0 deletions
21
src/main/scala/com/virtuslab/shared_indexes/generator/JarIndexesGenerator.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| package com.virtuslab.shared_indexes.generator | ||
|
|
||
| import com.virtuslab.shared_indexes.core.{IntelliJ, JdkAliases, SharedIndexes, Workspace} | ||
| import com.virtuslab.shared_indexes.locator.JdkLocator | ||
| import os.Path | ||
|
|
||
| class JarIndexesGenerator(intelliJ: IntelliJ, workspace: Workspace, val jarPaths: Seq[os.Path]) | ||
| extends SharedIndexesGenerator[Seq[os.Path]](intelliJ, workspace) { | ||
|
|
||
| override protected val cacheDir: Path = workspace.jarIndexes | ||
|
|
||
| override protected def findInputs(): Seq[os.Path] = { | ||
| logger.info(s"Found ${jarPaths.size} JDKs:") | ||
| jarPaths.foreach { p => logger.info(p.toString()) } | ||
| jarPaths | ||
| } | ||
|
|
||
| override protected def process(inputs: Seq[os.Path]): Unit = { | ||
| SharedIndexes.dumpJarSharedIndex(intelliJ, inputs, workspace, "jars", SharedIndexes.key(jarPaths)) | ||
| } | ||
| } |
28 changes: 28 additions & 0 deletions
28
src/main/scala/com/virtuslab/shared_indexes/generator/JdkIndexesGenerator.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package com.virtuslab.shared_indexes.generator | ||
| import com.virtuslab.shared_indexes.core.{IntelliJ, JdkAliases, SharedIndexes, Workspace} | ||
| import com.virtuslab.shared_indexes.locator.JdkLocator | ||
| import os.Path | ||
|
|
||
| class JdkIndexesGenerator(intelliJ: IntelliJ, workspace: Workspace, val customJdkPaths: Seq[os.Path]) | ||
| extends SharedIndexesGenerator[Seq[os.Path]](intelliJ, workspace) { | ||
|
|
||
| override protected val cacheDir: Path = workspace.jdkIndexes | ||
|
|
||
| override protected def findInputs(): Seq[os.Path] = { | ||
| val jdkPaths = if (customJdkPaths.isEmpty) JdkLocator.findAllInstalledJdks() else customJdkPaths | ||
| logger.info(s"Found ${jdkPaths.size} JDKs:") | ||
| jdkPaths.foreach { p => logger.info(p.toString()) } | ||
| jdkPaths | ||
| } | ||
|
|
||
| override protected def process(inputs: Seq[os.Path]): Unit = { | ||
| // I generate indexes one by one. Otherwise, IntelliJ merges | ||
| // them into one big index which doesn't seem to be picked | ||
| // up by the existing logic. | ||
| for (jdkPath <- inputs) { | ||
| val aliases = JdkAliases.resolve(jdkPath) | ||
| logger.info(s"Generating shared indexes for JDK $jdkPath with aliases $aliases") | ||
| SharedIndexes.dumpJdkSharedIndexes(intelliJ, jdkPath, aliases, workspace) | ||
| } | ||
| } | ||
| } |
28 changes: 28 additions & 0 deletions
28
src/main/scala/com/virtuslab/shared_indexes/generator/ProjectIndexesGenerator.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package com.virtuslab.shared_indexes.generator | ||
|
|
||
| import com.virtuslab.shared_indexes.core.{IntelliJ, SharedIndexes, Workspace} | ||
| import com.virtuslab.shared_indexes.locator.ProjectLocator | ||
| import os.Path | ||
|
|
||
| class ProjectIndexesGenerator( | ||
| intelliJ: IntelliJ, | ||
| workspace: Workspace, | ||
| projectRootOpt: Option[os.Path], | ||
| commitOpt: Option[String] | ||
| ) extends SharedIndexesGenerator[(Path, String)](intelliJ, workspace) { | ||
|
|
||
| override protected val cacheDir: Path = workspace.projectIndexes | ||
|
|
||
| override protected def findInputs(): (Path, String) = { | ||
| val projectRoot = projectRootOpt.getOrElse(ProjectLocator.exampleProjectHome) | ||
| val commit = commitOpt.getOrElse(ProjectLocator.getGitCommit(projectRoot)) | ||
|
|
||
| logger.info(s"Found project at $projectRoot with HEAD $commit") | ||
| (projectRoot, commit) | ||
| } | ||
|
|
||
| override protected def process(inputs: (Path, String)): Unit = { | ||
| val (projectRoot, commit) = inputs | ||
| SharedIndexes.dumpProjectSharedIndex(intelliJ, projectRoot, commit, workspace) | ||
| } | ||
| } |
22 changes: 22 additions & 0 deletions
22
src/main/scala/com/virtuslab/shared_indexes/generator/SharedIndexesGenerator.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package com.virtuslab.shared_indexes.generator | ||
|
|
||
| import com.virtuslab.shared_indexes.core.{IntelliJ, Workspace} | ||
| import org.slf4j.{Logger, LoggerFactory} | ||
|
|
||
| abstract class SharedIndexesGenerator[I](protected val intelliJ: IntelliJ, protected val workspace: Workspace) { | ||
| protected val logger: Logger = LoggerFactory.getLogger(this.getClass) | ||
|
|
||
| protected val cacheDir: os.Path | ||
|
|
||
| protected def findInputs(): I | ||
| protected def process(inputs: I): Unit | ||
| def generateIndexes(): os.Path = { | ||
| if (os.list(cacheDir).isEmpty) { | ||
| process(findInputs()) | ||
| } else { | ||
| // FIXME verify that the cached indexes actually correspond to our inputs | ||
| logger.info("Indexes already exist, skipping generation") | ||
| } | ||
| cacheDir | ||
| } | ||
| } |