diff --git a/app/uk/gov/hmrc/teamsandrepositories/config/GithubConfig.scala b/app/uk/gov/hmrc/teamsandrepositories/config/GithubConfig.scala index 28d0c188..0ea57f47 100644 --- a/app/uk/gov/hmrc/teamsandrepositories/config/GithubConfig.scala +++ b/app/uk/gov/hmrc/teamsandrepositories/config/GithubConfig.scala @@ -29,14 +29,6 @@ class GithubConfig @Inject()(configuration: Configuration) { val apiUrl = configuration.get[String]("github.open.api.url") val rawUrl = configuration.get[String]("github.open.api.rawurl") - val hiddenRepositories: Set[String] = - configuration.getOptional[String]("github.hidden.repositories") - .fold(Set.empty[String])(_.split(",").toSet) - - val hiddenTeams: Set[String] = - configuration.getOptional[String]("github.hidden.teams") - .fold(Set.empty[String])(_.split(",").toSet) - val tokens: List[(String, String)] = configuration.get[ConfigList]("ratemetrics.githubtokens").asScala.toList .map(cv => new Configuration(cv.asInstanceOf[ConfigObject].toConfig)) diff --git a/app/uk/gov/hmrc/teamsandrepositories/controller/v2/TeamsAndRepositoriesController.scala b/app/uk/gov/hmrc/teamsandrepositories/controller/v2/TeamsAndRepositoriesController.scala index 020090fc..3fc5e46e 100644 --- a/app/uk/gov/hmrc/teamsandrepositories/controller/v2/TeamsAndRepositoriesController.scala +++ b/app/uk/gov/hmrc/teamsandrepositories/controller/v2/TeamsAndRepositoriesController.scala @@ -21,7 +21,7 @@ import play.api.mvc.{Action, ControllerComponents} import uk.gov.hmrc.internalauth.client.{BackendAuthComponents, IAAction, Predicate, Resource} import uk.gov.hmrc.play.bootstrap.backend.controller.BackendController import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, ServiceType, Tag, TeamSummary} -import uk.gov.hmrc.teamsandrepositories.persistence.RepositoriesPersistence +import uk.gov.hmrc.teamsandrepositories.persistence.{RepositoriesPersistence, TeamSummaryPersistence} import uk.gov.hmrc.teamsandrepositories.services.BranchProtectionService import javax.inject.{Inject, Singleton} @@ -30,6 +30,7 @@ import scala.concurrent.{ExecutionContext, Future} @Singleton class TeamsAndRepositoriesController @Inject()( repositoriesPersistence: RepositoriesPersistence, + teamSummaryPersistence : TeamSummaryPersistence, branchProtectionService: BranchProtectionService, auth : BackendAuthComponents, cc : ControllerComponents @@ -40,20 +41,22 @@ class TeamsAndRepositoriesController @Inject()( implicit val grf: Format[GitRepository] = GitRepository.apiFormat implicit val tnf: Format[TeamSummary] = TeamSummary.apiFormat - def allRepos( + def repositories( name : Option[String], team : Option[String], + owningTeam : Option[String], archived : Option[Boolean], repoType : Option[RepoType], serviceType: Option[ServiceType], tags : Option[List[Tag]], ) = Action.async { request => - repositoriesPersistence.search(name, team, archived, repoType, serviceType, tags) + repositoriesPersistence.find(name, team, owningTeam, archived, repoType, serviceType, tags) .map(result => Ok(Json.toJson(result.sortBy(_.name)))) } def allTeams() = Action.async { request => - repositoriesPersistence.findTeamSummaries().map(result => Ok(Json.toJson(result))) + teamSummaryPersistence.findTeamSummaries() + .map(result => Ok(Json.toJson(result))) } def findRepo(repoName:String) = Action.async { request => diff --git a/app/uk/gov/hmrc/teamsandrepositories/models/GitRepository.scala b/app/uk/gov/hmrc/teamsandrepositories/models/GitRepository.scala index f93eff89..6469d62b 100644 --- a/app/uk/gov/hmrc/teamsandrepositories/models/GitRepository.scala +++ b/app/uk/gov/hmrc/teamsandrepositories/models/GitRepository.scala @@ -73,10 +73,6 @@ object GitRepository { ~ (__ \ "prototypeAutoPublish").formatNullable[Boolean] ~ (__ \ "repositoryYamlText" ).formatNullable[String] )(apply, unlift(unapply)) - .bimap( - identity, - repo => if (repo.owningTeams.isEmpty) repo.copy(owningTeams = repo.teams) else repo - ) } val mongoFormat: OFormat[GitRepository] = { diff --git a/app/uk/gov/hmrc/teamsandrepositories/models/TeamName.scala b/app/uk/gov/hmrc/teamsandrepositories/models/TeamName.scala deleted file mode 100644 index 0210b500..00000000 --- a/app/uk/gov/hmrc/teamsandrepositories/models/TeamName.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2023 HM Revenue & Customs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package uk.gov.hmrc.teamsandrepositories.models - -import play.api.libs.json.{OFormat, __} -import play.api.libs.functional.syntax._ -import uk.gov.hmrc.mongo.play.json.formats.MongoJavatimeFormats - -import java.time.Instant - -case class TeamSummary( - name : String, - createdDate : Instant, - lastActiveDate: Instant, - repos : Int -) - -object TeamSummary { - - val apiFormat: OFormat[TeamSummary] = { - ( (__ \ "name" ).format[String] - ~ (__ \ "createdDate" ).format[Instant] - ~ (__ \ "lastActiveDate").format[Instant] - ~ (__ \ "repos" ).format[Int] - )(TeamSummary.apply, unlift(TeamSummary.unapply)) - } - - val mongoFormat: OFormat[TeamSummary] = { - implicit val inf = MongoJavatimeFormats.instantFormat - ( (__ \ "_id" ).format[String] - ~ (__ \ "createdDate" ).format[Instant] - ~ (__ \ "lastActiveDate").format[Instant] - ~ (__ \ "repos" ).format[Int] - )(TeamSummary.apply, unlift(TeamSummary.unapply)) - } -} diff --git a/app/uk/gov/hmrc/teamsandrepositories/models/TeamSummary.scala b/app/uk/gov/hmrc/teamsandrepositories/models/TeamSummary.scala new file mode 100644 index 00000000..1bed4b77 --- /dev/null +++ b/app/uk/gov/hmrc/teamsandrepositories/models/TeamSummary.scala @@ -0,0 +1,59 @@ +/* + * Copyright 2023 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.teamsandrepositories.models + +import play.api.libs.json.{OFormat, __} +import play.api.libs.functional.syntax._ +import uk.gov.hmrc.mongo.play.json.formats.MongoJavatimeFormats + +import java.time.Instant + +case class TeamSummary( + name : String, + lastActiveDate: Option[Instant], + repos : Seq[String] +) + +object TeamSummary { + def createTeamSummaries(gitRepositories: List[GitRepository]): Seq[TeamSummary] = + gitRepositories + .flatMap(repo => repo.teams.map(team => (team, repo))) + .groupBy { case (team, _) => team } + .view.mapValues(_.map{ case (_, repo) => repo }) + .map { + case (team, gitRepos) => + TeamSummary( + name = team, + lastActiveDate = Some(gitRepos.map(_.lastActiveDate).max), + repos = gitRepos.collect { + case gitRepo if gitRepo.owningTeams.contains(team) && !gitRepo.isArchived => gitRepo.name + } + ) + }.toSeq + + val apiFormat: OFormat[TeamSummary] = + ( (__ \ "name" ).format[String] + ~ (__ \ "lastActiveDate").formatNullable[Instant] + ~ (__ \ "repos" ).format[Seq[String]] + )(TeamSummary.apply, unlift(TeamSummary.unapply)) + + val mongoFormat: OFormat[TeamSummary] = + ( (__ \ "name" ).format[String] + ~ (__ \ "lastActiveDate" ).formatNullable[Instant](MongoJavatimeFormats.instantFormat) + ~ (__ \ "repos" ).format[Seq[String]] + )(TeamSummary.apply, unlift(TeamSummary.unapply)) +} diff --git a/app/uk/gov/hmrc/teamsandrepositories/persistence/RepositoriesPersistence.scala b/app/uk/gov/hmrc/teamsandrepositories/persistence/RepositoriesPersistence.scala index 3c79156a..336b1c39 100644 --- a/app/uk/gov/hmrc/teamsandrepositories/persistence/RepositoriesPersistence.scala +++ b/app/uk/gov/hmrc/teamsandrepositories/persistence/RepositoriesPersistence.scala @@ -23,7 +23,7 @@ import org.mongodb.scala.model.Aggregates.{`match`, addFields, group, sort, unwi import org.mongodb.scala.model._ import uk.gov.hmrc.mongo.MongoComponent import uk.gov.hmrc.mongo.play.json.{Codecs, CollectionFactory, PlayMongoRepository} -import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, ServiceType, Tag, TeamRepositories, TeamSummary} +import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, ServiceType, Tag, TeamRepositories} import uk.gov.hmrc.teamsandrepositories.persistence.Collations.caseInsensitive import org.mongodb.scala.model.Accumulators.{addToSet, first, max, min} import org.mongodb.scala.model.Filters.equal @@ -56,12 +56,10 @@ class RepositoriesPersistence @Inject()( private val legacyCollection: MongoCollection[TeamRepositories] = CollectionFactory.collection(mongoComponent.database, collectionName, TeamRepositories.mongoFormat) - private val teamsCollection: MongoCollection[TeamSummary] = - CollectionFactory.collection(mongoComponent.database, collectionName, TeamSummary.mongoFormat) - - def search( + def find( name : Option[String] = None, team : Option[String] = None, + owningTeam : Option[String] = None, isArchived : Option[Boolean] = None, repoType : Option[RepoType] = None, serviceType: Option[ServiceType] = None, @@ -70,32 +68,22 @@ class RepositoriesPersistence @Inject()( val filters = Seq( name .map(n => Filters.regex("name" , n)), team .map(t => Filters.equal("teamNames" , t)), + owningTeam .map(t => Filters.equal("owningTeams", t)), isArchived .map(b => Filters.equal("isArchived" , b)), repoType .map(rt => Filters.equal("repoType" , rt.asString)), serviceType.map(st => Filters.equal("serviceType", st.asString)), - tags .map(ts => Filters.and(ts.map(t => Filters.equal("tags", t.asString)):_*)), + tags .map(ts => Filters.and(ts.map(t => Filters.equal("tags", t.asString)): _*)), ).flatten - filters match { - case Nil => collection.find().toFuture() - case more => collection.find(Filters.and(more:_*)).toFuture() - } + + collection + .find(if (filters.isEmpty) BsonDocument() else Filters.and(filters: _*)) + .toFuture() } def findRepo(repoName: String): Future[Option[GitRepository]] = collection .find(filter = Filters.equal("name", repoName)).headOption() - def findTeamSummaries(): Future[Seq[TeamSummary]] = - teamsCollection - .aggregate(Seq( - addFields(Field("teamSize", BsonDocument("$size" -> "$teamNames"))), - `match`(Filters.lt("teamSize", 8)), // ignore repos shared by more than n teams - unwind("$teamNames"), - group("$teamNames", Accumulators.min("createdDate", "$createdDate"), Accumulators.max("lastActiveDate", "$lastActiveDate"), Accumulators.sum("repos", 1)), - sort(Sorts.ascending("_id")) - )) - .toFuture() - def updateRepos(repos: Seq[GitRepository]): Future[Int] = for { oldRepos <- collection.find().map(_.name).toFuture().map(_.toSet) @@ -140,19 +128,21 @@ class RepositoriesPersistence @Inject()( // This exists to provide backward compatible data to the old API. Dont use it in new functionality! def getAllTeamsAndRepos(archived: Option[Boolean]): Future[Seq[TeamRepositories]] = - legacyCollection.aggregate(Seq( - `match`(archived.fold[Bson](BsonDocument())(a => Filters.eq("isArchived", a))), - unwind("$teamNames"), - addFields( Field("teamid", "$teamNames"), Field("teamNames", BsonArray())), - group( - id = "$teamid", - first("teamName", "$teamid"), - addToSet("repositories", "$$ROOT"), - min("createdDate", "$createdDate"), - max("updateDate", "$lastActiveDate") - ), - sort(Sorts.ascending("_id")) - )).toFuture() + legacyCollection + .aggregate(Seq( + `match`(archived.fold[Bson](BsonDocument())(a => Filters.eq("isArchived", a))), + unwind("$teamNames"), + addFields( Field("teamid", "$teamNames"), Field("teamNames", BsonArray())), + group( + id = "$teamid", + first("teamName", "$teamid"), + addToSet("repositories", "$$ROOT"), + min("createdDate", "$createdDate"), + max("updateDate", "$lastActiveDate") + ), + sort(Sorts.ascending("_id")) + )) + .toFuture() def updateRepoBranchProtection(repoName: String, branchProtection: Option[BranchProtection]): Future[Unit] = { implicit val bpf = BranchProtection.format diff --git a/app/uk/gov/hmrc/teamsandrepositories/persistence/TeamSummaryPersistence.scala b/app/uk/gov/hmrc/teamsandrepositories/persistence/TeamSummaryPersistence.scala new file mode 100644 index 00000000..11535956 --- /dev/null +++ b/app/uk/gov/hmrc/teamsandrepositories/persistence/TeamSummaryPersistence.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2024 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.teamsandrepositories.persistence + +import org.mongodb.scala.model.Indexes.ascending +import org.mongodb.scala.model.{DeleteOneModel, Filters, IndexModel, IndexOptions, Indexes, ReplaceOneModel, ReplaceOptions} +import play.api.Logger +import uk.gov.hmrc.mongo.MongoComponent +import uk.gov.hmrc.mongo.play.json.PlayMongoRepository +import uk.gov.hmrc.teamsandrepositories.models.TeamSummary +import uk.gov.hmrc.teamsandrepositories.persistence.Collations.caseInsensitive + +import javax.inject.{Inject, Singleton} +import scala.concurrent.{ExecutionContext, Future} + +@Singleton +class TeamSummaryPersistence @Inject()( + mongoComponent: MongoComponent +)(implicit + ec: ExecutionContext +) extends PlayMongoRepository( + mongoComponent = mongoComponent, + collectionName = "teamSummaries", + domainFormat = TeamSummary.mongoFormat, + indexes = Seq( + IndexModel(Indexes.ascending("name"), IndexOptions().name("nameIdx").collation(caseInsensitive).unique(true)), + ) +){ + + private val logger = Logger(this.getClass) + + override lazy val requiresTtlIndex = false + + def updateTeamSummaries(teams: Seq[TeamSummary]): Future[Int] = + for { + oldTeams <- collection.find().map(_.name).toFuture().map(_.toSet) + update <- collection + .bulkWrite(teams.map(teamSummary => ReplaceOneModel(Filters.eq("name", teamSummary.name), teamSummary, ReplaceOptions().collation(caseInsensitive).upsert(true)))) + .toFuture() + .map(_.getModifiedCount) + toDelete = (oldTeams -- teams.map(_.name)).toSeq + _ <- if (toDelete.nonEmpty) { + logger.info(s"about to remove ${toDelete.length} deleted teams: ${toDelete.mkString(", ")}") + collection.bulkWrite(toDelete.map(team => DeleteOneModel(Filters.eq("name", team)))).toFuture() + .map(res => logger.info(s"removed ${res.getModifiedCount} deleted teams")) + } else Future.unit + } yield update + + def findTeamSummaries(): Future[Seq[TeamSummary]] = + collection + .find() + .sort(ascending("name")) + .toFuture() +} diff --git a/app/uk/gov/hmrc/teamsandrepositories/schedulers/DataReloadScheduler.scala b/app/uk/gov/hmrc/teamsandrepositories/schedulers/DataReloadScheduler.scala index f5b82038..3171f979 100644 --- a/app/uk/gov/hmrc/teamsandrepositories/schedulers/DataReloadScheduler.scala +++ b/app/uk/gov/hmrc/teamsandrepositories/schedulers/DataReloadScheduler.scala @@ -56,7 +56,7 @@ class DataReloadScheduler @Inject()( scheduleWithLock("Teams and Repos Reloader", config.dataReloadScheduler, lockService) { for { - count <- persistingService.updateRepositories() + count <- persistingService.updateTeamsAndRepositories() _ = logger.info(s"Finished updating Teams and Repos - $count records updated") } yield () } @@ -65,7 +65,7 @@ class DataReloadScheduler @Inject()( lockService .withLock { logger.info(s"Starting mongo update") - persistingService.updateRepositories() + persistingService.updateTeamsAndRepositories() } .map(_.getOrElse(sys.error(s"Mongo is locked for ${lockService.lockId}"))) .map { _ => diff --git a/app/uk/gov/hmrc/teamsandrepositories/services/GithubV3RepositoryDataSource.scala b/app/uk/gov/hmrc/teamsandrepositories/services/GithubV3RepositoryDataSource.scala index 307fb9ce..b9d880d1 100644 --- a/app/uk/gov/hmrc/teamsandrepositories/services/GithubV3RepositoryDataSource.scala +++ b/app/uk/gov/hmrc/teamsandrepositories/services/GithubV3RepositoryDataSource.scala @@ -16,8 +16,7 @@ package uk.gov.hmrc.teamsandrepositories.services import com.google.inject.{Inject, Singleton} -import play.api.{Configuration, Logger} -import uk.gov.hmrc.teamsandrepositories.config.GithubConfig +import play.api.Logger import uk.gov.hmrc.teamsandrepositories.connectors.{GhTeam, GithubConnector} import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, TeamRepositories} @@ -31,23 +30,15 @@ class TimeStamper { } @Singleton class GithubV3RepositoryDataSource @Inject()( - githubConfig : GithubConfig, githubConnector: GithubConnector, - timeStamper : TimeStamper, - configuration : Configuration + timeStamper : TimeStamper ) { private val logger = Logger(this.getClass) - val sharedRepos: List[String] = - configuration.get[Seq[String]]("shared.repositories").toList - def getTeams()(implicit ec: ExecutionContext): Future[List[GhTeam]] = { - def notHidden(team: GhTeam) = - !githubConfig.hiddenTeams.contains(team.name) githubConnector .getTeams() - .map(_.filter(team => notHidden(team))) .recoverWith { case NonFatal(ex) => logger.error("Could not retrieve teams for organisation list.", ex) @@ -56,12 +47,8 @@ class GithubV3RepositoryDataSource @Inject()( } def getTeams(repoName: String)(implicit ec: ExecutionContext): Future[List[String]] = { - def notHidden(teamName: String) = - !githubConfig.hiddenTeams.contains(teamName) - githubConnector .getTeams(repoName) - .map(_.filter(team => notHidden(team))) .recoverWith { case NonFatal(ex) => logger.error(s"Could not retrieve teams for repo: $repoName.", ex) @@ -78,12 +65,9 @@ class GithubV3RepositoryDataSource @Inject()( ): Future[TeamRepositories] = { logger.info(s"Fetching TeamRepositories for team: ${team.name}") - def notHidden(repoName: String) = - !githubConfig.hiddenRepositories.contains(repoName) - for { ghRepos <- githubConnector.getReposForTeam(team) - repos = ghRepos.collect { case r if notHidden(r.name) => cache.getOrElse(r.name, r.toGitRepository) } + repos = ghRepos.map(repo => cache.getOrElse(repo.name, repo.toGitRepository)) } yield TeamRepositories( teamName = team.name, @@ -98,13 +82,11 @@ class GithubV3RepositoryDataSource @Inject()( } def getAllRepositories()(implicit ec: ExecutionContext): Future[List[GitRepository]] = { - def notHidden(repoName: String) = - !githubConfig.hiddenRepositories.contains(repoName) logger.info("Fetching all repositories from GitHub") githubConnector.getRepos() - .map(_.collect { case repo if notHidden(repo.name) => repo.toGitRepository }) + .map(_.map(_.toGitRepository)) .map { repos => logger.info(s"Finished fetching all repositories from GitHub (total fetched: ${repos.size})") repos @@ -117,13 +99,9 @@ class GithubV3RepositoryDataSource @Inject()( } def getRepo(repoName: String)(implicit ec: ExecutionContext): Future[Option[GitRepository]] = { - def notHidden(repoName: String) = - !githubConfig.hiddenRepositories.contains(repoName) - logger.info(s"Fetching repo: $repoName from GitHub") githubConnector.getRepo(repoName) - .map(_.filter(repo => notHidden(repo.name))) .map(_.map(_.toGitRepository)) .recoverWith { case NonFatal(ex) => diff --git a/app/uk/gov/hmrc/teamsandrepositories/services/PersistingService.scala b/app/uk/gov/hmrc/teamsandrepositories/services/PersistingService.scala index 3f5599f1..1f761287 100644 --- a/app/uk/gov/hmrc/teamsandrepositories/services/PersistingService.scala +++ b/app/uk/gov/hmrc/teamsandrepositories/services/PersistingService.scala @@ -22,14 +22,15 @@ import com.google.inject.{Inject, Singleton} import play.api.{Configuration, Logger} import uk.gov.hmrc.teamsandrepositories.connectors.ServiceConfigsConnector import uk.gov.hmrc.teamsandrepositories.models._ -import uk.gov.hmrc.teamsandrepositories.persistence.{RepositoriesPersistence, TestRepoRelationshipsPersistence} +import uk.gov.hmrc.teamsandrepositories.persistence.{RepositoriesPersistence, TeamSummaryPersistence, TestRepoRelationshipsPersistence} import uk.gov.hmrc.teamsandrepositories.persistence.TestRepoRelationshipsPersistence.TestRepoRelationship import scala.concurrent.{ExecutionContext, Future} @Singleton case class PersistingService @Inject()( - persister : RepositoriesPersistence, + repositoriesPersistence : RepositoriesPersistence, + teamSummaryPersistence : TeamSummaryPersistence, relationshipsPersistence: TestRepoRelationshipsPersistence, dataSource : GithubV3RepositoryDataSource, configuration : Configuration, @@ -37,59 +38,86 @@ case class PersistingService @Inject()( ) { private val logger = Logger(this.getClass) - def updateRepositories()(implicit ec: ExecutionContext): Future[Int] = + private def updateTeams(gitRepos: List[GitRepository], teamRepos: List[TeamRepositories])(implicit ec: ExecutionContext): Future[Unit] = { + for { + count <- teamSummaryPersistence.updateTeamSummaries( + TeamSummary.createTeamSummaries(gitRepos) ++ + teamRepos.collect { + // we also store teams with no repos + case trs if trs.repositories.isEmpty => TeamSummary(trs.teamName, None, Seq.empty) + } + ) + _ = logger.info(s"Persisted: $count Teams") + } yield () + } + + private def updateRepositories(reposWithTeams: Map[String, GitRepository])(implicit ec: ExecutionContext): Future[Unit] = for { frontendRoutes <- serviceConfigsConnector.getFrontendServices() adminFrontendRoutes <- serviceConfigsConnector.getAdminFrontendServices() - teams <- dataSource.getTeams() - teamRepos <- teams.foldLeftM(List.empty[TeamRepositories]) { case (acc, team) => - dataSource.getTeamRepositories(team).map(_ :: acc) - } - reposWithTeams = teamRepos.foldLeft(Map.empty[String, GitRepository]) { case (acc, trs) => - trs.repositories.foldLeft(acc) { case (acc, repo) => - val r = acc.getOrElse(repo.name, repo) - acc + (r.name -> r.copy(teams = trs.teamName :: r.teams)) - } - } allRepos <- dataSource.getAllRepositoriesByName() orphanRepos = (allRepos -- reposWithTeams.keys).values reposToPersist = (reposWithTeams.values.toSeq ++ orphanRepos) .map(defineServiceType(_, frontendRoutes = frontendRoutes, adminFrontendRoutes = adminFrontendRoutes)) .map(defineTag(_, adminFrontendRoutes = adminFrontendRoutes)) _ = logger.info(s"found ${reposToPersist.length} repos") - count <- persister.updateRepos(reposToPersist) + count <- repositoriesPersistence.updateRepos(reposToPersist) + _ = logger.info(s"Persisted: $count repos") _ <- reposToPersist.toList.traverse(updateTestRepoRelationships) - } yield count + } yield () + + def updateTeamsAndRepositories()(implicit ec: ExecutionContext): Future[Unit] = + for { + teams <- dataSource.getTeams() + teamRepos <- teams.foldLeftM(List.empty[TeamRepositories]) { case (acc, team) => + dataSource.getTeamRepositories(team).map(_ :: acc) + } + reposWithTeams = teamRepos.foldLeft(Map.empty[String, GitRepository]) { case (acc, trs) => + trs.repositories.foldLeft(acc) { case (acc, repo) => + val r = acc.getOrElse(repo.name, repo) + acc + (r.name -> r.copy(teams = trs.teamName :: r.teams)) + } + }.view.mapValues(repo => + repo.copy(owningTeams = if (repo.owningTeams.isEmpty) repo.teams else repo.owningTeams) + ) + _ <- updateTeams(reposWithTeams.values.toList, teamRepos) + _ <- updateRepositories(reposWithTeams.toMap) + } yield () - def updateRepository(name: String)(implicit ec: ExecutionContext): EitherT[Future, String, Unit] = + def updateRepository(repoName: String)(implicit ec: ExecutionContext): EitherT[Future, String, Unit] = for { rawRepo <- EitherT.fromOptionF( - dataSource.getRepo(name) + dataSource.getRepo(repoName) , s"not found on github" ) - teams <- EitherT.liftF(dataSource.getTeams(name)) + teams <- EitherT.liftF(dataSource.getTeams(repoName)) frontendRoutes <- EitherT - .liftF(serviceConfigsConnector.hasFrontendRoutes(name)) - .map(x => if (x) Set(name) else Set.empty[String]) + .liftF(serviceConfigsConnector.hasFrontendRoutes(repoName)) + .map(x => if (x) Set(repoName) else Set.empty[String]) adminFrontendRoutes <- EitherT - .liftF(serviceConfigsConnector.hasAdminFrontendRoutes(name)) - .map(x => if (x) Set(name) else Set.empty[String]) + .liftF(serviceConfigsConnector.hasAdminFrontendRoutes(repoName)) + .map(x => if (x) Set(repoName) else Set.empty[String]) repo <- EitherT .pure[Future, String](rawRepo) - .map(_.copy(teams = teams)) + .map(repo => + repo.copy( + teams = teams, + owningTeams = if (repo.owningTeams.isEmpty) teams else repo.owningTeams + ) + ) .map(defineServiceType(_, frontendRoutes = frontendRoutes, adminFrontendRoutes = adminFrontendRoutes)) .map(defineTag(_, adminFrontendRoutes = adminFrontendRoutes)) _ <- EitherT - .liftF(persister.putRepo(repo)) + .liftF(repositoriesPersistence.putRepo(repo)) _ <- EitherT .liftF(updateTestRepoRelationships(rawRepo)) } yield () def repositoryArchived(repoName: String): Future[Unit] = - persister.archiveRepo(repoName) + repositoriesPersistence.archiveRepo(repoName) def repositoryDeleted(repoName: String): Future[Unit] = - persister.deleteRepo(repoName) + repositoriesPersistence.deleteRepo(repoName) private def updateTestRepoRelationships(repo: GitRepository)(implicit ec: ExecutionContext): Future[Unit] = { import uk.gov.hmrc.teamsandrepositories.util.YamlMap diff --git a/conf/app.routes b/conf/app.routes index bcc889b4..f823f825 100644 --- a/conf/app.routes +++ b/conf/app.routes @@ -24,7 +24,7 @@ GET /api/cache/reload/:serviceName uk.gov.hmrc # v2 api -GET /api/v2/repositories uk.gov.hmrc.teamsandrepositories.controller.v2.TeamsAndRepositoriesController.allRepos(name: Option[String], team: Option[String], archived: Option[Boolean], repoType: Option[RepoType], serviceType: Option[ServiceType], tag: Option[List[Tag]]) +GET /api/v2/repositories uk.gov.hmrc.teamsandrepositories.controller.v2.TeamsAndRepositoriesController.repositories(name: Option[String], team: Option[String], owningTeam: Option[String], archived: Option[Boolean], repoType: Option[RepoType], serviceType: Option[ServiceType], tag: Option[List[Tag]]) GET /api/v2/repositories/:repoName uk.gov.hmrc.teamsandrepositories.controller.v2.TeamsAndRepositoriesController.findRepo(repoName) GET /api/v2/repositories/:repoName/jenkins-url uk.gov.hmrc.teamsandrepositories.controller.JenkinsController.lookup(repoName) GET /api/v2/repositories/:repoName/jenkins-jobs uk.gov.hmrc.teamsandrepositories.controller.JenkinsController.findAllJobsByRepo(repoName) diff --git a/project/AppDependencies.scala b/project/AppDependencies.scala index 6bfc231e..f8ed0440 100644 --- a/project/AppDependencies.scala +++ b/project/AppDependencies.scala @@ -2,14 +2,14 @@ import play.sbt.PlayImport.ws import sbt._ object AppDependencies { - val bootstrapPlayVersion = "8.1.0" - val hmrcMongoVersion = "1.6.0" + val bootstrapPlayVersion = "8.4.0" + val hmrcMongoVersion = "1.7.0" val compile = Seq( ws, "uk.gov.hmrc" %% "bootstrap-backend-play-30" % bootstrapPlayVersion, "uk.gov.hmrc.mongo" %% "hmrc-mongo-metrix-play-30" % hmrcMongoVersion, - "uk.gov.hmrc" %% "internal-auth-client-play-30" % "1.8.0", + "uk.gov.hmrc" %% "internal-auth-client-play-30" % "1.10.0", "org.yaml" % "snakeyaml" % "2.0", "org.typelevel" %% "cats-core" % "2.10.0", "org.codehaus.groovy" % "groovy-astbuilder" % "3.0.16", diff --git a/project/plugins.sbt b/project/plugins.sbt index 76298e7b..7cb20728 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,6 @@ resolvers += MavenRepository("HMRC-open-artefacts-maven2", "https://open.artefacts.tax.service.gov.uk/maven2") resolvers += Resolver.url("HMRC-open-artefacts-ivy2", url("https://open.artefacts.tax.service.gov.uk/ivy2"))(Resolver.ivyStylePatterns) -addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.18.0") -addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.4.0") -addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-auto-build" % "3.20.0") +addSbtPlugin("uk.gov.hmrc" % "sbt-distributables" % "2.5.0") +addSbtPlugin("org.playframework" % "sbt-plugin" % "3.0.1") diff --git a/test/uk/gov/hmrc/teamsandrepositories/models/TeamSummarySpec.scala b/test/uk/gov/hmrc/teamsandrepositories/models/TeamSummarySpec.scala new file mode 100644 index 00000000..a76348c3 --- /dev/null +++ b/test/uk/gov/hmrc/teamsandrepositories/models/TeamSummarySpec.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2024 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.teamsandrepositories.models + +import org.scalatest.matchers.must.Matchers +import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper +import org.scalatest.wordspec.AnyWordSpec + +import java.time.Instant +import java.time.temporal.ChronoUnit + +class TeamSummarySpec extends AnyWordSpec with Matchers { + + "TeamSummary.createTeamSummaries" in new Setup { + val gitRepo1 = gitRepository.copy(name = "repo-one", owningTeams = Seq("A"), teams = List("A", "B", "C")) + val gitRepo2 = gitRepository.copy(name = "repo-two", owningTeams = Seq("B"), teams = List("A", "B", "C"), lastActiveDate = now.minus(5, ChronoUnit.DAYS)) + val gitRepo3 = gitRepository.copy(name = "repo-three", owningTeams = Seq("A", "B"), teams = List("A", "B", "C"), lastActiveDate = now.minus(15, ChronoUnit.DAYS)) + val gitRepo4 = gitRepository.copy(name = "repo-four", owningTeams = Seq("A", "B"), teams = List("A", "B", "C"), isArchived = true) + + TeamSummary.createTeamSummaries(List(gitRepo1, gitRepo2, gitRepo3, gitRepo4)) shouldBe Seq( + TeamSummary("A", Some(now), Seq("repo-one", "repo-three")), + TeamSummary("B", Some(now), Seq("repo-two", "repo-three")), + TeamSummary("C", Some(now), Seq.empty) + ) + } + + trait Setup { + val now: Instant = Instant.now() + + val gitRepository: GitRepository = + GitRepository( + name = "", + description = "some description", + url = "url", + createdDate = now, + lastActiveDate= now, + repoType = RepoType.Other, + owningTeams = Seq.empty, + teams = List.empty, + language = Some("Scala"), + isArchived = false, + defaultBranch = "main" + ) + } + +} diff --git a/test/uk/gov/hmrc/teamsandrepositories/persistence/RepositoriesPersistenceSpec.scala b/test/uk/gov/hmrc/teamsandrepositories/persistence/RepositoriesPersistenceSpec.scala index dfe0117d..13793f99 100644 --- a/test/uk/gov/hmrc/teamsandrepositories/persistence/RepositoriesPersistenceSpec.scala +++ b/test/uk/gov/hmrc/teamsandrepositories/persistence/RepositoriesPersistenceSpec.scala @@ -45,132 +45,138 @@ class RepositoriesPersistenceSpec private val repo1 = GitRepository( - "repo1", - "desc 1", - "git/repo1", - now, - now, - isPrivate = false, - RepoType.Service, - serviceType = None, - tags = None, - None, - Nil, - None, - isArchived = false, - "main", - branchProtection = Some(BranchProtection(requiresApprovingReviews = true, dismissesStaleReview = true, requiresCommitSignatures = true)), - isDeprecated = false, - List("team1", "team2"), - None + name = "repo1", + description = "desc 1", + url = "git/repo1", + createdDate = now, + lastActiveDate = now, + isPrivate = false, + repoType = RepoType.Service, + serviceType = None, + tags = None, + digitalServiceName = None, + owningTeams = Nil, + language = None, + isArchived = false, + defaultBranch = "main", + branchProtection = Some(BranchProtection(requiresApprovingReviews = true, dismissesStaleReview = true, requiresCommitSignatures = true)), + isDeprecated = false, + teams = List("team1", "team2"), + prototypeName = None ) private val repo2 = GitRepository( - "repo2", - "desc 2", - "git/repo2", - now, - now, - isPrivate = false, - RepoType.Service, - serviceType = None, - tags = None, - None, - Nil, - None, - isArchived = true, - "main", - branchProtection = None, - isDeprecated = false, - List("team2", "team3"), - None + name = "repo2", + description = "desc 2", + url = "git/repo2", + createdDate = now, + lastActiveDate = now, + isPrivate = false, + repoType = RepoType.Service, + serviceType = None, + tags = None, + digitalServiceName = None, + owningTeams = List("team4", "team5"), + language = None, + isArchived = true, + defaultBranch = "main", + branchProtection = None, + isDeprecated = false, + teams = List("team2", "team3"), + prototypeName = None ) private val repo3 = GitRepository( - "repo3", - "desc 3", - "git/repo3", - now, - now, - isPrivate = false, - RepoType.Prototype, - serviceType = None, - tags = None, - None, - Nil, - None, - isArchived = true, - "main", - branchProtection = None, - isDeprecated = false, - List("team1","team2", "team3"), - Some("https://repo3.herokuapp.com") + name = "repo3", + description = "desc 3", + url = "git/repo3", + createdDate = now, + lastActiveDate = now, + isPrivate = false, + repoType = RepoType.Prototype, + serviceType = None, + tags = None, + digitalServiceName = None, + owningTeams = Nil, + language = None, + isArchived = true, + defaultBranch = "main", + branchProtection = None, + isDeprecated = false, + teams = List("team1","team2", "team3"), + prototypeName = Some("https://repo3.herokuapp.com") ) private val repo4 = GitRepository( - "repo4", - "desc 4", - "git/repo4", - now, - now, - isPrivate = false, - RepoType.Service, - serviceType = Some(ServiceType.Frontend), - tags = None, - None, - Nil, - None, - isArchived = true, - "main", - branchProtection = None, - isDeprecated = false, - List("team2", "team3"), - None + name = "repo4", + description = "desc 4", + url = "git/repo4", + createdDate = now, + lastActiveDate = now, + isPrivate = false, + repoType = RepoType.Service, + serviceType = Some(ServiceType.Frontend), + tags = None, + digitalServiceName = None, + owningTeams = Nil, + language = None, + isArchived = true, + defaultBranch = "main", + branchProtection = None, + isDeprecated = false, + teams = List("team2", "team3"), + prototypeName = None ) private val repo5 = GitRepository( - "repo5", - "desc 5", - "git/repo5", - now, - now, - isPrivate = false, - RepoType.Service, - serviceType = Some(ServiceType.Backend), - tags = None, - None, - Nil, - None, - isArchived = true, - "main", - branchProtection = None, - isDeprecated = false, - List("team2", "team3"), - None + name = "repo5", + description = "desc 5", + url = "git/repo5", + createdDate = now, + lastActiveDate = now, + isPrivate = false, + repoType = RepoType.Service, + serviceType = Some(ServiceType.Backend), + tags = None, + digitalServiceName = None, + owningTeams = Nil, + language = None, + isArchived = true, + defaultBranch = "main", + branchProtection = None, + isDeprecated = false, + teams = List("team2", "team3"), + prototypeName = None ) - "search" must { + "find" must { "find all repos" in { repository.collection.insertMany(Seq(repo1, repo2)).toFuture().futureValue - val results = repository.search().futureValue + val results = repository.find().futureValue results must contain (repo1) results must contain (repo2) } "exclude archived repos" in { repository.collection.insertMany(Seq(repo1, repo2)).toFuture().futureValue - val results = repository.search(isArchived = Some(false)).futureValue + val results = repository.find(isArchived = Some(false)).futureValue results must contain (repo1) results must not contain (repo2) } - "find repos belonging to a team" in { + "find repos with team write-access" in { repository.collection.insertMany(Seq(repo1, repo2)).toFuture().futureValue - val results = repository.search(team = Some("team3")).futureValue + val results = repository.find(team = Some("team3")).futureValue + results must contain only (repo2) + } + + "find repos owned by team" in { + repository.collection.insertMany(Seq(repo1, repo2)).toFuture().futureValue + val results = repository.find(owningTeam = Some("team4")).futureValue results must contain only (repo2) } @@ -178,36 +184,28 @@ class RepositoriesPersistenceSpec val foo = repo1.copy(name = "foo") val bar = repo1.copy(name = "bar") repository.collection.insertMany(Seq(repo1, repo2, foo, bar)).toFuture().futureValue - val results = repository.search(name = Some("repo")).futureValue + val results = repository.find(name = Some("repo")).futureValue results must contain theSameElementsAs Seq(repo1, repo2) } "find repos by repo type" in { repository.collection.insertMany(Seq(repo1, repo2, repo3)).toFuture().futureValue - val results = repository.search(repoType = Some(RepoType.Prototype)).futureValue + val results = repository.find(repoType = Some(RepoType.Prototype)).futureValue results must contain only (repo3) - val results2 = repository.search(repoType = Some(RepoType.Service)).futureValue + val results2 = repository.find(repoType = Some(RepoType.Service)).futureValue results2 must contain theSameElementsAs Seq(repo1, repo2) } "find repos by service type" in { repository.collection.insertMany(Seq(repo3, repo4, repo5)).toFuture().futureValue - val results = repository.search(serviceType = Some(ServiceType.Frontend)).futureValue + val results = repository.find(serviceType = Some(ServiceType.Frontend)).futureValue results must contain only repo4 - val results2 = repository.search(serviceType = Some(ServiceType.Backend)).futureValue + val results2 = repository.find(serviceType = Some(ServiceType.Backend)).futureValue results2 must contain only repo5 } } - "findTeamSummaries" must { - "return all the unique team names" in { - repository.collection.insertMany(Seq(repo1, repo2)).toFuture().futureValue - val results = repository.findTeamSummaries().futureValue - results.map(_.name) must contain theSameElementsAs Seq("team1", "team2", "team3") - } - } - "update" must { "insert new repositories" in { repository.updateRepos(Seq(repo1,repo2)).futureValue @@ -232,14 +230,15 @@ class RepositoriesPersistenceSpec "updateRepoBranchProtection" should { "update the branch protection policy of the given repository" in { (for { - _ <- insert(repo1) - bpBefore <- repository.findRepo(repo1.name).map(_.value.branchProtection) - expectedBpAfter = BranchProtection(false, false, false) - _ <- repository.updateRepoBranchProtection(repo1.name, Some(expectedBpAfter)) - bpAfter <- repository.findRepo(repo1.name).map(_.value.branchProtection) - _ = bpAfter mustNot be(bpBefore) - _ = bpAfter mustBe Some(expectedBpAfter) - } yield ()).futureValue + _ <- insert(repo1) + bpBefore <- repository.findRepo(repo1.name).map(_.value.branchProtection) + expectedBpAfter = BranchProtection(false, false, false) + _ <- repository.updateRepoBranchProtection(repo1.name, Some(expectedBpAfter)) + bpAfter <- repository.findRepo(repo1.name).map(_.value.branchProtection) + _ = bpAfter mustNot be(bpBefore) + _ = bpAfter mustBe Some(expectedBpAfter) + } yield () + ).futureValue } } @@ -252,14 +251,15 @@ class RepositoriesPersistenceSpec "update an existing repository" in { (for { - _ <- repository.putRepo(repo1) - serviceTypeBefore <- repository.findRepo(repo1.name).map(_.value.serviceType) - expectedServiceTypeAfter = ServiceType.Backend - _ <- repository.putRepo(repo1.copy(serviceType = Some(expectedServiceTypeAfter))) - serviceTypeAfter <- repository.findRepo(repo1.name).map(_.value.serviceType) - _ = serviceTypeAfter mustNot be(serviceTypeBefore) - _ = serviceTypeAfter mustBe Some(expectedServiceTypeAfter) - } yield ()).futureValue + _ <- repository.putRepo(repo1) + serviceTypeBefore <- repository.findRepo(repo1.name).map(_.value.serviceType) + expectedServiceTypeAfter = ServiceType.Backend + _ <- repository.putRepo(repo1.copy(serviceType = Some(expectedServiceTypeAfter))) + serviceTypeAfter <- repository.findRepo(repo1.name).map(_.value.serviceType) + _ = serviceTypeAfter mustNot be(serviceTypeBefore) + _ = serviceTypeAfter mustBe Some(expectedServiceTypeAfter) + } yield () + ).futureValue } } diff --git a/test/uk/gov/hmrc/teamsandrepositories/persistence/TeamSummaryPersistenceSpec.scala b/test/uk/gov/hmrc/teamsandrepositories/persistence/TeamSummaryPersistenceSpec.scala new file mode 100644 index 00000000..63484b1b --- /dev/null +++ b/test/uk/gov/hmrc/teamsandrepositories/persistence/TeamSummaryPersistenceSpec.scala @@ -0,0 +1,86 @@ +/* + * Copyright 2024 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.gov.hmrc.teamsandrepositories.persistence + +import org.mockito.MockitoSugar +import org.scalatest.matchers.must.Matchers +import org.scalatest.wordspec.AnyWordSpecLike +import uk.gov.hmrc.mongo.test.DefaultPlayMongoRepositorySupport +import uk.gov.hmrc.teamsandrepositories.models.TeamSummary + +import java.time.Instant +import java.time.temporal.ChronoUnit +import scala.concurrent.ExecutionContext.Implicits.global + +class TeamSummaryPersistenceSpec + extends AnyWordSpecLike + with Matchers + with MockitoSugar + with DefaultPlayMongoRepositorySupport[TeamSummary] { + + override protected val repository = new TeamSummaryPersistence(mongoComponent) + + override protected val checkIndexedQueries: Boolean = + // we run unindexed queries + false + + private val now = Instant.now().truncatedTo(ChronoUnit.MILLIS) + + private val teamSummary1 = + TeamSummary( + name = "team-one", + lastActiveDate = Some(now), + repos = Seq("repo-one") + ) + + private val teamSummary2 = + TeamSummary( + name = "team-two", + lastActiveDate = Some(now), + repos = Seq("repo-one") + ) + + "update" must { + "insert new team summaries" in { + repository.updateTeamSummaries(List(teamSummary1, teamSummary2)).futureValue + findAll().futureValue must contain theSameElementsAs List(teamSummary1, teamSummary2) + } + + "update existing teams summaries" in { + insert(teamSummary1.copy(lastActiveDate = Some(now.minus(5, ChronoUnit.DAYS)))).futureValue + repository.updateTeamSummaries(List(teamSummary1, teamSummary2)).futureValue + findAll().futureValue must contain theSameElementsAs List(teamSummary1, teamSummary2) + } + + "delete team summaries not in the update list" in { + insert(teamSummary1).futureValue + insert(teamSummary2).futureValue + repository.updateTeamSummaries(List(teamSummary1)).futureValue + findAll().futureValue must contain (teamSummary1) + findAll().futureValue must not contain teamSummary2 + } + } + + "findTeamSummaries" must { + "return all summaries for all teams" in { + insert(teamSummary1).futureValue + insert(teamSummary2).futureValue + repository.findTeamSummaries().futureValue must contain theSameElementsAs Seq(teamSummary1, teamSummary2) + } + } + +} diff --git a/test/uk/gov/hmrc/teamsandrepositories/services/GithubV3RepositoryDataSourceSpec.scala b/test/uk/gov/hmrc/teamsandrepositories/services/GithubV3RepositoryDataSourceSpec.scala index 72349afe..6eae162e 100644 --- a/test/uk/gov/hmrc/teamsandrepositories/services/GithubV3RepositoryDataSourceSpec.scala +++ b/test/uk/gov/hmrc/teamsandrepositories/services/GithubV3RepositoryDataSourceSpec.scala @@ -23,9 +23,7 @@ import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.time.SpanSugar import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import play.api.Configuration import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, TeamRepositories} -import uk.gov.hmrc.teamsandrepositories.config.GithubConfig import uk.gov.hmrc.teamsandrepositories.connectors.GhRepository.RepoTypeHeuristics import uk.gov.hmrc.teamsandrepositories.connectors.{GhRepository, GhTeam, GithubConnector} @@ -52,23 +50,11 @@ class GithubV3RepositoryDataSourceSpec trait Setup { val mockGithubConnector = mock[GithubConnector] - val githubConfig: GithubConfig = mock[GithubConfig] - val dataSource = new GithubV3RepositoryDataSource( - githubConfig = githubConfig, githubConnector = mockGithubConnector, - timeStamper = testTimeStamper, - configuration = Configuration( - "shared.repositories" -> Seq() - ) + timeStamper = testTimeStamper ) - - when(githubConfig.hiddenRepositories) - .thenReturn(testHiddenRepositories) - - when(githubConfig.hiddenTeams) - .thenReturn(testHiddenTeams) } val teamA = GhTeam(name = "A", createdAt = teamCreatedDate) @@ -101,26 +87,22 @@ class GithubV3RepositoryDataSourceSpec repoTypeHeuristics = dummyRepoTypeHeuristics ) - val testHiddenRepositories = Set("hidden_repo1", "hidden_repo2") - val testHiddenTeams = Set("hidden_team1", "hidden_team2") - "GithubV3RepositoryDataSource.getTeams" should { - "return a list of teams and data sources filtering out hidden teams" in new Setup { - private val teamB = GhTeam(name = "B" , createdAt = teamCreatedDate) - private val hiddenTeam1 = GhTeam(name = "hidden_team1", createdAt = teamCreatedDate) + "return a list of teams and data sources" in new Setup { + private val teamB = GhTeam(name = "B", createdAt = teamCreatedDate) when(mockGithubConnector.getTeams()) - .thenReturn(Future.successful(List(teamA, teamB, hiddenTeam1))) + .thenReturn(Future.successful(List(teamA, teamB))) val result = dataSource.getTeams().futureValue result.size shouldBe 2 - result should contain theSameElementsAs Seq(teamA, teamB) + result should contain theSameElementsAs Seq(teamA, teamB) } } "GithubV3RepositoryDataSource.getAllRepositories" should { - "return a list of teams and data sources filtering out hidden teams" in new Setup { + "return a list of teams and data sources" in new Setup { private val repo1 = GhRepository( name = "repo1", htmlUrl = "http://github.com/repo1", @@ -189,69 +171,7 @@ class GithubV3RepositoryDataSourceSpec } } - "GithubV3RepositoryDataSource.getTeamRepositories" should { - "filter out repositories according to the hidden config" in new Setup { - when(mockGithubConnector.getTeams()) - .thenReturn(Future.successful(List(teamA))) - when(mockGithubConnector.getReposForTeam(teamA)) - .thenReturn(Future.successful(List( - GhRepository( - name = "hidden_repo1", - htmlUrl = "url_A", - fork = false, - createdDate = now, - pushedAt = now, - isPrivate = false, - language = Some("Scala"), - isArchived = false, - defaultBranch = "main", - branchProtection = None, - repositoryYamlText = None, - repoTypeHeuristics = dummyRepoTypeHeuristics - ), - GhRepository( - name = "A_r", - htmlUrl = "url_A", - fork = false, - createdDate = now, - pushedAt = now, - isPrivate = false, - language = Some("Scala"), - isArchived = false, - defaultBranch = "main", - branchProtection = None, - repositoryYamlText = Some("description: a test repo"), - repoTypeHeuristics = dummyRepoTypeHeuristics - ) - ))) - - dataSource - .getTeamRepositories(teamA) - .futureValue shouldBe - TeamRepositories( - teamName = "A", - repositories = List( - GitRepository( - name = "A_r", - description = "a test repo", - url = "url_A", - createdDate = now, - lastActiveDate = now, - isPrivate = false, - repoType = RepoType.Other, - digitalServiceName = None, - owningTeams = Nil, - language = Some("Scala"), - isArchived = false, - defaultBranch = "main", - repositoryYamlText = Some("description: a test repo") - ) - ), - createdDate = Some(teamCreatedDate), - updateDate = now - ) - } - + "GithubV3RepositoryDataSource.getAllRepositories" should { "set repoType Service if the repository contains an app/application.conf file" in new Setup { when(mockGithubConnector.getTeams()) .thenReturn(Future.successful(List(teamA))) @@ -262,14 +182,13 @@ class GithubV3RepositoryDataSourceSpec List( ghRepo .copy(repoTypeHeuristics = ghRepo.repoTypeHeuristics.copy(hasApplicationConf = true)) + ) ) - ) - ) + ) dataSource .getTeamRepositories(teamA) - .futureValue shouldBe - TeamRepositories( + .futureValue shouldBe TeamRepositories( teamName = "A", repositories = List( GitRepository( @@ -286,8 +205,8 @@ class GithubV3RepositoryDataSourceSpec prototypeName = None ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -307,8 +226,7 @@ class GithubV3RepositoryDataSourceSpec dataSource .getTeamRepositories(teamA) - .futureValue shouldBe - TeamRepositories( + .futureValue shouldBe TeamRepositories( teamName = "A", repositories = List( GitRepository( @@ -325,8 +243,8 @@ class GithubV3RepositoryDataSourceSpec prototypeName = None ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -348,25 +266,25 @@ class GithubV3RepositoryDataSourceSpec dataSource .getTeamRepositories(teamA) .futureValue shouldBe TeamRepositories( - teamName = "A", - repositories = List( - GitRepository( - name = "A_r", - description = "", - url = "url_A", - createdDate = now, - lastActiveDate = now, - repoType = RepoType.Service, - digitalServiceName = None, - language = Some("Scala"), - isArchived = false, - defaultBranch = "main", - prototypeName = None - ) - ), - createdDate = Some(teamCreatedDate), - updateDate = now - ) + teamName = "A", + repositories = List( + GitRepository( + name = "A_r", + description = "", + url = "url_A", + createdDate = now, + lastActiveDate = now, + repoType = RepoType.Service, + digitalServiceName = None, + language = Some("Scala"), + isArchived = false, + defaultBranch = "main", + prototypeName = None + ) + ), + createdDate = Some(teamCreatedDate), + updateDate = now + ) } "set type as Deployable according if the repository.yaml contains a type of 'service'" in new Setup { @@ -404,8 +322,8 @@ class GithubV3RepositoryDataSourceSpec repositoryYamlText = Some("type: service") ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -444,8 +362,8 @@ class GithubV3RepositoryDataSourceSpec repositoryYamlText = Some("digital-service: service-abcd") ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -488,8 +406,8 @@ class GithubV3RepositoryDataSourceSpec repositoryYamlText = Some(manifestYaml) ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -525,8 +443,8 @@ class GithubV3RepositoryDataSourceSpec repositoryYamlText = Some("type: library") ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -562,8 +480,8 @@ class GithubV3RepositoryDataSourceSpec repositoryYamlText = Some("type: somethingelse") ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -577,40 +495,42 @@ class GithubV3RepositoryDataSourceSpec .successful( List( ghRepo - .copy(repoTypeHeuristics = - ghRepo.repoTypeHeuristics.copy( - hasSrcMainScala = true, - hasTags = true + .copy( + repoTypeHeuristics = + ghRepo.repoTypeHeuristics.copy( + hasSrcMainScala = true, + hasTags = true + ) ) ) ) - ) ) val repositories = dataSource .getTeamRepositories(teamA) .futureValue - repositories shouldBe TeamRepositories( - teamName = "A", - repositories = List( - GitRepository( - name = "A_r", - description = "", - url = "url_A", - createdDate = now, - lastActiveDate = now, - repoType = RepoType.Library, - digitalServiceName = None, - language = Some("Scala"), - isArchived = false, - defaultBranch = "main", - prototypeName = None - ) - ), - createdDate = Some(teamCreatedDate), - updateDate = now - ) + repositories shouldBe + TeamRepositories( + teamName = "A", + repositories = List( + GitRepository( + name = "A_r", + description = "", + url = "url_A", + createdDate = now, + lastActiveDate = now, + repoType = RepoType.Library, + digitalServiceName = None, + language = Some("Scala"), + isArchived = false, + defaultBranch = "main", + prototypeName = None + ) + ), + createdDate = Some(teamCreatedDate), + updateDate = now + ) } "set type Library if not Service and has src/main/java and has tags" in new Setup { @@ -623,11 +543,12 @@ class GithubV3RepositoryDataSourceSpec .successful( List( ghRepo - .copy(repoTypeHeuristics = - ghRepo.repoTypeHeuristics.copy( - hasSrcMainJava = true, - hasTags = true - ) + .copy( + repoTypeHeuristics = + ghRepo.repoTypeHeuristics.copy( + hasSrcMainJava = true, + hasTags = true + ) ) ) ) @@ -654,8 +575,8 @@ class GithubV3RepositoryDataSourceSpec prototypeName = None ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -693,8 +614,8 @@ class GithubV3RepositoryDataSourceSpec prototypeAutoPublish = None ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -727,10 +648,11 @@ class GithubV3RepositoryDataSourceSpec digitalServiceName = None, language = Some("Scala"), isArchived = false, - defaultBranch = "main") + defaultBranch = "main" + ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -762,8 +684,8 @@ class GithubV3RepositoryDataSourceSpec prototypeName = None ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -810,8 +732,8 @@ class GithubV3RepositoryDataSourceSpec repositoryYamlText = Some(repositoryYamlContents) ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } @@ -845,12 +767,12 @@ class GithubV3RepositoryDataSourceSpec prototypeName = None ) ), - createdDate = Some(teamCreatedDate), - updateDate = now + createdDate = Some(teamCreatedDate), + updateDate = now ) } - "not update repostiories in updatedRepos list" in new Setup { + "not update repositories in updatedRepos list" in new Setup { val githubRepository = ghRepo.copy( createdDate = Instant.ofEpochMilli(0L), diff --git a/test/uk/gov/hmrc/teamsandrepositories/services/PersistingServiceSpec.scala b/test/uk/gov/hmrc/teamsandrepositories/services/PersistingServiceSpec.scala index acf8ef47..aebce5c2 100644 --- a/test/uk/gov/hmrc/teamsandrepositories/services/PersistingServiceSpec.scala +++ b/test/uk/gov/hmrc/teamsandrepositories/services/PersistingServiceSpec.scala @@ -21,14 +21,14 @@ import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import play.api.Configuration -import uk.gov.hmrc.teamsandrepositories.config.GithubConfig import uk.gov.hmrc.teamsandrepositories.connectors.GhRepository.RepoTypeHeuristics import uk.gov.hmrc.teamsandrepositories.connectors.{GhRepository, GhTeam, GithubConnector, ServiceConfigsConnector} -import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, ServiceType, Tag} +import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, ServiceType, Tag, TeamSummary} import uk.gov.hmrc.teamsandrepositories.persistence.TestRepoRelationshipsPersistence.TestRepoRelationship -import uk.gov.hmrc.teamsandrepositories.persistence.{RepositoriesPersistence, TestRepoRelationshipsPersistence} +import uk.gov.hmrc.teamsandrepositories.persistence.{RepositoriesPersistence, TeamSummaryPersistence, TestRepoRelationshipsPersistence} import java.time.Instant +import java.time.temporal.ChronoUnit import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future @@ -41,7 +41,7 @@ class PersistingServiceSpec with IntegrationPatience { "PersistingService" when { - "updating repositories" should { + "updating teams and repositories" should { "assign teams to repositories" in new Setup { val repo1 = aRepo.copy(name = "repo-1") val repo2 = aRepo.copy(name = "repo-2") @@ -49,14 +49,14 @@ class PersistingServiceSpec when(githubConnector.getTeams()).thenReturn(Future.successful(List(teamA, teamB))) when(githubConnector.getReposForTeam(teamA)).thenReturn(Future.successful(List(repo1, repo2))) when(githubConnector.getReposForTeam(teamB)).thenReturn(Future.successful(List(repo3))) - when(githubConnector.getRepos()).thenReturn(Future.successful(List(repo1, repo2, repo3))) when(serviceConfigsConnector.getFrontendServices()).thenReturn(Future.successful(Set())) when(serviceConfigsConnector.getAdminFrontendServices()).thenReturn(Future.successful(Set())) - onTest.updateRepositories().futureValue + when(githubConnector.getRepos()).thenReturn(Future.successful(List(repo1, repo2, repo3))) + onTest.updateTeamsAndRepositories().futureValue val persistedRepos: Seq[GitRepository] = { val argCaptor: ArgumentCaptor[Seq[GitRepository]] = ArgumentCaptor.forClass(classOf[Seq[GitRepository]]) - verify(persister).updateRepos(argCaptor.capture()) + verify(reposPersistence).updateRepos(argCaptor.capture()) argCaptor.getValue } persistedRepos.length shouldBe 3 @@ -77,11 +77,11 @@ class PersistingServiceSpec when(githubConnector.getRepos()).thenReturn(Future.successful(List(repo1, repo2, repo3))) when(serviceConfigsConnector.getFrontendServices()).thenReturn(Future.successful(Set())) when(serviceConfigsConnector.getAdminFrontendServices()).thenReturn(Future.successful(Set())) - onTest.updateRepositories().futureValue + onTest.updateTeamsAndRepositories().futureValue val persistedRepos: Seq[GitRepository] = { val argCaptor: ArgumentCaptor[Seq[GitRepository]] = ArgumentCaptor.forClass(classOf[Seq[GitRepository]]) - verify(persister).updateRepos(argCaptor.capture()) + verify(reposPersistence).updateRepos(argCaptor.capture()) argCaptor.getValue } persistedRepos.length shouldBe 3 @@ -105,11 +105,11 @@ class PersistingServiceSpec when(githubConnector.getRepos()).thenReturn(Future.successful(List(repo1, repo2, repo3, repo4))) when(serviceConfigsConnector.getFrontendServices()).thenReturn(Future.successful(Set())) when(serviceConfigsConnector.getAdminFrontendServices()).thenReturn(Future.successful(Set())) - onTest.updateRepositories().futureValue + onTest.updateTeamsAndRepositories().futureValue val persistedRepos: Seq[GitRepository] = { val argCaptor: ArgumentCaptor[Seq[GitRepository]] = ArgumentCaptor.forClass(classOf[Seq[GitRepository]]) - verify(persister).updateRepos(argCaptor.capture()) + verify(reposPersistence).updateRepos(argCaptor.capture()) argCaptor.getValue } persistedRepos.length shouldBe 4 @@ -127,11 +127,11 @@ class PersistingServiceSpec when(githubConnector.getRepos()).thenReturn(Future.successful(List(repo1, repo2, repo3, repo4, repo5))) when(serviceConfigsConnector.getFrontendServices()).thenReturn(Future.successful(Set(repo2.name))) when(serviceConfigsConnector.getAdminFrontendServices()).thenReturn(Future.successful(Set(repo3.name))) - onTest.updateRepositories().futureValue + onTest.updateTeamsAndRepositories().futureValue val persistedRepos: Seq[GitRepository] = { val argCaptor: ArgumentCaptor[Seq[GitRepository]] = ArgumentCaptor.forClass(classOf[Seq[GitRepository]]) - verify(persister).updateRepos(argCaptor.capture()) + verify(reposPersistence).updateRepos(argCaptor.capture()) argCaptor.getValue } persistedRepos @@ -156,11 +156,11 @@ class PersistingServiceSpec when(githubConnector.getRepos()).thenReturn(Future.successful(List(repo1, repo2, repo3, repo4, repo5, repo6))) when(serviceConfigsConnector.getFrontendServices()).thenReturn(Future.successful(Set.empty)) when(serviceConfigsConnector.getAdminFrontendServices()).thenReturn(Future.successful(Set.empty)) - onTest.updateRepositories().futureValue + onTest.updateTeamsAndRepositories().futureValue val persistedRepos: Seq[GitRepository] = { val argCaptor: ArgumentCaptor[Seq[GitRepository]] = ArgumentCaptor.forClass(classOf[Seq[GitRepository]]) - verify(persister).updateRepos(argCaptor.capture()) + verify(reposPersistence).updateRepos(argCaptor.capture()) argCaptor.getValue } persistedRepos @@ -190,7 +190,7 @@ class PersistingServiceSpec when(serviceConfigsConnector.getFrontendServices()).thenReturn(Future.successful(Set.empty)) when(serviceConfigsConnector.getAdminFrontendServices()).thenReturn(Future.successful(Set.empty)) - onTest.updateRepositories().futureValue + onTest.updateTeamsAndRepositories().futureValue val persistedRelationships: Seq[TestRepoRelationship] = { val argCaptor: ArgumentCaptor[Seq[TestRepoRelationship]] = ArgumentCaptor.forClass(classOf[Seq[TestRepoRelationship]]) @@ -204,31 +204,73 @@ class PersistingServiceSpec TestRepoRelationship("repo-1-performance-tests", "repo-1") ) } + + "create team summaries from repos that have teams" in new Setup { + val repo1 = aRepo.copy(name = "repo-1", pushedAt = now.minus(5, ChronoUnit.DAYS)) + val repo2 = aRepo.copy(name = "repo-2") + val repo3 = aRepo.copy(name = "repo-3") + when(githubConnector.getTeams()).thenReturn(Future.successful(List(teamA, teamB))) + when(githubConnector.getReposForTeam(teamA)).thenReturn(Future.successful(List(repo1, repo2))) + when(githubConnector.getReposForTeam(teamB)).thenReturn(Future.successful(List(repo3))) + when(serviceConfigsConnector.getFrontendServices()).thenReturn(Future.successful(Set())) + when(serviceConfigsConnector.getAdminFrontendServices()).thenReturn(Future.successful(Set())) + when(githubConnector.getRepos()).thenReturn(Future.successful(List(repo1, repo2, repo3))) + onTest.updateTeamsAndRepositories().futureValue + + val persistedTeams: List[TeamSummary] = { + val argCaptor: ArgumentCaptor[List[TeamSummary]] = ArgumentCaptor.forClass(classOf[List[TeamSummary]]) + verify(teamsPersistence).updateTeamSummaries(argCaptor.capture()) + argCaptor.getValue + } + persistedTeams.length shouldBe 2 + persistedTeams should contain theSameElementsAs List(TeamSummary("team-a", Some(now), Seq("repo-1", "repo-2")), TeamSummary("team-b", Some(now), Seq("repo-3"))) + } + + "create team summaries from teams that have no repos" in new Setup { + val repo1 = aRepo.copy(name = "repo-1") + val repo2 = aRepo.copy(name = "repo-2") + val repo3 = aRepo.copy(name = "repo-3") + when(githubConnector.getTeams()).thenReturn(Future.successful(List(teamA, teamB))) + when(githubConnector.getReposForTeam(teamA)).thenReturn(Future.successful(List.empty[GhRepository])) + when(githubConnector.getReposForTeam(teamB)).thenReturn(Future.successful(List.empty[GhRepository])) + when(serviceConfigsConnector.getFrontendServices()).thenReturn(Future.successful(Set())) + when(serviceConfigsConnector.getAdminFrontendServices()).thenReturn(Future.successful(Set())) + when(githubConnector.getRepos()).thenReturn(Future.successful(List(repo1, repo2, repo3))) + onTest.updateTeamsAndRepositories().futureValue + + val persistedTeams: List[TeamSummary] = { + val argCaptor: ArgumentCaptor[List[TeamSummary]] = ArgumentCaptor.forClass(classOf[List[TeamSummary]]) + verify(teamsPersistence).updateTeamSummaries(argCaptor.capture()) + argCaptor.getValue + } + persistedTeams.length shouldBe 2 + persistedTeams should contain theSameElementsAs List(TeamSummary("team-a", None, Seq.empty), TeamSummary("team-b", None, Seq.empty)) + } } } trait Setup { - val githubConfig : GithubConfig = mock[GithubConfig] - val persister : RepositoriesPersistence = mock[RepositoriesPersistence] + val reposPersistence : RepositoriesPersistence = mock[RepositoriesPersistence] + val teamsPersistence : TeamSummaryPersistence = mock[TeamSummaryPersistence] val relationshipsPersistence: TestRepoRelationshipsPersistence = mock[TestRepoRelationshipsPersistence] val githubConnector : GithubConnector = mock[GithubConnector] val serviceConfigsConnector : ServiceConfigsConnector = mock[ServiceConfigsConnector] val timestamper : TimeStamper = new TimeStamper val configuration : Configuration = mock[Configuration] - when(githubConfig.hiddenTeams).thenReturn(Set.empty) - when(githubConfig.hiddenRepositories).thenReturn(Set.empty) - when(configuration.get[Seq[String]]("shared.repositories")).thenReturn(Seq.empty) - when(persister.updateRepos(any)).thenReturn(Future.successful(0)) + when(teamsPersistence.updateTeamSummaries(any)).thenReturn(Future.successful(0)) + when(reposPersistence.updateRepos(any)).thenReturn(Future.successful(0)) when(relationshipsPersistence.putRelationships(any[String], anySeq[TestRepoRelationship])).thenReturn(Future.unit) - val datasource = new GithubV3RepositoryDataSource(githubConfig, githubConnector, timestamper, configuration) + val datasource = new GithubV3RepositoryDataSource(githubConnector, timestamper) val onTest: PersistingService = - PersistingService(persister, relationshipsPersistence, datasource, configuration, serviceConfigsConnector) + PersistingService(reposPersistence, teamsPersistence, relationshipsPersistence, datasource, configuration, serviceConfigsConnector) + + val now: Instant = Instant.now() - val teamA: GhTeam = GhTeam("team-a", Instant.now()) - val teamB: GhTeam = GhTeam("team-b", Instant.now()) + val teamA: GhTeam = GhTeam("team-a", now) + val teamB: GhTeam = GhTeam("team-b", now) val aHeuristics: RepoTypeHeuristics = RepoTypeHeuristics( prototypeInName = false, @@ -245,8 +287,8 @@ class PersistingServiceSpec name = "repo", htmlUrl = "http://github.com/repo1", fork = false, - createdDate = Instant.now(), - pushedAt = Instant.now(), + createdDate = now, + pushedAt = now, isPrivate = false, language = Some("Scala"), isArchived = false,