Skip to content

Commit

Permalink
BDOG-2939: adds team summary collection and removes filters
Browse files Browse the repository at this point in the history
  • Loading branch information
colin-lamed authored and Tyrpix committed Feb 7, 2024
2 parents a9bbe0f + aa145b3 commit 5006dda
Show file tree
Hide file tree
Showing 18 changed files with 657 additions and 483 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,6 @@ class GithubConfig @Inject()(configuration: Configuration) {
val apiUrl = configuration.get[String]("github.open.api.url")
val rawUrl = configuration.get[String]("github.open.api.rawurl")

val hiddenRepositories: Set[String] =
configuration.getOptional[String]("github.hidden.repositories")
.fold(Set.empty[String])(_.split(",").toSet)

val hiddenTeams: Set[String] =
configuration.getOptional[String]("github.hidden.teams")
.fold(Set.empty[String])(_.split(",").toSet)

val tokens: List[(String, String)] =
configuration.get[ConfigList]("ratemetrics.githubtokens").asScala.toList
.map(cv => new Configuration(cv.asInstanceOf[ConfigObject].toConfig))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import play.api.mvc.{Action, ControllerComponents}
import uk.gov.hmrc.internalauth.client.{BackendAuthComponents, IAAction, Predicate, Resource}
import uk.gov.hmrc.play.bootstrap.backend.controller.BackendController
import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, ServiceType, Tag, TeamSummary}
import uk.gov.hmrc.teamsandrepositories.persistence.RepositoriesPersistence
import uk.gov.hmrc.teamsandrepositories.persistence.{RepositoriesPersistence, TeamSummaryPersistence}
import uk.gov.hmrc.teamsandrepositories.services.BranchProtectionService

import javax.inject.{Inject, Singleton}
Expand All @@ -30,6 +30,7 @@ import scala.concurrent.{ExecutionContext, Future}
@Singleton
class TeamsAndRepositoriesController @Inject()(
repositoriesPersistence: RepositoriesPersistence,
teamSummaryPersistence : TeamSummaryPersistence,
branchProtectionService: BranchProtectionService,
auth : BackendAuthComponents,
cc : ControllerComponents
Expand All @@ -40,20 +41,22 @@ class TeamsAndRepositoriesController @Inject()(
implicit val grf: Format[GitRepository] = GitRepository.apiFormat
implicit val tnf: Format[TeamSummary] = TeamSummary.apiFormat

def allRepos(
def repositories(
name : Option[String],
team : Option[String],
owningTeam : Option[String],
archived : Option[Boolean],
repoType : Option[RepoType],
serviceType: Option[ServiceType],
tags : Option[List[Tag]],
) = Action.async { request =>
repositoriesPersistence.search(name, team, archived, repoType, serviceType, tags)
repositoriesPersistence.find(name, team, owningTeam, archived, repoType, serviceType, tags)
.map(result => Ok(Json.toJson(result.sortBy(_.name))))
}

def allTeams() = Action.async { request =>
repositoriesPersistence.findTeamSummaries().map(result => Ok(Json.toJson(result)))
teamSummaryPersistence.findTeamSummaries()
.map(result => Ok(Json.toJson(result)))
}

def findRepo(repoName:String) = Action.async { request =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,6 @@ object GitRepository {
~ (__ \ "prototypeAutoPublish").formatNullable[Boolean]
~ (__ \ "repositoryYamlText" ).formatNullable[String]
)(apply, unlift(unapply))
.bimap(
identity,
repo => if (repo.owningTeams.isEmpty) repo.copy(owningTeams = repo.teams) else repo
)
}

val mongoFormat: OFormat[GitRepository] = {
Expand Down
50 changes: 0 additions & 50 deletions app/uk/gov/hmrc/teamsandrepositories/models/TeamName.scala

This file was deleted.

59 changes: 59 additions & 0 deletions app/uk/gov/hmrc/teamsandrepositories/models/TeamSummary.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Copyright 2023 HM Revenue & Customs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.hmrc.teamsandrepositories.models

import play.api.libs.json.{OFormat, __}
import play.api.libs.functional.syntax._
import uk.gov.hmrc.mongo.play.json.formats.MongoJavatimeFormats

import java.time.Instant

case class TeamSummary(
name : String,
lastActiveDate: Option[Instant],
repos : Seq[String]
)

object TeamSummary {
def createTeamSummaries(gitRepositories: List[GitRepository]): Seq[TeamSummary] =
gitRepositories
.flatMap(repo => repo.teams.map(team => (team, repo)))
.groupBy { case (team, _) => team }
.view.mapValues(_.map{ case (_, repo) => repo })
.map {
case (team, gitRepos) =>
TeamSummary(
name = team,
lastActiveDate = Some(gitRepos.map(_.lastActiveDate).max),
repos = gitRepos.collect {
case gitRepo if gitRepo.owningTeams.contains(team) && !gitRepo.isArchived => gitRepo.name
}
)
}.toSeq

val apiFormat: OFormat[TeamSummary] =
( (__ \ "name" ).format[String]
~ (__ \ "lastActiveDate").formatNullable[Instant]
~ (__ \ "repos" ).format[Seq[String]]
)(TeamSummary.apply, unlift(TeamSummary.unapply))

val mongoFormat: OFormat[TeamSummary] =
( (__ \ "name" ).format[String]
~ (__ \ "lastActiveDate" ).formatNullable[Instant](MongoJavatimeFormats.instantFormat)
~ (__ \ "repos" ).format[Seq[String]]
)(TeamSummary.apply, unlift(TeamSummary.unapply))
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import org.mongodb.scala.model.Aggregates.{`match`, addFields, group, sort, unwi
import org.mongodb.scala.model._
import uk.gov.hmrc.mongo.MongoComponent
import uk.gov.hmrc.mongo.play.json.{Codecs, CollectionFactory, PlayMongoRepository}
import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, ServiceType, Tag, TeamRepositories, TeamSummary}
import uk.gov.hmrc.teamsandrepositories.models.{GitRepository, RepoType, ServiceType, Tag, TeamRepositories}
import uk.gov.hmrc.teamsandrepositories.persistence.Collations.caseInsensitive
import org.mongodb.scala.model.Accumulators.{addToSet, first, max, min}
import org.mongodb.scala.model.Filters.equal
Expand Down Expand Up @@ -56,12 +56,10 @@ class RepositoriesPersistence @Inject()(
private val legacyCollection: MongoCollection[TeamRepositories] =
CollectionFactory.collection(mongoComponent.database, collectionName, TeamRepositories.mongoFormat)

private val teamsCollection: MongoCollection[TeamSummary] =
CollectionFactory.collection(mongoComponent.database, collectionName, TeamSummary.mongoFormat)

def search(
def find(
name : Option[String] = None,
team : Option[String] = None,
owningTeam : Option[String] = None,
isArchived : Option[Boolean] = None,
repoType : Option[RepoType] = None,
serviceType: Option[ServiceType] = None,
Expand All @@ -70,32 +68,22 @@ class RepositoriesPersistence @Inject()(
val filters = Seq(
name .map(n => Filters.regex("name" , n)),
team .map(t => Filters.equal("teamNames" , t)),
owningTeam .map(t => Filters.equal("owningTeams", t)),
isArchived .map(b => Filters.equal("isArchived" , b)),
repoType .map(rt => Filters.equal("repoType" , rt.asString)),
serviceType.map(st => Filters.equal("serviceType", st.asString)),
tags .map(ts => Filters.and(ts.map(t => Filters.equal("tags", t.asString)):_*)),
tags .map(ts => Filters.and(ts.map(t => Filters.equal("tags", t.asString)): _*)),
).flatten
filters match {
case Nil => collection.find().toFuture()
case more => collection.find(Filters.and(more:_*)).toFuture()
}

collection
.find(if (filters.isEmpty) BsonDocument() else Filters.and(filters: _*))
.toFuture()
}

def findRepo(repoName: String): Future[Option[GitRepository]] =
collection
.find(filter = Filters.equal("name", repoName)).headOption()

def findTeamSummaries(): Future[Seq[TeamSummary]] =
teamsCollection
.aggregate(Seq(
addFields(Field("teamSize", BsonDocument("$size" -> "$teamNames"))),
`match`(Filters.lt("teamSize", 8)), // ignore repos shared by more than n teams
unwind("$teamNames"),
group("$teamNames", Accumulators.min("createdDate", "$createdDate"), Accumulators.max("lastActiveDate", "$lastActiveDate"), Accumulators.sum("repos", 1)),
sort(Sorts.ascending("_id"))
))
.toFuture()

def updateRepos(repos: Seq[GitRepository]): Future[Int] =
for {
oldRepos <- collection.find().map(_.name).toFuture().map(_.toSet)
Expand Down Expand Up @@ -140,19 +128,21 @@ class RepositoriesPersistence @Inject()(

// This exists to provide backward compatible data to the old API. Dont use it in new functionality!
def getAllTeamsAndRepos(archived: Option[Boolean]): Future[Seq[TeamRepositories]] =
legacyCollection.aggregate(Seq(
`match`(archived.fold[Bson](BsonDocument())(a => Filters.eq("isArchived", a))),
unwind("$teamNames"),
addFields( Field("teamid", "$teamNames"), Field("teamNames", BsonArray())),
group(
id = "$teamid",
first("teamName", "$teamid"),
addToSet("repositories", "$$ROOT"),
min("createdDate", "$createdDate"),
max("updateDate", "$lastActiveDate")
),
sort(Sorts.ascending("_id"))
)).toFuture()
legacyCollection
.aggregate(Seq(
`match`(archived.fold[Bson](BsonDocument())(a => Filters.eq("isArchived", a))),
unwind("$teamNames"),
addFields( Field("teamid", "$teamNames"), Field("teamNames", BsonArray())),
group(
id = "$teamid",
first("teamName", "$teamid"),
addToSet("repositories", "$$ROOT"),
min("createdDate", "$createdDate"),
max("updateDate", "$lastActiveDate")
),
sort(Sorts.ascending("_id"))
))
.toFuture()

def updateRepoBranchProtection(repoName: String, branchProtection: Option[BranchProtection]): Future[Unit] = {
implicit val bpf = BranchProtection.format
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* Copyright 2024 HM Revenue & Customs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.hmrc.teamsandrepositories.persistence

import org.mongodb.scala.model.Indexes.ascending
import org.mongodb.scala.model.{DeleteOneModel, Filters, IndexModel, IndexOptions, Indexes, ReplaceOneModel, ReplaceOptions}
import play.api.Logger
import uk.gov.hmrc.mongo.MongoComponent
import uk.gov.hmrc.mongo.play.json.PlayMongoRepository
import uk.gov.hmrc.teamsandrepositories.models.TeamSummary
import uk.gov.hmrc.teamsandrepositories.persistence.Collations.caseInsensitive

import javax.inject.{Inject, Singleton}
import scala.concurrent.{ExecutionContext, Future}

@Singleton
class TeamSummaryPersistence @Inject()(
mongoComponent: MongoComponent
)(implicit
ec: ExecutionContext
) extends PlayMongoRepository(
mongoComponent = mongoComponent,
collectionName = "teamSummaries",
domainFormat = TeamSummary.mongoFormat,
indexes = Seq(
IndexModel(Indexes.ascending("name"), IndexOptions().name("nameIdx").collation(caseInsensitive).unique(true)),
)
){

private val logger = Logger(this.getClass)

override lazy val requiresTtlIndex = false

def updateTeamSummaries(teams: Seq[TeamSummary]): Future[Int] =
for {
oldTeams <- collection.find().map(_.name).toFuture().map(_.toSet)
update <- collection
.bulkWrite(teams.map(teamSummary => ReplaceOneModel(Filters.eq("name", teamSummary.name), teamSummary, ReplaceOptions().collation(caseInsensitive).upsert(true))))
.toFuture()
.map(_.getModifiedCount)
toDelete = (oldTeams -- teams.map(_.name)).toSeq
_ <- if (toDelete.nonEmpty) {
logger.info(s"about to remove ${toDelete.length} deleted teams: ${toDelete.mkString(", ")}")
collection.bulkWrite(toDelete.map(team => DeleteOneModel(Filters.eq("name", team)))).toFuture()
.map(res => logger.info(s"removed ${res.getModifiedCount} deleted teams"))
} else Future.unit
} yield update

def findTeamSummaries(): Future[Seq[TeamSummary]] =
collection
.find()
.sort(ascending("name"))
.toFuture()
}
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class DataReloadScheduler @Inject()(

scheduleWithLock("Teams and Repos Reloader", config.dataReloadScheduler, lockService) {
for {
count <- persistingService.updateRepositories()
count <- persistingService.updateTeamsAndRepositories()
_ = logger.info(s"Finished updating Teams and Repos - $count records updated")
} yield ()
}
Expand All @@ -65,7 +65,7 @@ class DataReloadScheduler @Inject()(
lockService
.withLock {
logger.info(s"Starting mongo update")
persistingService.updateRepositories()
persistingService.updateTeamsAndRepositories()
}
.map(_.getOrElse(sys.error(s"Mongo is locked for ${lockService.lockId}")))
.map { _ =>
Expand Down
Loading

0 comments on commit 5006dda

Please sign in to comment.