diff --git a/.github/workflows/test-and-deploy.yml b/.github/workflows/test-and-deploy.yml index a256c31bcd..b967cd44d4 100644 --- a/.github/workflows/test-and-deploy.yml +++ b/.github/workflows/test-and-deploy.yml @@ -39,7 +39,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "temurin" - java-version: "17" + java-version: "21" - name: Setup cache uses: actions/cache@v2 with: @@ -60,7 +60,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "temurin" - java-version: "17" + java-version: "21" - name: Setup cache uses: actions/cache@v2 with: @@ -81,7 +81,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "temurin" - java-version: "17" + java-version: "21" - name: Setup cache uses: actions/cache@v2 with: diff --git a/.github/workflows/test-and-publish.yml b/.github/workflows/test-and-publish.yml index 822f79e4c0..3dd8660135 100644 --- a/.github/workflows/test-and-publish.yml +++ b/.github/workflows/test-and-publish.yml @@ -24,7 +24,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "temurin" - java-version: "17" + java-version: "21" - name: Setup cache uses: actions/cache@v2 with: @@ -45,7 +45,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "temurin" - java-version: "17" + java-version: "21" - name: Setup cache uses: actions/cache@v2 with: @@ -66,7 +66,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "temurin" - java-version: "17" + java-version: "21" - name: Setup cache uses: actions/cache@v2 with: @@ -93,7 +93,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: "temurin" - java-version: "17" + java-version: "21" - name: Setup cache uses: actions/cache@v2 with: diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/Project.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/Project.scala index 5d07fc7c4a..e632707dfc 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/Project.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/Project.scala @@ -31,7 +31,7 @@ import io.renku.graph.model.testentities import io.renku.tinytypes._ import io.renku.tinytypes.constraints._ -import java.net.{MalformedURLException, URL} +import java.net.URI final case class Project(entitiesProject: testentities.RenkuProject, id: GitLabId, @@ -120,7 +120,7 @@ object Project { addConstraint( check = url => (url endsWith ".git") && Validated - .catchOnly[MalformedURLException](new URL(url)) + .catchOnly[IllegalArgumentException](new URI(url).toURL) .isValid, message = url => s"$url is not a valid repository http url" ) diff --git a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/RemoteTriplesGenerator.scala b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/RemoteTriplesGenerator.scala index b18112c3a9..fa002bdc50 100644 --- a/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/RemoteTriplesGenerator.scala +++ b/acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/RemoteTriplesGenerator.scala @@ -32,7 +32,7 @@ import io.renku.graph.model._ import io.renku.graph.model.events.CommitId import io.renku.jsonld.JsonLD -import java.net.URL +import java.net.URI trait RemoteTriplesGenerator { self: ApplicationServices => @@ -122,9 +122,9 @@ trait RemoteTriplesGenerator { private object RemoteTriplesGeneratorWiremockInstance { private val logger = TestLogger() - private val remoteTriplesGeneratorUrl = new URL( + private val remoteTriplesGeneratorUrl = new URI( ConfigFactory.load().getString("services.remote-triples-generator.url") - ) + ).toURL private val port: Int = remoteTriplesGeneratorUrl.getPort diff --git a/build.sbt b/build.sbt index fde4e488b1..89dfa0b820 100644 --- a/build.sbt +++ b/build.sbt @@ -328,7 +328,7 @@ lazy val commonSettings = Seq( // Format: on organizationName := "Swiss Data Science Center (SDSC)", startYear := Some(java.time.LocalDate.now().getYear), - licenses += ("Apache-2.0", new URL("https://www.apache.org/licenses/LICENSE-2.0.txt")), + licenses += ("Apache-2.0", new URI("https://www.apache.org/licenses/LICENSE-2.0.txt").toURL), headerLicense := Some( HeaderLicense.Custom( s"""|Copyright ${java.time.LocalDate.now().getYear} Swiss Data Science Center (SDSC) diff --git a/commit-event-service/Dockerfile b/commit-event-service/Dockerfile index ab04632557..ff7cb28aab 100644 --- a/commit-event-service/Dockerfile +++ b/commit-event-service/Dockerfile @@ -1,7 +1,7 @@ # This is a multi-stage build, see reference: # https://docs.docker.com/develop/develop-images/multistage-build/ -FROM eclipse-temurin:17-jre-alpine as builder +FROM eclipse-temurin:21-jre-alpine as builder WORKDIR /work @@ -16,7 +16,7 @@ RUN export PATH="/usr/local/sbt/bin:$PATH" && \ sbt "project commit-event-service" stage && \ apk del .build-dependencies -FROM eclipse-temurin:17-jre-alpine +FROM eclipse-temurin:21-jre-alpine WORKDIR /opt/commit-event-service diff --git a/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/commands/DatasetInfoDeleteQuery.scala b/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/commands/DatasetInfoDeleteQuery.scala index 13164e6542..6fea9963c6 100644 --- a/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/commands/DatasetInfoDeleteQuery.scala +++ b/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/commands/DatasetInfoDeleteQuery.scala @@ -34,7 +34,6 @@ private object DatasetInfoDeleteQuery { Prefixes of (renku -> "renku", schema -> "schema"), sparql"""|DELETE { | GRAPH ${GraphClass.Datasets.id} { - | ?imageId ?imagePred ?imageObj. | ?linkId ?linkPred ?linkObj. | ?topSameAs ?dsPred ?dsObj. | } @@ -42,11 +41,6 @@ private object DatasetInfoDeleteQuery { |WHERE { | GRAPH ${GraphClass.Datasets.id} { | BIND (${topmostSameAs.asEntityId} AS ?topSameAs) - | - | OPTIONAL { - | ?topSameAs schema:image ?imageId. - | ?imageId ?imagePred ?imageObj. - | } | OPTIONAL { | ?topSameAs renku:datasetProjectLink ?linkId. diff --git a/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/commands/Encoders.scala b/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/commands/Encoders.scala index 676d75b74b..3cd8749241 100644 --- a/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/commands/Encoders.scala +++ b/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/commands/Encoders.scala @@ -22,7 +22,7 @@ package commands import DatasetSearchInfoOntology._ import Link.{ImportedDataset, OriginalDataset} import cats.syntax.all._ -import io.renku.entities.searchgraphs.toConcatValue +import io.renku.entities.searchgraphs.maybeTripleObject import io.renku.graph.model.Schemas.{rdf, renku} import io.renku.graph.model.images.Image import io.renku.graph.model.{datasets, persons} @@ -33,14 +33,6 @@ import io.renku.triplesstore.client.syntax._ private[datasets] object Encoders { - implicit val imageEncoder: QuadsEncoder[Image] = QuadsEncoder.instance { case Image(resourceId, uri, position) => - Set( - DatasetsQuad(resourceId, rdf / "type", Image.Ontology.typeClass.id), - DatasetsQuad(resourceId, Image.Ontology.contentUrlProperty.id, uri.asObject), - DatasetsQuad(resourceId, Image.Ontology.positionProperty.id, position.asObject) - ) - } - implicit val linkEncoder: QuadsEncoder[Link] = QuadsEncoder.instance { link => val typeQuads = link match { case _: OriginalDataset => @@ -59,7 +51,7 @@ private[datasets] object Encoders { implicit val projectsVisibilitiesConcatEncoder: QuadsEncoder[(datasets.TopmostSameAs, List[Link])] = QuadsEncoder.instance { case (topSameAs, links) => - toConcatValue[Link](links, link => s"${link.projectSlug.value}:${link.visibility.value}") + maybeTripleObject[Link](links, link => s"${link.projectSlug.value}:${link.visibility.value}") .map(DatasetsQuad(topSameAs, projectsVisibilitiesConcatProperty.id, _)) .toSet } @@ -69,7 +61,7 @@ private[datasets] object Encoders { DatasetsQuad(info.topmostSameAs, predicate, obj) def maybeConcatQuad[A](property: Property, values: List[A], toValue: A => String): Option[Quad] = - toConcatValue(values, toValue).map(searchInfoQuad(property, _)) + maybeTripleObject(values, toValue).map(searchInfoQuad(property, _)) val createdOrPublishedQuad = info.createdOrPublished match { case d: datasets.DateCreated => @@ -93,18 +85,9 @@ private[datasets] object Encoders { val maybeCreatorsNamesConcatQuad = maybeConcatQuad[persons.Name](creatorsNamesConcatProperty.id, info.creators.toList.map(_.name).distinct, _.value) - val keywordsQuads = info.keywords.toSet.map { (k: datasets.Keyword) => - searchInfoQuad(keywordsProperty.id, k.asObject) - } - val maybeKeywordsConcatQuad = maybeConcatQuad[datasets.Keyword](keywordsConcatProperty.id, info.keywords.distinct, _.value) - val imagesQuads = info.images.toSet.flatMap { (i: Image) => - i.asQuads + - searchInfoQuad(imageProperty, i.resourceId.asEntityId) - } - val maybeImagesConcatQuad = maybeConcatQuad[Image](imagesConcatProperty.id, info.images, @@ -129,6 +112,6 @@ private[datasets] object Encoders { maybeCreatorsNamesConcatQuad, maybeKeywordsConcatQuad, maybeImagesConcatQuad - ).flatten ++ projectsVisibilitiesConcatQuads ++ creatorsQuads ++ keywordsQuads ++ imagesQuads ++ linksQuads + ).flatten ++ projectsVisibilitiesConcatQuads ++ creatorsQuads ++ linksQuads } } diff --git a/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/ontology.scala b/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/ontology.scala index 5d21bab46c..872e71f556 100644 --- a/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/ontology.scala +++ b/entities-search/src/main/scala/io/renku/entities/searchgraphs/datasets/ontology.scala @@ -20,7 +20,6 @@ package io.renku.entities.searchgraphs.datasets import io.renku.graph.model.Schemas.{xsd, _} import io.renku.graph.model.entities.{Dataset, Person, Project} -import io.renku.graph.model.images.Image import io.renku.jsonld.Property import io.renku.jsonld.ontology._ @@ -31,12 +30,10 @@ object DatasetSearchInfoOntology { val dateCreatedProperty: DataProperty.Def = Dataset.Ontology.dateCreatedProperty val datePublishedProperty: DataProperty.Def = Dataset.Ontology.datePublishedProperty val dateModifiedProperty: DataProperty.Def = DataProperty(schema / "dateModified", xsd / "dateTime") - val keywordsProperty: DataProperty.Def = Dataset.Ontology.keywordsProperty val keywordsConcatProperty: DataProperty.Def = DataProperty(renku / "keywordsConcat", xsd / "string") val descriptionProperty: DataProperty.Def = Dataset.Ontology.descriptionProperty val creatorProperty: Property = Dataset.Ontology.creator val creatorsNamesConcatProperty: DataProperty.Def = DataProperty(renku / "creatorsNamesConcat", xsd / "string") - val imageProperty: Property = Dataset.Ontology.image val imagesConcatProperty: DataProperty.Def = DataProperty(renku / "imagesConcat", xsd / "string") val linkProperty: Property = renku / "datasetProjectLink" val projectsVisibilitiesConcatProperty: DataProperty.Def = @@ -46,7 +43,6 @@ object DatasetSearchInfoOntology { Class(renku / "DiscoverableDataset"), ObjectProperties( ObjectProperty(creatorProperty, Person.Ontology.typeDef), - ObjectProperty(imageProperty, Image.Ontology.typeDef), ObjectProperty(linkProperty, LinkOntology.typeDef) ), DataProperties( @@ -55,7 +51,6 @@ object DatasetSearchInfoOntology { dateCreatedProperty, datePublishedProperty, dateModifiedProperty, - keywordsProperty, keywordsConcatProperty, descriptionProperty, creatorsNamesConcatProperty, diff --git a/entities-search/src/main/scala/io/renku/entities/searchgraphs/package.scala b/entities-search/src/main/scala/io/renku/entities/searchgraphs/package.scala index d75d1cd296..084edaa834 100644 --- a/entities-search/src/main/scala/io/renku/entities/searchgraphs/package.scala +++ b/entities-search/src/main/scala/io/renku/entities/searchgraphs/package.scala @@ -24,7 +24,7 @@ import io.renku.triplesstore.client.syntax._ package object searchgraphs { val concatSeparator: Char = ';' - private[searchgraphs] def toConcatValue[A](values: List[A], toValue: A => String): Option[TripleObject] = + private[searchgraphs] def maybeTripleObject[A](values: List[A], toValue: A => String): Option[TripleObject] = values match { case Nil => Option.empty[TripleObject] case vls => diff --git a/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/Encoders.scala b/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/Encoders.scala index b1a0d8ffbb..d571566416 100644 --- a/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/Encoders.scala +++ b/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/Encoders.scala @@ -30,23 +30,24 @@ import io.renku.jsonld.syntax._ import io.renku.triplesstore.client.model.{Quad, QuadsEncoder, TripleObject} import io.renku.triplesstore.client.syntax._ -private object Encoders { +object Encoders { - implicit val imageEncoder: QuadsEncoder[Image] = QuadsEncoder.instance { case Image(resourceId, uri, position) => - Set( - ProjectsQuad(resourceId, rdf / "type", Image.Ontology.typeClass.id), - ProjectsQuad(resourceId, Image.Ontology.contentUrlProperty.id, uri.asObject), - ProjectsQuad(resourceId, Image.Ontology.positionProperty.id, position.asObject) - ) - } + def maybeKeywordsObject(keywords: List[projects.Keyword]): Option[TripleObject] = + maybeTripleObject[projects.Keyword](keywords.distinct, _.value) + + def maybeKeywordsQuad(id: projects.ResourceId, keywords: List[projects.Keyword]): Option[Quad] = + maybeKeywordsObject(keywords).map(ProjectsQuad(id, keywordsConcatProperty.id, _)) + + def maybeImagesObject(images: List[Image]): Option[TripleObject] = + maybeTripleObject[Image](images, image => s"${image.position.value}:${image.uri.value}") + + def maybeImagesQuad(id: projects.ResourceId, images: List[Image]): Option[Quad] = + maybeImagesObject(images).map(ProjectsQuad(id, imagesConcatProperty.id, _)) - implicit val searchInfoEncoder: QuadsEncoder[ProjectSearchInfo] = QuadsEncoder.instance { info => + private[commands] implicit val searchInfoEncoder: QuadsEncoder[ProjectSearchInfo] = QuadsEncoder.instance { info => def searchInfoQuad(predicate: Property, obj: TripleObject): Quad = ProjectsQuad(info.id, predicate, obj) - def maybeConcatQuad[A](property: Property, values: List[A], toValue: A => String): Option[Quad] = - toConcatValue(values, toValue).map(searchInfoQuad(property, _)) - val maybeDescriptionQuad = info.maybeDescription.map { d => searchInfoQuad(descriptionProperty.id, d.asObject) } @@ -55,22 +56,9 @@ private object Encoders { searchInfoQuad(creatorProperty, resourceId.asEntityId) } - val keywordsQuads = info.keywords.toSet.map { (k: projects.Keyword) => - searchInfoQuad(keywordsProperty.id, k.asObject) - } - - val maybeKeywordsConcatQuad = - maybeConcatQuad[projects.Keyword](keywordsConcatProperty.id, info.keywords.distinct, _.value) - - val imagesQuads = info.images.toSet.flatMap { (i: Image) => - i.asQuads + searchInfoQuad(imageProperty, i.resourceId.asEntityId) - } + val maybeKeywordsConcatQuad = maybeKeywordsQuad(info.id, info.keywords) - val maybeImagesConcatQuad = - maybeConcatQuad[Image](imagesConcatProperty.id, - info.images, - image => s"${image.position.value}:${image.uri.value}" - ) + val maybeImagesConcatQuad = maybeImagesQuad(info.id, info.images) Set( searchInfoQuad(rdf / "type", typeDef.clazz.id).some, @@ -83,6 +71,6 @@ private object Encoders { maybeKeywordsConcatQuad, maybeImagesConcatQuad, maybeDescriptionQuad - ).flatten ++ creatorQuads ++ keywordsQuads ++ imagesQuads + ).flatten ++ creatorQuads } } diff --git a/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/ProjectInfoDeleteQuery.scala b/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/ProjectInfoDeleteQuery.scala index 52545180ef..7570d5cbfd 100644 --- a/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/ProjectInfoDeleteQuery.scala +++ b/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/commands/ProjectInfoDeleteQuery.scala @@ -34,19 +34,12 @@ private[projects] object ProjectInfoDeleteQuery { Prefixes of schema -> "schema", sparql"""|DELETE { | GRAPH ${GraphClass.Projects.id} { - | ?imageId ?imagePred ?imageObj. | ?projId ?projPred ?projObj. | } |} |WHERE { | GRAPH ${GraphClass.Projects.id} { | BIND (${projectId.asEntityId} AS ?projId) - | - | OPTIONAL { - | ?projId schema:image ?imageId. - | ?imageId ?imagePred ?imageObj. - | } - | | ?projId ?projPred ?projObj. | } |} diff --git a/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/ontology.scala b/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/ontology.scala index 761c1bbcf5..9262cf5a1f 100644 --- a/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/ontology.scala +++ b/entities-search/src/main/scala/io/renku/entities/searchgraphs/projects/ontology.scala @@ -19,8 +19,7 @@ package io.renku.entities.searchgraphs.projects import io.renku.graph.model.Schemas.renku -import io.renku.graph.model.entities.{Dataset, Person, Project} -import io.renku.graph.model.images.Image +import io.renku.graph.model.entities.{Person, Project} import io.renku.jsonld.Property import io.renku.jsonld.ontology._ @@ -32,18 +31,15 @@ object ProjectSearchInfoOntology { val visibilityProperty: DataProperty.Def = Project.Ontology.visibilityProperty val dateCreatedProperty: DataProperty.Def = Project.Ontology.dateCreatedProperty val dateModifiedProperty: DataProperty.Def = Project.Ontology.dateModifiedProperty - val keywordsProperty: DataProperty.Def = Project.Ontology.keywordsProperty val keywordsConcatProperty: DataProperty.Def = DataProperty(renku / "keywordsConcat", xsd / "string") val descriptionProperty: DataProperty.Def = Project.Ontology.descriptionProperty val creatorProperty: Property = Project.Ontology.creator - val imageProperty: Property = Project.Ontology.image val imagesConcatProperty: DataProperty.Def = DataProperty(renku / "imagesConcat", xsd / "string") lazy val typeDef: Type = Type.Def( Class(renku / "DiscoverableProject"), ObjectProperties( - ObjectProperty(creatorProperty, Person.Ontology.typeDef), - ObjectProperty(imageProperty, Image.Ontology.typeDef) + ObjectProperty(creatorProperty, Person.Ontology.typeDef) ), DataProperties( nameProperty, @@ -52,25 +48,9 @@ object ProjectSearchInfoOntology { visibilityProperty, dateCreatedProperty, dateModifiedProperty, - keywordsProperty, keywordsConcatProperty, descriptionProperty, imagesConcatProperty ) ) } - -object LinkOntology { - - val project: Property = renku / "project" - val dataset: Property = renku / "dataset" - - lazy val typeDef: Type = Type.Def( - Class(renku / "DatasetProjectLink"), - ObjectProperties( - ObjectProperty(project, Project.Ontology.typeDef), - ObjectProperty(dataset, Dataset.Ontology.typeDef) - ), - DataProperties() - ) -} diff --git a/entities-search/src/test/scala/io/renku/entities/searchgraphs/datasets/commands/EncodersSpec.scala b/entities-search/src/test/scala/io/renku/entities/searchgraphs/datasets/commands/EncodersSpec.scala index cec70397f5..d57458ea90 100644 --- a/entities-search/src/test/scala/io/renku/entities/searchgraphs/datasets/commands/EncodersSpec.scala +++ b/entities-search/src/test/scala/io/renku/entities/searchgraphs/datasets/commands/EncodersSpec.scala @@ -24,12 +24,9 @@ import Generators.{datasetSearchInfoObjects, importedDatasetLinkObjectsGen, orig import cats.syntax.all._ import io.renku.entities.searchgraphs.concatSeparator import io.renku.generators.Generators.Implicits._ -import io.renku.generators.Generators.positiveInts -import io.renku.generators.jsonld.JsonLDGenerators.entityIds -import io.renku.graph.model.GraphModelGenerators.{datasetTopmostSameAs, imageUris} -import io.renku.graph.model.Schemas.{rdf, renku, schema} +import io.renku.graph.model.GraphModelGenerators.datasetTopmostSameAs +import io.renku.graph.model.Schemas.{rdf, renku} import io.renku.graph.model.datasets -import io.renku.graph.model.images.{Image, ImagePosition, ImageResourceId} import io.renku.jsonld.syntax._ import io.renku.triplesstore.client.model.Quad import io.renku.triplesstore.client.syntax._ @@ -41,26 +38,6 @@ class EncodersSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope import io.renku.entities.searchgraphs.datasets.commands.Encoders._ - "imageEncoder" should { - - "turn an Image object into a Set of relevant Quads" in { - - val image = { - for { - id <- entityIds.toGeneratorOf(id => ImageResourceId(id.toString)) - uri <- imageUris - position <- positiveInts().toGeneratorOf(p => ImagePosition(p.value)) - } yield Image(id, uri, position) - }.generateOne - - image.asQuads shouldBe Set( - DatasetsQuad(image.resourceId, rdf / "type", schema / "ImageObject"), - DatasetsQuad(image.resourceId, Image.Ontology.contentUrlProperty.id, image.uri.asObject), - DatasetsQuad(image.resourceId, Image.Ontology.positionProperty.id, image.position.asObject) - ) - } - } - "linkEncoder" should { "turn a OriginalDataset object into a Set of relevant Quads" in { @@ -105,11 +82,9 @@ class EncodersSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope ) ++ maybeDateModifiedToQuad(searchInfo.topmostSameAs)(searchInfo.maybeDateModified) ++ creatorsToQuads(searchInfo) ++ - keywordsToQuads(searchInfo) ++ maybeDescToQuad(searchInfo.topmostSameAs)(searchInfo.maybeDescription) ++ maybeKeywordsConcatToQuad(searchInfo).toSet ++ maybeImagesConcatToQuad(searchInfo).toSet ++ - imagesToQuads(searchInfo) ++ linksToQuads(searchInfo) } } @@ -139,11 +114,6 @@ class EncodersSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope private def creatorsNamesConcat(searchInfo: DatasetSearchInfo) = searchInfo.creators.map(_.name.value).intercalate(concatSeparator.toString).asTripleObject - private def keywordsToQuads(searchInfo: DatasetSearchInfo): Set[Quad] = - searchInfo.keywords - .map(k => DatasetsQuad(searchInfo.topmostSameAs, keywordsProperty.id, k.asObject)) - .toSet - private def maybeKeywordsConcatToQuad(searchInfo: DatasetSearchInfo): Option[Quad] = searchInfo.keywords match { case Nil => Option.empty[Quad] @@ -171,12 +141,6 @@ class EncodersSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope ).some } - private def imagesToQuads(searchInfo: DatasetSearchInfo): Set[Quad] = - searchInfo.images - .map(i => i.asQuads + DatasetsQuad(searchInfo.topmostSameAs, imageProperty, i.resourceId.asEntityId)) - .toSet - .flatten - private def linksToQuads(searchInfo: DatasetSearchInfo): Set[Quad] = searchInfo.links .map(l => l.asQuads + DatasetsQuad(searchInfo.topmostSameAs, linkProperty, l.resourceId.asEntityId)) diff --git a/entities-search/src/test/scala/io/renku/entities/searchgraphs/projects/commands/EncodersSpec.scala b/entities-search/src/test/scala/io/renku/entities/searchgraphs/projects/commands/EncodersSpec.scala index b7435827e1..d240b11637 100644 --- a/entities-search/src/test/scala/io/renku/entities/searchgraphs/projects/commands/EncodersSpec.scala +++ b/entities-search/src/test/scala/io/renku/entities/searchgraphs/projects/commands/EncodersSpec.scala @@ -23,12 +23,7 @@ package commands import Generators._ import ProjectSearchInfoOntology._ import cats.syntax.all._ -import io.renku.generators.Generators.Implicits._ -import io.renku.generators.Generators.positiveInts -import io.renku.generators.jsonld.JsonLDGenerators.entityIds -import io.renku.graph.model.GraphModelGenerators.imageUris -import io.renku.graph.model.Schemas.{rdf, renku, schema} -import io.renku.graph.model.images.{Image, ImagePosition, ImageResourceId} +import io.renku.graph.model.Schemas.{rdf, renku} import io.renku.graph.model.{persons, projects} import io.renku.jsonld.syntax._ import io.renku.triplesstore.client.model.Quad @@ -41,26 +36,6 @@ class EncodersSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope import Encoders._ - "imageEncoder" should { - - "turn an Image object into a Set of relevant Quads" in { - - val image = { - for { - id <- entityIds.toGeneratorOf(id => ImageResourceId(id.toString)) - uri <- imageUris - position <- positiveInts().toGeneratorOf(p => ImagePosition(p.value)) - } yield Image(id, uri, position) - }.generateOne - - image.asQuads shouldBe Set( - ProjectsQuad(image.resourceId, rdf / "type", schema / "ImageObject"), - ProjectsQuad(image.resourceId, Image.Ontology.contentUrlProperty.id, image.uri.asObject), - ProjectsQuad(image.resourceId, Image.Ontology.positionProperty.id, image.position.asObject) - ) - } - } - "searchInfoEncoder" should { "turn a SearchInfo object into a Set of relevant Quads" in { @@ -76,11 +51,9 @@ class EncodersSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope ProjectsQuad(searchInfo.id, dateModifiedProperty.id, searchInfo.dateModified.asObject) ) ++ creatorToQuads(searchInfo) ++ - keywordsToQuads(searchInfo) ++ maybeDescToQuad(searchInfo) ++ maybeKeywordsConcatToQuad(searchInfo).toSet ++ - maybeImagesConcatToQuad(searchInfo).toSet ++ - imagesToQuads(searchInfo) + maybeImagesConcatToQuad(searchInfo).toSet } } @@ -91,11 +64,6 @@ class EncodersSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope ProjectsQuad(searchInfo.id, creatorProperty, resourceId.asEntityId) } - private def keywordsToQuads(searchInfo: ProjectSearchInfo): Set[Quad] = - searchInfo.keywords - .map(k => ProjectsQuad(searchInfo.id, keywordsProperty.id, k.asObject)) - .toSet - private def maybeDescToQuad(searchInfo: ProjectSearchInfo): Set[Quad] = searchInfo.maybeDescription.toSet.map { (d: projects.Description) => ProjectsQuad(searchInfo.id, descriptionProperty.id, d.asObject) @@ -117,10 +85,4 @@ class EncodersSpec extends AnyWordSpec with should.Matchers with ScalaCheckPrope images.map(i => s"${i.position}:${i.uri}").mkString(concatSeparator.toString).asTripleObject ).some } - - private def imagesToQuads(searchInfo: ProjectSearchInfo): Set[Quad] = - searchInfo.images - .map(i => i.asQuads + ProjectsQuad(searchInfo.id, imageProperty, i.resourceId.asEntityId)) - .toSet - .flatten } diff --git a/event-log/Dockerfile b/event-log/Dockerfile index 84238f42a2..b8eda91af6 100644 --- a/event-log/Dockerfile +++ b/event-log/Dockerfile @@ -1,7 +1,7 @@ # This is a multi-stage build, see reference: # https://docs.docker.com/develop/develop-images/multistage-build/ -FROM eclipse-temurin:17-jre-alpine as builder +FROM eclipse-temurin:21-jre-alpine as builder WORKDIR /work @@ -16,7 +16,7 @@ RUN export PATH="/usr/local/sbt/bin:$PATH" && \ sbt "project event-log" stage && \ apk del .build-dependencies -FROM eclipse-temurin:17-jre-alpine +FROM eclipse-temurin:21-jre-alpine WORKDIR /opt/event-log diff --git a/graph-commons/src/main/scala/io/renku/events/Subscription.scala b/graph-commons/src/main/scala/io/renku/events/Subscription.scala index 5dfc204dc5..4387c41724 100644 --- a/graph-commons/src/main/scala/io/renku/events/Subscription.scala +++ b/graph-commons/src/main/scala/io/renku/events/Subscription.scala @@ -28,7 +28,7 @@ import io.renku.tinytypes.constraints.{NonBlank, PositiveInt, Url} import Subscription._ import io.renku.tinytypes.json.TinyTypeDecoders.{intDecoder, stringDecoder} -import java.net.URL +import java.net.URI trait Subscription { val categoryName: CategoryName @@ -64,7 +64,7 @@ object Subscription { implicit val microserviceBaseUrlConverter: TinyTypeConverter[SubscriberUrl, MicroserviceBaseUrl] = (subscriberUrl: SubscriberUrl) => { - val url = new URL(subscriberUrl.value) + val url = new URI(subscriberUrl.value).toURL MicroserviceBaseUrl(s"${url.getProtocol}://${url.getHost}:${url.getPort}").asRight[IllegalArgumentException] } } diff --git a/graph-commons/src/test/scala/io/renku/triplesstore/InMemoryJena.scala b/graph-commons/src/test/scala/io/renku/triplesstore/InMemoryJena.scala index b04279e9d6..408f090f93 100644 --- a/graph-commons/src/test/scala/io/renku/triplesstore/InMemoryJena.scala +++ b/graph-commons/src/test/scala/io/renku/triplesstore/InMemoryJena.scala @@ -105,6 +105,9 @@ trait InMemoryJena { def runUpdate(on: DatasetName, query: SparqlQuery): IO[Unit] = queryRunnerFor(on).flatMap(_.runUpdate(query)) + def runUpdates(on: DatasetName, queries: List[SparqlQuery]): IO[Unit] = + queries.map(runUpdate(on, _)).sequence.void + def triplesCount(on: DatasetName): IO[Long] = queryRunnerFor(on) .flatMap( diff --git a/knowledge-graph/Dockerfile b/knowledge-graph/Dockerfile index c53c5498f3..4c620fcd50 100644 --- a/knowledge-graph/Dockerfile +++ b/knowledge-graph/Dockerfile @@ -1,7 +1,7 @@ # This is a multi-stage build, see reference: # https://docs.docker.com/develop/develop-images/multistage-build/ -FROM eclipse-temurin:17-jre-alpine as builder +FROM eclipse-temurin:21-jre-alpine as builder WORKDIR /work @@ -16,7 +16,7 @@ RUN export PATH="/usr/local/sbt/bin:$PATH" && \ sbt "project knowledge-graph" stage && \ apk del .build-dependencies -FROM eclipse-temurin:17-jre-alpine +FROM eclipse-temurin:21-jre-alpine WORKDIR /opt/knowledge-graph diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4a66f31101..34738c6402 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -18,8 +18,8 @@ object Dependencies { val http4s = "0.23.23" val http4sEmber = "0.23.23" val http4sPrometheus = "0.24.6" - val ip4s = "3.3.0" - val jsonld4s = "0.12.0" + val ip4s = "3.4.0" + val jsonld4s = "0.13.0" val log4cats = "2.6.0" val log4jCore = "2.21.1" val logback = "1.4.11" @@ -33,12 +33,12 @@ object Dependencies { val scalamock = "5.2.0" val scalatest = "3.2.17" val scalatestScalacheck = "3.2.14.0" - val sentryLogback = "6.32.0" + val sentryLogback = "6.33.0" val skunk = "0.6.1" val swaggerParser = "2.1.18" val testContainersScala = "0.41.0" val widoco = "1.4.20" - val wiremock = "3.2.0" + val wiremock = "3.3.1" } val ip4s = Seq( diff --git a/renku-model-tiny-types/src/main/scala/io/renku/graph/model/projects.scala b/renku-model-tiny-types/src/main/scala/io/renku/graph/model/projects.scala index fcec508a49..d50959b087 100644 --- a/renku-model-tiny-types/src/main/scala/io/renku/graph/model/projects.scala +++ b/renku-model-tiny-types/src/main/scala/io/renku/graph/model/projects.scala @@ -29,7 +29,7 @@ import io.renku.jsonld.{EntityId, JsonLDDecoder, JsonLDEncoder} import io.renku.tinytypes._ import io.renku.tinytypes.constraints._ -import java.net.{MalformedURLException, URL} +import java.net.URI import java.time.Instant import java.time.temporal.ChronoUnit @@ -226,7 +226,7 @@ object projects { addConstraint( check = url => (url endsWith ".git") && Validated - .catchOnly[MalformedURLException](new URL(url)) + .catchOnly[IllegalArgumentException](new URI(url).toURL) .isValid, message = url => s"$url is not a valid repository http url" ) diff --git a/tiny-types/src/main/scala/io/renku/tinytypes/constraints/Url.scala b/tiny-types/src/main/scala/io/renku/tinytypes/constraints/Url.scala index 453fe4007c..41dc130d78 100644 --- a/tiny-types/src/main/scala/io/renku/tinytypes/constraints/Url.scala +++ b/tiny-types/src/main/scala/io/renku/tinytypes/constraints/Url.scala @@ -20,13 +20,13 @@ package io.renku.tinytypes.constraints import io.renku.tinytypes._ -import java.net.URL +import java.net.URI import scala.language.implicitConversions import scala.util.Try trait Url[TT <: TinyType { type V = String }] extends Constraints[TT] { addConstraint( - check = url => Try(new URL(url)).isSuccess, + check = url => Try(new URI(url).toURL).isSuccess, message = (url: String) => s"Cannot instantiate $typeName with '$url'" ) } diff --git a/token-repository/Dockerfile b/token-repository/Dockerfile index ca7696a668..6ecf69d236 100644 --- a/token-repository/Dockerfile +++ b/token-repository/Dockerfile @@ -1,7 +1,7 @@ # This is a multi-stage build, see reference: # https://docs.docker.com/develop/develop-images/multistage-build/ -FROM eclipse-temurin:17-jre-alpine as builder +FROM eclipse-temurin:21-jre-alpine as builder WORKDIR /work @@ -16,7 +16,7 @@ RUN export PATH="/usr/local/sbt/bin:$PATH" && \ sbt "project token-repository" stage && \ apk del .build-dependencies -FROM eclipse-temurin:17-jre-alpine +FROM eclipse-temurin:21-jre-alpine WORKDIR /opt/token-repository diff --git a/triples-generator/Dockerfile b/triples-generator/Dockerfile index ca5966accf..f001d77e33 100644 --- a/triples-generator/Dockerfile +++ b/triples-generator/Dockerfile @@ -1,7 +1,7 @@ # This is a multi-stage build, see reference: # https://docs.docker.com/develop/develop-images/multistage-build/ -FROM eclipse-temurin:17-jre-alpine as builder +FROM eclipse-temurin:21-jre-alpine as builder WORKDIR /work @@ -16,7 +16,7 @@ RUN export PATH="/usr/local/sbt/bin:$PATH" && \ sbt "project triples-generator" stage && \ apk del .build-dependencies -FROM eclipse-temurin:17-jre-alpine +FROM eclipse-temurin:21-jre-alpine WORKDIR /opt/triples-generator diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/awaitinggeneration/triplesgeneration/renkulog/Commands.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/awaitinggeneration/triplesgeneration/renkulog/Commands.scala index 1a30d7d853..912954db21 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/awaitinggeneration/triplesgeneration/renkulog/Commands.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/awaitinggeneration/triplesgeneration/renkulog/Commands.scala @@ -54,7 +54,7 @@ private object Commands { class GitLabRepoUrlFinderImpl[F[_]: MonadThrow](gitLabUrl: GitLabUrl) extends GitLabRepoUrlFinder[F] { - import java.net.URL + import java.net.URI override def findRepositoryUrl(projectSlug: projects.Slug)(implicit maybeAccessToken: Option[AccessToken] @@ -72,7 +72,7 @@ private object Commands { MonadThrow[F].fromEither { ServiceUrl.from { val url = gitLabUrl.value - val protocol = new URL(url).getProtocol + val protocol = new URI(url).toURL.getProtocol val serviceWithToken = url.replace(s"$protocol://", s"$protocol://$urlTokenPart") s"$serviceWithToken/$projectSlug.git" } diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/syncrepometadata/processor/UpdateCommandsCalculator.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/syncrepometadata/processor/UpdateCommandsCalculator.scala index 59711cee1c..a4705ba880 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/syncrepometadata/processor/UpdateCommandsCalculator.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/syncrepometadata/processor/UpdateCommandsCalculator.scala @@ -22,6 +22,8 @@ package processor import cats.Monad import cats.syntax.all._ import eu.timepit.refined.auto._ +import io.renku.entities.searchgraphs.projects.ProjectSearchInfoOntology.{imagesConcatProperty, keywordsConcatProperty} +import io.renku.entities.searchgraphs.projects.commands.Encoders.{maybeImagesQuad, maybeKeywordsQuad} import io.renku.eventlog.api.events.StatusChangeEvent import io.renku.graph.model.Schemas.{rdf, schema} import io.renku.graph.model.images.Image @@ -225,14 +227,17 @@ private class UpdateCommandsCalculatorImpl[F[_]: Monad: Logger](newValuesCalcula SparqlQuery.ofUnsafe( show"$categoryName: update keywords in Projects", Prefixes of schema -> "schema", - sparql"""|DELETE { GRAPH ${GraphClass.Projects.id} { ?id schema:keywords ?keyword } } - |INSERT { GRAPH ${GraphClass.Projects.id} { - | ${newValue.map(k => fr"""?id schema:keywords ${k.asObject}.""").toList.intercalate(fr"\n")} - |} } + sparql"""|DELETE { GRAPH ${GraphClass.Projects.id} { + | ?id ${keywordsConcatProperty.id} ?keywords + | } + |} + |INSERT { + | ${maybeKeywordsQuad(id, newValue.toList)} + |} |WHERE { | BIND (${id.asEntityId} AS ?id) | GRAPH ${GraphClass.Projects.id} { - | OPTIONAL { ?id schema:keywords ?keyword } + | OPTIONAL { ?id ${keywordsConcatProperty.id} ?keywords } | } |}""".stripMargin ) @@ -266,21 +271,20 @@ private class UpdateCommandsCalculatorImpl[F[_]: Monad: Logger](newValuesCalcula private def imagesInProjectsUpdate(id: projects.ResourceId, newValue: List[Image]) = SparqlQuery.ofUnsafe( - show"$categoryName: update keywords in Projects", + show"$categoryName: update images in Projects", Prefixes of (rdf -> "rdf", schema -> "schema"), sparql"""|DELETE { GRAPH ${GraphClass.Projects.id} { - | ?id schema:image ?imageId. - | ?imageId ?p ?o - |} } - |INSERT { GRAPH ${GraphClass.Projects.id} { - | ${newValue.flatMap(toTriple).intercalate(fr"\n ")} - |} } + | ?id ${imagesConcatProperty.id} ?images + | } + |} + |INSERT { + | ${maybeImagesQuad(id, newValue)} + |} |WHERE { | BIND (${id.asEntityId} AS ?id) | GRAPH ${GraphClass.Projects.id} { | OPTIONAL { - | ?id schema:image ?imageId. - | ?imageId ?p ?o + | ?id ${imagesConcatProperty.id} ?images. | } | } |}""".stripMargin diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphImagesRemover.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphImagesRemover.scala new file mode 100644 index 0000000000..8805fcdb98 --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphImagesRemover.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest +package migrations + +import cats.effect.Async +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.graph.model.GraphClass +import io.renku.graph.model.Schemas.schema +import io.renku.metrics.MetricsRegistry +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.client.syntax._ +import io.renku.triplesstore.{SparqlQuery, SparqlQueryTimeRecorder} +import migrations.tooling.RegisteredUpdateQueryMigration +import org.typelevel.log4cats.Logger + +private object DatasetsGraphImagesRemover { + + private lazy val name = Migration.Name("Datasets graph images remover") + + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder: MetricsRegistry]: F[Migration[F]] = + RegisteredUpdateQueryMigration[F](name, query).widen + + private[migrations] lazy val query = SparqlQuery.of( + name.asRefined, + Prefixes of schema -> "schema", + sparql"""|DELETE { + | GRAPH ${GraphClass.Datasets.id} { + | ?topSameAs schema:image ?imageId. + | ?imageId ?imagePred ?imageObj. + | } + |} + |WHERE { + | GRAPH ${GraphClass.Datasets.id} { + | OPTIONAL { + | ?topSameAs schema:image ?imageId. + | ?imageId ?imagePred ?imageObj. + | } + | } + |} + |""".stripMargin + ) +} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphKeywordsRemover.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphKeywordsRemover.scala new file mode 100644 index 0000000000..0f0c38a0df --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphKeywordsRemover.scala @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest +package migrations + +import cats.effect.Async +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.graph.model.GraphClass +import io.renku.graph.model.Schemas.schema +import io.renku.metrics.MetricsRegistry +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.client.syntax._ +import io.renku.triplesstore.{SparqlQuery, SparqlQueryTimeRecorder} +import migrations.tooling.RegisteredUpdateQueryMigration +import org.typelevel.log4cats.Logger + +private object DatasetsGraphKeywordsRemover { + + private lazy val name = Migration.Name("Datasets graph keywords remover") + + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder: MetricsRegistry]: F[Migration[F]] = + RegisteredUpdateQueryMigration[F](name, query).widen + + private[migrations] lazy val query = SparqlQuery.of( + name.asRefined, + Prefixes of schema -> "schema", + sparql"""|DELETE { + | GRAPH ${GraphClass.Datasets.id} { + | ?topSameAs schema:keywords ?keywords. + | } + |} + |WHERE { + | GRAPH ${GraphClass.Datasets.id} { + | OPTIONAL { + | ?topSameAs schema:keywords ?keywords + | } + | } + |} + |""".stripMargin + ) +} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala index e72637ca87..0bec2318cf 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala @@ -60,6 +60,10 @@ private[tsmigrationrequest] object Migrations { datasetsGraphCreatorsFlattener <- DatasetsGraphCreatorsFlattener[F] datasetsGraphSlugsVisibsFlattener <- DatasetsGraphSlugsVisibilitiesFlattener[F] projectMembersRemover <- ProjectMembersRemover[F] + datasetsGraphKeywordsRemover <- DatasetsGraphKeywordsRemover[F] + datasetsGraphImagesRemover <- DatasetsGraphImagesRemover[F] + projectsGraphKeywordsRemover <- ProjectsGraphKeywordsRemover[F] + projectsGraphImagesRemover <- ProjectsGraphImagesRemover[F] migrations <- validateNames( datasetsCreator, datasetsRemover, @@ -86,7 +90,11 @@ private[tsmigrationrequest] object Migrations { datasetsGraphImagesFlattener, datasetsGraphCreatorsFlattener, datasetsGraphSlugsVisibsFlattener, - projectMembersRemover + projectMembersRemover, + datasetsGraphKeywordsRemover, + datasetsGraphImagesRemover, + projectsGraphKeywordsRemover, + projectsGraphImagesRemover ) } yield migrations diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphImagesRemover.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphImagesRemover.scala new file mode 100644 index 0000000000..a78b5d921f --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphImagesRemover.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest +package migrations + +import cats.effect.Async +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.graph.model.GraphClass +import io.renku.graph.model.Schemas.schema +import io.renku.metrics.MetricsRegistry +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.client.syntax._ +import io.renku.triplesstore.{SparqlQuery, SparqlQueryTimeRecorder} +import migrations.tooling.RegisteredUpdateQueryMigration +import org.typelevel.log4cats.Logger + +private object ProjectsGraphImagesRemover { + + private lazy val name = Migration.Name("Projects graph images remover") + + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder: MetricsRegistry]: F[Migration[F]] = + RegisteredUpdateQueryMigration[F](name, query).widen + + private[migrations] lazy val query = SparqlQuery.of( + name.asRefined, + Prefixes of schema -> "schema", + sparql"""|DELETE { + | GRAPH ${GraphClass.Projects.id} { + | ?projectId schema:image ?imageId. + | ?imageId ?imagePred ?imageObj. + | } + |} + |WHERE { + | GRAPH ${GraphClass.Projects.id} { + | OPTIONAL { + | ?projectId schema:image ?imageId. + | ?imageId ?imagePred ?imageObj. + | } + | } + |} + |""".stripMargin + ) +} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphKeywordsRemover.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphKeywordsRemover.scala new file mode 100644 index 0000000000..33a1d4f386 --- /dev/null +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphKeywordsRemover.scala @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest +package migrations + +import cats.effect.Async +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.graph.model.GraphClass +import io.renku.graph.model.Schemas.schema +import io.renku.metrics.MetricsRegistry +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore.client.syntax._ +import io.renku.triplesstore.{SparqlQuery, SparqlQueryTimeRecorder} +import migrations.tooling.RegisteredUpdateQueryMigration +import org.typelevel.log4cats.Logger + +private object ProjectsGraphKeywordsRemover { + + private lazy val name = Migration.Name("Projects graph keywords remover") + + def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder: MetricsRegistry]: F[Migration[F]] = + RegisteredUpdateQueryMigration[F](name, query).widen + + private[migrations] lazy val query = SparqlQuery.of( + name.asRefined, + Prefixes of schema -> "schema", + sparql"""|DELETE { + | GRAPH ${GraphClass.Projects.id} { + | ?projectId schema:keywords ?keywords + | } + |} + |WHERE { + | GRAPH ${GraphClass.Projects.id} { + | OPTIONAL { + | ?projectId schema:keywords ?keywords + | } + | } + |} + |""".stripMargin + ) +} diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/projects/update/UpdateQueriesCalculator.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/projects/update/UpdateQueriesCalculator.scala index f87c69bb11..44a11b625d 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/projects/update/UpdateQueriesCalculator.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/projects/update/UpdateQueriesCalculator.scala @@ -21,6 +21,8 @@ package io.renku.triplesgenerator.projects.update import cats.syntax.all._ import cats.{Applicative, MonadThrow} import eu.timepit.refined.auto._ +import io.renku.entities.searchgraphs.projects.ProjectSearchInfoOntology.{imagesConcatProperty, keywordsConcatProperty} +import io.renku.entities.searchgraphs.projects.commands.Encoders._ import io.renku.graph.config.RenkuUrlLoader import io.renku.graph.model.Schemas.{rdf, renku, schema} import io.renku.graph.model.images.{Image, ImageUri} @@ -149,14 +151,18 @@ private class UpdateQueriesCalculatorImpl[F[_]: Applicative: Logger](implicit ru SparqlQuery.ofUnsafe( show"$reportingPrefix: update keywords in Projects", Prefixes of (renku -> "renku", schema -> "schema"), - sparql"""|DELETE { GRAPH ${GraphClass.Projects.id} { ?id schema:keywords ?keyword } } + sparql"""|DELETE { GRAPH ${GraphClass.Projects.id} { + | ?id ${keywordsConcatProperty.id} ?keywords + | } + |} |INSERT { GRAPH ${GraphClass.Projects.id} { - | ${newValue.map(k => fr"""?id schema:keywords ${k.asObject}.""").toList.intercalate(fr"\n")} - |} } + | ${maybeKeywordsObject(newValue.toList).map(k => fr"""?id ${keywordsConcatProperty.id} $k""")} + | } + |} |WHERE { | GRAPH ${GraphClass.Projects.id} { | ?id renku:slug ${slug.asObject}. - | OPTIONAL { ?id schema:keywords ?keyword } + | OPTIONAL { ?id ${keywordsConcatProperty.id} ?keywords } | } |}""".stripMargin ) @@ -194,21 +200,19 @@ private class UpdateQueriesCalculatorImpl[F[_]: Applicative: Logger](implicit ru private def imagesInProjectsUpdate(slug: projects.Slug, newValue: List[Image]) = SparqlQuery.ofUnsafe( - show"$reportingPrefix: update keywords in Projects", + show"$reportingPrefix: update images in Projects", Prefixes of (rdf -> "rdf", renku -> "renku", schema -> "schema"), sparql"""|DELETE { GRAPH ${GraphClass.Projects.id} { - | ?id schema:image ?imageId. - | ?imageId ?p ?o + | ?id ${imagesConcatProperty.id} ?images. |} } |INSERT { GRAPH ${GraphClass.Projects.id} { - | ${newValue.flatMap(toTriple).intercalate(fr"\n ")} + | ${maybeImagesObject(newValue).map(i => fr"""?id ${imagesConcatProperty.id} $i""")} |} } |WHERE { | GRAPH ${GraphClass.Projects.id} { | ?id renku:slug ${slug.asObject}. | OPTIONAL { - | ?id schema:image ?imageId. - | ?imageId ?p ?o + | ?id ${imagesConcatProperty.id} ?images. | } | } |}""".stripMargin diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/syncrepometadata/processor/UpdateCommandsCalculatorSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/syncrepometadata/processor/UpdateCommandsCalculatorSpec.scala index 809f1779e1..0d28bf14df 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/syncrepometadata/processor/UpdateCommandsCalculatorSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/syncrepometadata/processor/UpdateCommandsCalculatorSpec.scala @@ -22,7 +22,7 @@ import Generators._ import cats.effect.IO import cats.syntax.all._ import eu.timepit.refined.auto._ -import io.renku.entities.searchgraphs.SearchInfoDatasets +import io.renku.entities.searchgraphs.{SearchInfoDatasets, concatSeparator} import io.renku.eventlog.api.events.StatusChangeEvent import io.renku.generators.Generators.Implicits._ import io.renku.graph.model.RenkuTinyTypeGenerators.projectNames @@ -379,8 +379,8 @@ class UpdateCommandsCalculatorSpec "UpdateCommandsCalculator Project fetch", Prefixes of (renku -> "renku", schema -> "schema"), sparql"""|SELECT ?id ?slug ?name ?visibility ?maybeDateModified ?maybeDesc - | (GROUP_CONCAT(DISTINCT ?keyword; separator=',') AS ?keywords) - | (GROUP_CONCAT(?encodedImageUrl; separator=',') AS ?images) + | (GROUP_CONCAT(DISTINCT ?keyword; separator=$concatSeparator) AS ?keywords) + | (GROUP_CONCAT(?encodedImageUrl; separator=$concatSeparator) AS ?images) |WHERE { | BIND (${GraphClass.Project.id(project.resourceId)} AS ?id) | GRAPH ?id { @@ -409,9 +409,7 @@ class UpdateCommandsCalculatorSpec SparqlQuery.ofUnsafe( "UpdateCommandsCalculator Projects fetch", Prefixes of (renku -> "renku", schema -> "schema"), - sparql"""|SELECT ?id ?slug ?name ?visibility ?maybeDateModified ?maybeDesc - | (GROUP_CONCAT(DISTINCT ?keyword; separator=',') AS ?keywords) - | (GROUP_CONCAT(?encodedImageUrl; separator=',') AS ?images) + sparql"""|SELECT ?id ?slug ?name ?visibility ?maybeDateModified ?maybeDesc ?keywords ?images |WHERE { | BIND (${project.resourceId.asEntityId} AS ?id) | GRAPH ${GraphClass.Projects.id} { @@ -420,16 +418,10 @@ class UpdateCommandsCalculatorSpec | renku:projectVisibility ?visibility. | OPTIONAL { ?id schema:dateModified ?maybeDateModified } | OPTIONAL { ?id schema:description ?maybeDesc } - | OPTIONAL { ?id schema:keywords ?keyword } - | OPTIONAL { - | ?id schema:image ?imageId. - | ?imageId schema:position ?imagePosition; - | schema:contentUrl ?imageUrl. - | BIND(CONCAT(STR(?imagePosition), STR(':'), STR(?imageUrl)) AS ?encodedImageUrl) - | } + | OPTIONAL { ?id renku:keywordsConcat ?keywords } + | OPTIONAL { ?id renku:imagesConcat ?images } | } |} - |GROUP BY ?id ?slug ?name ?visibility ?maybeDateModified ?maybeDesc |""".stripMargin ) ).map(toDataExtract).flatMap(toOptionOrFail) @@ -448,10 +440,10 @@ class UpdateCommandsCalculatorSpec } private lazy val toSetOfKeywords: Option[String] => Set[projects.Keyword] = - _.map(_.split(',').toList.map(projects.Keyword(_)).toSet).getOrElse(Set.empty) + _.map(_.split(concatSeparator).toList.map(projects.Keyword(_)).toSet).getOrElse(Set.empty) private lazy val toListOfImages: Option[String] => List[ImageUri] = - _.map(ImageUri.fromSplitString(',')(_).fold(throw _, identity)).getOrElse(Nil) + _.map(ImageUri.fromSplitString(concatSeparator)(_).fold(throw _, identity)).getOrElse(Nil) private def givenNewValuesFinding(tsData: DataExtract.TS, glData: DataExtract.GL, diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphFlattenersSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphFlattenersSpec.scala index 93fd9db478..6bbd408a3c 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphFlattenersSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphFlattenersSpec.scala @@ -89,6 +89,10 @@ class DatasetsGraphFlattenersSpec for { _ <- provisionProjects(project1, project2).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaKeywords(ds1TopSameAs, ds1)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaImages(ds1TopSameAs, ds1)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaKeywords(ds2TopSameAs, ds2)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaImages(ds2TopSameAs, ds2)).assertNoException _ <- fetchKeywords(ds1TopSameAs).asserting(_ shouldBe ds1.additionalInfo.keywords) _ <- fetchImages(ds1TopSameAs).asserting(_ shouldBe ds1.additionalInfo.images.map(_.uri)) @@ -230,6 +234,37 @@ class DatasetsGraphFlattenersSpec .getOrElse(List.empty[(projects.Slug, projects.Visibility)]) ) + private def insertSchemaKeywords(topSameAs: datasets.TopmostSameAs, ds: entities.Dataset[_]) = + ds.additionalInfo.keywords.map { keyword => + SparqlQuery.ofUnsafe( + "test insert ds keyword", + Prefixes of (renku -> "renku", schema -> "schema"), + sparql"""|INSERT DATA { + | GRAPH ${GraphClass.Datasets.id} { + | ${topSameAs.asEntityId} schema:keywords ${keyword.asObject}. + | } + |} + |""".stripMargin + ) + } + + private def insertSchemaImages(topSameAs: datasets.TopmostSameAs, ds: entities.Dataset[_]) = + ds.additionalInfo.images.map { image => + SparqlQuery.ofUnsafe( + "test insert ds image", + Prefixes of (rdf -> "rdf", renku -> "renku", schema -> "schema"), + sparql"""|INSERT DATA { + | GRAPH ${GraphClass.Datasets.id} { + | ${topSameAs.asEntityId} schema:image ${image.resourceId.asEntityId}. + | ${image.resourceId.asEntityId} rdf:type schema:ImageObject. + | ${image.resourceId.asEntityId} schema:contentUrl ${image.uri.asObject}. + | ${image.resourceId.asEntityId} schema:position ${image.position.asObject}. + | } + |} + |""".stripMargin + ) + } + private def deleteKeywords(topSameAs: datasets.TopmostSameAs) = SparqlQuery.ofUnsafe( "test delete ds keywords", diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphPropertiesRemoverSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphPropertiesRemoverSpec.scala new file mode 100644 index 0000000000..436266f19b --- /dev/null +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/DatasetsGraphPropertiesRemoverSpec.scala @@ -0,0 +1,179 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest.migrations + +import cats.effect.IO +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.entities.searchgraphs.SearchInfoDatasets +import io.renku.generators.Generators.Implicits._ +import io.renku.graph.model._ +import io.renku.graph.model.testentities._ +import io.renku.interpreters.TestLogger +import io.renku.jsonld.syntax._ +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.metrics.TestMetricsRegistry +import io.renku.testtools.CustomAsyncIOSpec +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore._ +import io.renku.triplesstore.client.syntax._ +import org.scalamock.scalatest.AsyncMockFactory +import org.scalatest.Succeeded +import org.scalatest.flatspec.AsyncFlatSpec +import org.scalatest.matchers.should +import org.typelevel.log4cats.Logger +import tooling.RegisteredUpdateQueryMigration + +class DatasetsGraphPropertiesRemoverSpec + extends AsyncFlatSpec + with CustomAsyncIOSpec + with should.Matchers + with InMemoryJenaForSpec + with ProjectsDataset + with SearchInfoDatasets + with AsyncMockFactory { + + it should "be a RegisteredUpdateQueryMigration" in { + implicit val metricsRegistry: TestMetricsRegistry[IO] = TestMetricsRegistry[IO] + implicit val timeRecorder: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() + + DatasetsGraphKeywordsRemover[IO].asserting( + _.getClass shouldBe classOf[RegisteredUpdateQueryMigration[IO]] + ) + DatasetsGraphImagesRemover[IO].asserting( + _.getClass shouldBe classOf[RegisteredUpdateQueryMigration[IO]] + ) + } + + it should "remove old keywords and images properties " + + "from all datasets in the Datasets graph" in { + + val ds1 -> project1 = anyRenkuProjectEntities + .addDataset { + datasetEntities(provenanceInternal) + .modify(replaceDSKeywords(List.empty)) + .modify(replaceDSImages(imageUris.generateList(min = 1))) + } + .generateOne + .bimap(_.to[entities.Dataset[entities.Dataset.Provenance.Internal]], _.to[entities.Project]) + val ds1TopSameAs = ds1.provenance.topmostSameAs + + val ds2 -> project2 = anyRenkuProjectEntities + .addDataset { + datasetEntities(provenanceInternal) + .modify(replaceDSKeywords(datasetKeywords.generateList(min = 1))) + .modify(replaceDSImages(List.empty)) + } + .generateOne + .bimap(_.to[entities.Dataset[entities.Dataset.Provenance.Internal]], _.to[entities.Project]) + val ds2TopSameAs = ds2.provenance.topmostSameAs + + for { + _ <- provisionProjects(project1, project2).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaKeywords(ds1TopSameAs, ds1)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaImages(ds1TopSameAs, ds1)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaKeywords(ds2TopSameAs, ds2)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaImages(ds2TopSameAs, ds2)).assertNoException + + _ <- fetchKeywords(ds1TopSameAs).asserting(_ shouldBe ds1.additionalInfo.keywords) + _ <- fetchImages(ds1TopSameAs).asserting(_ shouldBe ds1.additionalInfo.images.map(_.uri)) + _ <- fetchKeywords(ds2TopSameAs).asserting(_ shouldBe ds2.additionalInfo.keywords) + _ <- fetchImages(ds2TopSameAs).asserting(_ shouldBe ds2.additionalInfo.images.map(_.uri)) + + _ <- runUpdate(projectsDataset, DatasetsGraphKeywordsRemover.query).assertNoException + _ <- runUpdate(projectsDataset, DatasetsGraphImagesRemover.query).assertNoException + + _ <- fetchKeywords(ds1TopSameAs).asserting(_ shouldBe Nil) + _ <- fetchImages(ds1TopSameAs).asserting(_ shouldBe Nil) + _ <- fetchKeywords(ds2TopSameAs).asserting(_ shouldBe Nil) + _ <- fetchImages(ds2TopSameAs).asserting(_ shouldBe Nil) + } yield Succeeded + } + + private def fetchKeywords(topSameAs: datasets.TopmostSameAs): IO[List[datasets.Keyword]] = + runSelect( + on = projectsDataset, + SparqlQuery.ofUnsafe( + "test ds keywords", + Prefixes of (renku -> "renku", schema -> "schema"), + sparql"""|SELECT ?keys + |WHERE { + | GRAPH ${GraphClass.Datasets.id} { + | ${topSameAs.asEntityId} schema:keywords ?keys. + | } + |} + |""".stripMargin + ) + ).map(_.flatMap(_.get("keys").map(datasets.Keyword))) + + private def fetchImages(topSameAs: datasets.TopmostSameAs): IO[List[images.ImageUri]] = + runSelect( + on = projectsDataset, + SparqlQuery.ofUnsafe( + "test ds images", + Prefixes of (renku -> "renku", schema -> "schema"), + sparql"""|SELECT ?url ?pos + |WHERE { + | GRAPH ${GraphClass.Datasets.id} { + | ${topSameAs.asEntityId} schema:image ?imgId. + | ?imgId schema:contentUrl ?url. + | ?imgId schema:position ?pos. + | } + |} + |""".stripMargin + ) + ).map( + _.flatMap(row => (row.get("url").map(images.ImageUri(_)) -> row.get("pos").map(_.toInt)).mapN(_ -> _)) + .sortBy(_._2) + .map(_._1) + ) + + private def insertSchemaKeywords(topSameAs: datasets.TopmostSameAs, ds: entities.Dataset[_]) = + ds.additionalInfo.keywords.map { keyword => + SparqlQuery.ofUnsafe( + "test insert ds keyword", + Prefixes of (renku -> "renku", schema -> "schema"), + sparql"""|INSERT DATA { + | GRAPH ${GraphClass.Datasets.id} { + | ${topSameAs.asEntityId} schema:keywords ${keyword.asObject}. + | } + |} + |""".stripMargin + ) + } + + private def insertSchemaImages(topSameAs: datasets.TopmostSameAs, ds: entities.Dataset[_]) = + ds.additionalInfo.images.map { image => + SparqlQuery.ofUnsafe( + "test insert ds image", + Prefixes of (rdf -> "rdf", renku -> "renku", schema -> "schema"), + sparql"""|INSERT DATA { + | GRAPH ${GraphClass.Datasets.id} { + | ${topSameAs.asEntityId} schema:image ${image.resourceId.asEntityId}. + | ${image.resourceId.asEntityId} rdf:type schema:ImageObject. + | ${image.resourceId.asEntityId} schema:contentUrl ${image.uri.asObject}. + | ${image.resourceId.asEntityId} schema:position ${image.position.asObject}. + | } + |} + |""".stripMargin + ) + } + + implicit override lazy val ioLogger: Logger[IO] = TestLogger[IO]() +} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphFlattenersSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphFlattenersSpec.scala index 436b9b834d..33c3283d73 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphFlattenersSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphFlattenersSpec.scala @@ -75,6 +75,10 @@ class ProjectsGraphFlattenersSpec for { _ <- provisionProjects(project1, project2).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaKeywords(project1.resourceId, project1)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaImages(project1.resourceId, project1)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaKeywords(project2.resourceId, project2)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaImages(project2.resourceId, project2)).assertNoException _ <- fetchKeywords(project1.resourceId).asserting(_ shouldBe project1.keywords) _ <- fetchImages(project1.resourceId).asserting(_ shouldBe project1.images.map(_.uri)) @@ -150,6 +154,37 @@ class ProjectsGraphFlattenersSpec .getOrElse(List.empty[images.ImageUri]) ) + private def insertSchemaKeywords(projectId: projects.ResourceId, project: entities.Project) = + project.keywords.toList.map { keyword => + SparqlQuery.ofUnsafe( + "test insert project keyword", + Prefixes of (renku -> "renku", schema -> "schema"), + sparql"""|INSERT DATA { + | GRAPH ${GraphClass.Projects.id} { + | ${projectId.asEntityId} schema:keywords ${keyword.asObject}. + | } + |} + |""".stripMargin + ) + } + + private def insertSchemaImages(projectId: projects.ResourceId, project: entities.Project) = + project.images.map { image => + SparqlQuery.ofUnsafe( + "test insert project image", + Prefixes of (rdf -> "rdf", renku -> "renku", schema -> "schema"), + sparql"""|INSERT DATA { + | GRAPH ${GraphClass.Projects.id} { + | ${projectId.asEntityId} schema:image ${image.resourceId.asEntityId}. + | ${image.resourceId.asEntityId} rdf:type schema:ImageObject. + | ${image.resourceId.asEntityId} schema:contentUrl ${image.uri.asObject}. + | ${image.resourceId.asEntityId} schema:position ${image.position.asObject}. + | } + |} + |""".stripMargin + ) + } + private def deleteKeywords(projectId: projects.ResourceId) = SparqlQuery.ofUnsafe( "test delete keywords", diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphPropertiesRemoverSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphPropertiesRemoverSpec.scala new file mode 100644 index 0000000000..d33acb2a9f --- /dev/null +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/ProjectsGraphPropertiesRemoverSpec.scala @@ -0,0 +1,170 @@ +/* + * Copyright 2023 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.triplesgenerator.events.consumers.tsmigrationrequest.migrations + +import cats.effect.IO +import cats.syntax.all._ +import eu.timepit.refined.auto._ +import io.renku.entities.searchgraphs.SearchInfoDatasets +import io.renku.generators.Generators.Implicits._ +import io.renku.graph.model._ +import io.renku.graph.model.testentities._ +import io.renku.interpreters.TestLogger +import io.renku.jsonld.syntax._ +import io.renku.logging.TestSparqlQueryTimeRecorder +import io.renku.metrics.TestMetricsRegistry +import io.renku.testtools.CustomAsyncIOSpec +import io.renku.triplesstore.SparqlQuery.Prefixes +import io.renku.triplesstore._ +import io.renku.triplesstore.client.syntax._ +import org.scalamock.scalatest.AsyncMockFactory +import org.scalatest.Succeeded +import org.scalatest.flatspec.AsyncFlatSpec +import org.scalatest.matchers.should +import org.typelevel.log4cats.Logger +import tooling.RegisteredUpdateQueryMigration + +class ProjectsGraphPropertiesRemoverSpec + extends AsyncFlatSpec + with CustomAsyncIOSpec + with should.Matchers + with InMemoryJenaForSpec + with ProjectsDataset + with SearchInfoDatasets + with AsyncMockFactory { + + it should "be a RegisteredUpdateQueryMigration" in { + implicit val metricsRegistry: TestMetricsRegistry[IO] = TestMetricsRegistry[IO] + implicit val timeRecorder: SparqlQueryTimeRecorder[IO] = TestSparqlQueryTimeRecorder[IO].unsafeRunSync() + + ProjectsGraphImagesRemover[IO].asserting( + _.getClass shouldBe classOf[RegisteredUpdateQueryMigration[IO]] + ) + ProjectsGraphImagesRemover[IO].asserting( + _.getClass shouldBe classOf[RegisteredUpdateQueryMigration[IO]] + ) + } + + it should "remove old keywords and images properties " + + "from all projects in the Projects graph" in { + + val project1 = anyRenkuProjectEntities + .modify(replaceProjectKeywords(Set.empty)) + .modify(replaceImages(imageUris.generateList(min = 1))) + .generateOne + .to[entities.Project] + val project2 = anyRenkuProjectEntities + .modify(replaceProjectKeywords(projectKeywords.generateSet(min = 1))) + .modify(replaceImages(Nil)) + .generateOne + .to[entities.Project] + + for { + _ <- provisionProjects(project1, project2).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaKeywords(project1.resourceId, project1)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaImages(project1.resourceId, project1)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaKeywords(project2.resourceId, project2)).assertNoException + _ <- runUpdates(projectsDataset, insertSchemaImages(project2.resourceId, project2)).assertNoException + + _ <- fetchKeywords(project1.resourceId).asserting(_ shouldBe project1.keywords) + _ <- fetchImages(project1.resourceId).asserting(_ shouldBe project1.images.map(_.uri)) + _ <- fetchKeywords(project2.resourceId).asserting(_ shouldBe project2.keywords) + _ <- fetchImages(project2.resourceId).asserting(_ shouldBe project2.images.map(_.uri)) + + _ <- runUpdate(projectsDataset, ProjectsGraphKeywordsRemover.query).assertNoException + _ <- runUpdate(projectsDataset, ProjectsGraphImagesRemover.query).assertNoException + + _ <- fetchKeywords(project1.resourceId).asserting(_ shouldBe Set.empty) + _ <- fetchImages(project1.resourceId).asserting(_ shouldBe Nil) + _ <- fetchKeywords(project2.resourceId).asserting(_ shouldBe Set.empty) + _ <- fetchImages(project2.resourceId).asserting(_ shouldBe Nil) + } yield Succeeded + } + + private def fetchKeywords(projectId: projects.ResourceId): IO[Set[projects.Keyword]] = + runSelect( + on = projectsDataset, + SparqlQuery.ofUnsafe( + "test project keywords", + Prefixes of (renku -> "renku", schema -> "schema"), + sparql"""|SELECT ?keys + |WHERE { + | GRAPH ${GraphClass.Projects.id} { + | ${projectId.asEntityId} schema:keywords ?keys. + | } + |} + |""".stripMargin + ) + ).map(_.flatMap(_.get("keys").map(projects.Keyword)).toSet) + + private def fetchImages(projectId: projects.ResourceId): IO[List[images.ImageUri]] = + runSelect( + on = projectsDataset, + SparqlQuery.ofUnsafe( + "test project images", + Prefixes of (renku -> "renku", schema -> "schema"), + sparql"""|SELECT ?url ?pos + |WHERE { + | GRAPH ${GraphClass.Projects.id} { + | ${projectId.asEntityId} schema:image ?imgId. + | ?imgId schema:contentUrl ?url. + | ?imgId schema:position ?pos. + | } + |} + |""".stripMargin + ) + ).map( + _.flatMap(row => (row.get("url").map(images.ImageUri(_)) -> row.get("pos").map(_.toInt)).mapN(_ -> _)) + .sortBy(_._2) + .map(_._1) + ) + + private def insertSchemaKeywords(projectId: projects.ResourceId, project: entities.Project) = + project.keywords.toList.map { keyword => + SparqlQuery.ofUnsafe( + "test insert project keyword", + Prefixes of (renku -> "renku", schema -> "schema"), + sparql"""|INSERT DATA { + | GRAPH ${GraphClass.Projects.id} { + | ${projectId.asEntityId} schema:keywords ${keyword.asObject}. + | } + |} + |""".stripMargin + ) + } + + private def insertSchemaImages(projectId: projects.ResourceId, project: entities.Project) = + project.images.map { image => + SparqlQuery.ofUnsafe( + "test insert project image", + Prefixes of (rdf -> "rdf", renku -> "renku", schema -> "schema"), + sparql"""|INSERT DATA { + | GRAPH ${GraphClass.Projects.id} { + | ${projectId.asEntityId} schema:image ${image.resourceId.asEntityId}. + | ${image.resourceId.asEntityId} rdf:type schema:ImageObject. + | ${image.resourceId.asEntityId} schema:contentUrl ${image.uri.asObject}. + | ${image.resourceId.asEntityId} schema:position ${image.position.asObject}. + | } + |} + |""".stripMargin + ) + } + + implicit override lazy val ioLogger: Logger[IO] = TestLogger[IO]() +} diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/projects/update/UpdateQueriesCalculatorSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/projects/update/UpdateQueriesCalculatorSpec.scala index 10dd3f3b02..79a46630cf 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/projects/update/UpdateQueriesCalculatorSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/projects/update/UpdateQueriesCalculatorSpec.scala @@ -21,7 +21,7 @@ package io.renku.triplesgenerator.projects.update import cats.effect.IO import cats.syntax.all._ import eu.timepit.refined.auto._ -import io.renku.entities.searchgraphs.SearchInfoDatasets +import io.renku.entities.searchgraphs.{SearchInfoDatasets, concatSeparator} import io.renku.generators.Generators.Implicits._ import io.renku.graph.model.images.ImageUri import io.renku.graph.model.testentities._ @@ -277,8 +277,8 @@ class UpdateQueriesCalculatorSpec "UpdateQueriesCalculator Project fetch", Prefixes of (renku -> "renku", schema -> "schema"), sparql"""|SELECT DISTINCT ?maybeDesc ?visibility - | (GROUP_CONCAT(DISTINCT ?keyword; separator=',') AS ?keywords) - | (GROUP_CONCAT(?encodedImageUrl; separator=',') AS ?images) + | (GROUP_CONCAT(DISTINCT ?keyword; separator=$concatSeparator) AS ?keywords) + | (GROUP_CONCAT(?encodedImageUrl; separator=$concatSeparator) AS ?images) |WHERE { | BIND (${GraphClass.Project.id(project.resourceId)} AS ?id) | GRAPH ?id { @@ -305,24 +305,16 @@ class UpdateQueriesCalculatorSpec SparqlQuery.ofUnsafe( "UpdateQueriesCalculator Projects fetch", Prefixes of (renku -> "renku", schema -> "schema"), - sparql"""|SELECT DISTINCT ?maybeDesc ?visibility - | (GROUP_CONCAT(DISTINCT ?keyword; separator=',') AS ?keywords) - | (GROUP_CONCAT(?encodedImageUrl; separator=',') AS ?images) + sparql"""|SELECT DISTINCT ?maybeDesc ?visibility ?keywords ?images |WHERE { | BIND (${project.resourceId.asEntityId} AS ?id) | GRAPH ${GraphClass.Projects.id} { | ?id renku:projectVisibility ?visibility. | OPTIONAL { ?id schema:description ?maybeDesc } - | OPTIONAL { ?id schema:keywords ?keyword } - | OPTIONAL { - | ?id schema:image ?imageId. - | ?imageId schema:position ?imagePosition; - | schema:contentUrl ?imageUrl. - | BIND(CONCAT(STR(?imagePosition), STR(':'), STR(?imageUrl)) AS ?encodedImageUrl) - | } + | OPTIONAL { ?id renku:keywordsConcat ?keywords } + | OPTIONAL { ?id renku:imagesConcat ?images } | } |} - |GROUP BY ?maybeDesc ?visibility |""".stripMargin ) ).map(toDataExtract).flatMap(toOptionOrFail) @@ -338,10 +330,10 @@ class UpdateQueriesCalculatorSpec } private lazy val toSetOfKeywords: Option[String] => Set[projects.Keyword] = - _.map(_.split(',').toList.map(projects.Keyword(_)).toSet).getOrElse(Set.empty) + _.map(_.split(concatSeparator).toList.map(projects.Keyword(_)).toSet).getOrElse(Set.empty) private lazy val toListOfImages: Option[String] => List[ImageUri] = - _.map(ImageUri.fromSplitString(',')(_).fold(throw _, identity)).getOrElse(Nil) + _.map(ImageUri.fromSplitString(concatSeparator)(_).fold(throw _, identity)).getOrElse(Nil) private lazy val toOptionOrFail: List[TSData] => IO[Option[TSData]] = { case Nil => Option.empty[TSData].pure[IO] diff --git a/triples-store-client/src/test/scala/io/renku/triplesstore/client/util/JenaContainer.scala b/triples-store-client/src/test/scala/io/renku/triplesstore/client/util/JenaContainer.scala index c96eaee81f..99f80d15cc 100644 --- a/triples-store-client/src/test/scala/io/renku/triplesstore/client/util/JenaContainer.scala +++ b/triples-store-client/src/test/scala/io/renku/triplesstore/client/util/JenaContainer.scala @@ -28,7 +28,7 @@ import org.testcontainers.containers import org.testcontainers.utility.DockerImageName object JenaContainer { - val version = "0.0.21" + val version = "0.0.22" val imageName = s"renku/renku-jena:$version" val image = DockerImageName.parse(imageName) diff --git a/webhook-service/Dockerfile b/webhook-service/Dockerfile index 14ba137b20..ed7890cb28 100644 --- a/webhook-service/Dockerfile +++ b/webhook-service/Dockerfile @@ -1,7 +1,7 @@ # This is a multi-stage build, see reference: # https://docs.docker.com/develop/develop-images/multistage-build/ -FROM eclipse-temurin:17-jre-alpine as builder +FROM eclipse-temurin:21-jre-alpine as builder WORKDIR /work @@ -16,7 +16,7 @@ RUN export PATH="/usr/local/sbt/bin:$PATH" && \ sbt "project webhook-service" stage && \ apk del .build-dependencies -FROM eclipse-temurin:17-jre-alpine +FROM eclipse-temurin:21-jre-alpine WORKDIR /opt/webhook-service