From 48897255bb43c61208e3cb0f7138ace963e47ca8 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Wed, 15 Jan 2025 12:07:02 +0100 Subject: [PATCH 1/4] common: Add `LanguageFields` & `OptLanguageFields` types Hopefully these types will be used with all stored language fields in the future. --- .../domain/language/LanguageFields.scala | 40 +++++++++ .../domain/language/OptLanguageFields.scala | 70 ++++++++++++++++ .../language/OptionalLanguageValue.scala | 36 +++++++++ .../model/domain/LanguageFieldsTest.scala | 81 +++++++++++++++++++ .../ndla/language/model/LanguageField.scala | 2 +- .../language/model/WithLanguageAndValue.scala | 19 +++++ project/languagelib.scala | 3 +- 7 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 common/src/main/scala/no/ndla/common/model/domain/language/LanguageFields.scala create mode 100644 common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala create mode 100644 common/src/main/scala/no/ndla/common/model/domain/language/OptionalLanguageValue.scala create mode 100644 common/src/test/scala/no/ndla/common/model/domain/LanguageFieldsTest.scala create mode 100644 language/src/main/scala/no/ndla/language/model/WithLanguageAndValue.scala diff --git a/common/src/main/scala/no/ndla/common/model/domain/language/LanguageFields.scala b/common/src/main/scala/no/ndla/common/model/domain/language/LanguageFields.scala new file mode 100644 index 0000000000..ba4fb89f7c --- /dev/null +++ b/common/src/main/scala/no/ndla/common/model/domain/language/LanguageFields.scala @@ -0,0 +1,40 @@ +/* + * Part of NDLA backend.common.main + * Copyright (C) 2025 NDLA + * + * See LICENSE + * + */ + +package no.ndla.common.model.domain.language + +import io.circe.* +import io.circe.syntax.EncoderOps +import no.ndla.language.Language +import no.ndla.language.model.{BaseWithLanguageAndValue, WithLanguageAndValue} + +case class LanguageFields[T: Encoder: Decoder](internal: Map[String, T]) { + def getWithLanguageFields: Seq[WithLanguageAndValue[T]] = internal.map { case (language, value) => + BaseWithLanguageAndValue(language, value) + }.toSeq + def get(language: String): Option[WithLanguageAndValue[T]] = + internal.get(language).map(BaseWithLanguageAndValue(language, _)) + def findByLanguageOrBestEffort(language: String): Option[WithLanguageAndValue[T]] = + Language.findByLanguageOrBestEffort(getWithLanguageFields, language) +} + +object LanguageFields { + def empty[T: Encoder: Decoder]: LanguageFields[T] = LanguageFields(Map.empty) + def fromFields[T]( + fields: Seq[WithLanguageAndValue[T]] + )(implicit encoder: Encoder[T], decoder: Decoder[T]): LanguageFields[T] = { + val underlyingMap = fields.map(f => f.language -> f.value).toMap + LanguageFields(underlyingMap) + } + + implicit def encoder[T: Encoder]: Encoder[LanguageFields[T]] = Encoder.instance { lf => lf.internal.asJson } + implicit def decoder[T: Decoder: Encoder]: Decoder[LanguageFields[T]] = Decoder.instance { json => + json.as[Map[String, T]].map { m => LanguageFields(m) } + + } +} diff --git a/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala b/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala new file mode 100644 index 0000000000..ea27e0bc9b --- /dev/null +++ b/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala @@ -0,0 +1,70 @@ +/* + * Part of NDLA backend.common.main + * Copyright (C) 2025 NDLA + * + * See LICENSE + * + */ + +package no.ndla.common.model.domain.language + +import io.circe.syntax.EncoderOps +import io.circe.{Decoder, Encoder, Json} +import no.ndla.common.model.domain.language.OptionalLanguageValue.{NotWantedKey, NotWantedKeyT} +import no.ndla.language.model.WithLanguageAndValue + +case class OptLanguageFields[T: Encoder: Decoder]( + internal: Map[String, Either[NotWantedKeyT, Option[T]]] +) { + def get(language: String): Option[OptionalLanguageValue[T]] = { + val res = internal.get(language) + res match { + case None => None + case Some(Right(Some(value))) => Some(Exists(value)) + case Some(Right(None)) => None + case Some(Left(_)) => Some(NotWanted()) + } + } + + def withUnwanted(language: String): OptLanguageFields[T] = { + val updated: Map[String, Either[NotWantedKeyT, Option[T]]] = internal.updated(language, Left(NotWantedKey)) + OptLanguageFields(updated) + } +} + +object OptLanguageFields { + + def fromFields[T]( + fields: Seq[WithLanguageAndValue[T]] + )(implicit encoder: Encoder[T], decoder: Decoder[T]): OptLanguageFields[T] = { + val underlyingMap = fields.map(f => f.language -> Right(Some(f.value))).toMap + OptLanguageFields(underlyingMap) + } + + implicit def eitherEncoder[T](implicit e: Encoder[T]): Encoder[Either[NotWantedKeyT, Option[T]]] = Encoder.instance { + case Right(value) => value.asJson + case Left(_) => Json.obj(NotWantedKey -> Json.True) + } + + implicit def eitherDecoder[T](implicit d: Decoder[T]): Decoder[Either[NotWantedKeyT, Option[T]]] = Decoder.instance { + cursor => + val x = cursor.downField(NotWantedKey) + val notWantedField = x.as[Option[Boolean]] + notWantedField match { + case Right(Some(true)) => + Right(Left(NotWantedKey)) + case _ => + cursor.as[Option[T]].map(Right(_)) + } + } + + implicit def encoder[T: Encoder]: Encoder[OptLanguageFields[T]] = Encoder.instance { lf => + lf.internal.asJson + } + + implicit def decoder[T: Decoder: Encoder]: Decoder[OptLanguageFields[T]] = Decoder.instance { json => + json.as[Map[String, Either[NotWantedKeyT, Option[T]]]].map { m => + OptLanguageFields(m) + } + } +} diff --git a/common/src/main/scala/no/ndla/common/model/domain/language/OptionalLanguageValue.scala b/common/src/main/scala/no/ndla/common/model/domain/language/OptionalLanguageValue.scala new file mode 100644 index 0000000000..0904023d01 --- /dev/null +++ b/common/src/main/scala/no/ndla/common/model/domain/language/OptionalLanguageValue.scala @@ -0,0 +1,36 @@ +/* + * Part of NDLA common + * Copyright (C) 2025 NDLA + * + * See LICENSE + * + */ + +package no.ndla.common.model.domain.language + +import io.circe.syntax.EncoderOps +import io.circe.{Decoder, Encoder, HCursor, Json} + +sealed trait OptionalLanguageValue[T] +case class Exists[T: Encoder: Decoder](value: T) extends OptionalLanguageValue[T] +case class NotWanted[T]() extends OptionalLanguageValue[T] + +object OptionalLanguageValue { + type NotWantedKeyT = "__notwanted__" + final val NotWantedKey = "__notwanted__" + implicit def encoder[T](implicit valueEncoder: Encoder[T]): Encoder[OptionalLanguageValue[T]] = Encoder.instance { + case Exists(value) => Json.obj("value" -> value.asJson) + case NotWanted() => Json.obj(NotWantedKey -> Json.True) + } + + implicit def decoder[T: Encoder: Decoder]: Decoder[OptionalLanguageValue[T]] = + (c: HCursor) => { + c.downField(NotWantedKey).as[Option[Boolean]].flatMap { + case Some(true) => Right(NotWanted()) + case _ => + val field = c.downField("value") + val parsed = field.as[T] + parsed.map(value => Exists(value)) + } + } +} diff --git a/common/src/test/scala/no/ndla/common/model/domain/LanguageFieldsTest.scala b/common/src/test/scala/no/ndla/common/model/domain/LanguageFieldsTest.scala new file mode 100644 index 0000000000..4a8d474b31 --- /dev/null +++ b/common/src/test/scala/no/ndla/common/model/domain/LanguageFieldsTest.scala @@ -0,0 +1,81 @@ +/* + * Part of NDLA common + * Copyright (C) 2025 NDLA + * + * See LICENSE + * + */ + +package no.ndla.common.model.domain + +import no.ndla.common.CirceUtil +import no.ndla.common.model.domain.language.* +import no.ndla.language.model.BaseWithLanguageAndValue +import no.ndla.scalatestsuite.UnitTestSuite + +class LanguageFieldsTest extends UnitTestSuite { + + test("That language fields serialize and deserialize as expected") { + import io.circe.syntax.* + val fields = Seq( + BaseWithLanguageAndValue("nb", "bokmål"), + BaseWithLanguageAndValue("nn", "nynorsk"), + BaseWithLanguageAndValue("en", "english") + ) + + val languageFields = LanguageFields.fromFields(fields) + val jsonString = languageFields.asJson.noSpaces + val result = CirceUtil.unsafeParseAs[LanguageFields[String]](jsonString) + + result should be(languageFields) + } + + test("That language fields are found by language or best effort according to language priority") { + val fields = Seq( + BaseWithLanguageAndValue("nb", "bokmål"), + BaseWithLanguageAndValue("nn", "nynorsk"), + BaseWithLanguageAndValue("en", "english") + ) + + val languageFields = LanguageFields.fromFields(fields) + + languageFields.findByLanguageOrBestEffort("nb") should be(Some(BaseWithLanguageAndValue("nb", "bokmål"))) + languageFields.findByLanguageOrBestEffort("sma") should be(Some(BaseWithLanguageAndValue("nb", "bokmål"))) + languageFields.findByLanguageOrBestEffort("nn") should be(Some(BaseWithLanguageAndValue("nn", "nynorsk"))) + } + + test("That the LanguageFields type is able to differentiate between a missing and not needed field") { + + val fields = Seq( + BaseWithLanguageAndValue[OptionalLanguageValue[String]]("nb", Exists("bokmål")), + BaseWithLanguageAndValue[OptionalLanguageValue[String]]("nn", NotWanted()) + ) + + val languageFields = LanguageFields.fromFields(fields) + val jsonString = CirceUtil.toJsonString(languageFields) + + val result = CirceUtil.unsafeParseAs[LanguageFields[OptionalLanguageValue[String]]](jsonString) + result should be(languageFields) + + result.get("nb") should be(Some(BaseWithLanguageAndValue("nb", Exists("bokmål")))) + result.get("nn") should be(Some(BaseWithLanguageAndValue("nn", NotWanted()))) + } + + test("That the OptLanguageFields type is able to differentiate between a missing and not needed field") { + + val fields = Seq( + BaseWithLanguageAndValue[String]("nb", "bokmål") + ) + + val languageFields = OptLanguageFields.fromFields(fields).withUnwanted("en") + val jsonString = CirceUtil.toJsonString(languageFields) + + val result = CirceUtil.unsafeParseAs[OptLanguageFields[String]](jsonString) + result should be(languageFields) + + result.get("nb") should be(Some(Exists("bokmål"))) + result.get("nn") should be(None) + result.get("en") should be(Some(NotWanted())) + } + +} diff --git a/language/src/main/scala/no/ndla/language/model/LanguageField.scala b/language/src/main/scala/no/ndla/language/model/LanguageField.scala index 449d96912e..a192034688 100644 --- a/language/src/main/scala/no/ndla/language/model/LanguageField.scala +++ b/language/src/main/scala/no/ndla/language/model/LanguageField.scala @@ -1,6 +1,6 @@ package no.ndla.language.model -trait LanguageField[T] extends WithLanguage { +trait LanguageField[T] extends WithLanguageAndValue[T] { def value: T def isEmpty: Boolean } diff --git a/language/src/main/scala/no/ndla/language/model/WithLanguageAndValue.scala b/language/src/main/scala/no/ndla/language/model/WithLanguageAndValue.scala new file mode 100644 index 0000000000..64c4b823e1 --- /dev/null +++ b/language/src/main/scala/no/ndla/language/model/WithLanguageAndValue.scala @@ -0,0 +1,19 @@ +/* + * Part of NDLA backend.language.main + * Copyright (C) 2025 NDLA + * + * See LICENSE + * + */ + +package no.ndla.language.model + +trait WithLanguageAndValue[T] extends WithLanguage { + def language: String + def value: T +} + +case class BaseWithLanguageAndValue[T]( + language: String, + value: T +) extends WithLanguageAndValue[T] diff --git a/project/languagelib.scala b/project/languagelib.scala index 829be061aa..4920a554d7 100644 --- a/project/languagelib.scala +++ b/project/languagelib.scala @@ -9,7 +9,8 @@ object languagelib extends Module { lazy val dependencies: Seq[ModuleID] = withLogging( Seq( "org.scalatest" %% "scalatest" % ScalaTestV % "test" - ) + ), + tapirHttp4sCirce ) override lazy val settings: Seq[Def.Setting[?]] = Seq( From 3605a14f561462dcb3296cd6add02d0f7841e3bc Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Wed, 15 Jan 2025 15:29:54 +0100 Subject: [PATCH 2/4] article-api + draft-api: Make use of `OptLanguageFields` for disclaimer This patch adds a migration of disclaimer to `OptLanguageFields` as well as retypes the `diclaimer` of the domain type to `OptLanguageFields`. --- .../no/ndla/articleapi/db/HtmlMigration.scala | 2 +- .../V56__DisclaimerToLanguageFields.scala | 17 ++++ .../articleapi/service/ConverterService.scala | 4 +- .../validation/ContentValidator.scala | 15 ++-- .../scala/no/ndla/articleapi/TestData.scala | 11 +-- .../V55__DisclaimerToLanguageFieldsTest.scala | 37 +++++++++ .../validation/ContentValidatorTest.scala | 15 +--- .../audioapi/service/WriteServiceTest.scala | 2 +- .../common/errors/ValidationException.scala | 9 ++- .../ndla/common/model/api/DisclaimerDTO.scala | 3 + .../common/model/domain/article/Article.scala | 7 +- .../common/model/domain/draft/Draft.scala | 7 +- .../domain/language/LanguageFields.scala | 12 ++- .../domain/language/OptLanguageFields.scala | 77 ++++++++++++++++++- .../language/OptionalLanguageValue.scala | 10 ++- .../model/domain/LanguageFieldsTest.scala | 26 ++++--- .../no/ndla/database/DocumentMigration.scala | 2 +- .../database/LanguageFieldMigration.scala | 41 ++++++++++ .../no/ndla/draftapi/db/HtmlMigration.scala | 2 +- .../V67__DisclaimerToLanguageFields.scala | 17 ++++ .../draftapi/service/ConverterService.scala | 27 ++----- .../validation/ContentValidator.scala | 12 +-- .../scala/no/ndla/draftapi/TestData.scala | 9 ++- .../service/ConverterServiceTest.scala | 15 ++-- .../service/StateTransitionRulesTest.scala | 13 ++-- .../validation/ContentValidatorTest.scala | 14 ++-- .../articleapi/ArticleApiClientTest.scala | 3 +- .../language/model/WithLanguageAndValue.scala | 2 +- .../V39__MadeAvailableForThePublished.scala | 2 +- .../service/ConverterServiceTest.scala | 2 +- .../parameters/GetSearchQueryParams.scala | 2 +- .../scala/no/ndla/searchapi/TestData.scala | 19 ++--- .../model/api/grep/GrepResultDTOTest.scala | 2 +- 33 files changed, 316 insertions(+), 122 deletions(-) create mode 100644 article-api/src/main/scala/no/ndla/articleapi/db/migration/V56__DisclaimerToLanguageFields.scala create mode 100644 article-api/src/test/scala/no/ndla/articleapi/db/migration/V55__DisclaimerToLanguageFieldsTest.scala create mode 100644 database/src/main/scala/no/ndla/database/LanguageFieldMigration.scala create mode 100644 draft-api/src/main/scala/no/ndla/draftapi/db/migration/V67__DisclaimerToLanguageFields.scala diff --git a/article-api/src/main/scala/no/ndla/articleapi/db/HtmlMigration.scala b/article-api/src/main/scala/no/ndla/articleapi/db/HtmlMigration.scala index da56d7c79f..46ae833722 100644 --- a/article-api/src/main/scala/no/ndla/articleapi/db/HtmlMigration.scala +++ b/article-api/src/main/scala/no/ndla/articleapi/db/HtmlMigration.scala @@ -28,7 +28,7 @@ abstract class HtmlMigration extends DocumentMigration { jsoupDocumentToString(converted) } - protected def convertColumn(document: String): String = { + def convertColumn(document: String): String = { val oldArticle = parser.parse(document).flatMap(_.as[Article]).toTry.get val convertedContent = oldArticle.content.map(c => { val converted = convertContent(c.content, c.language) diff --git a/article-api/src/main/scala/no/ndla/articleapi/db/migration/V56__DisclaimerToLanguageFields.scala b/article-api/src/main/scala/no/ndla/articleapi/db/migration/V56__DisclaimerToLanguageFields.scala new file mode 100644 index 0000000000..4dad184e34 --- /dev/null +++ b/article-api/src/main/scala/no/ndla/articleapi/db/migration/V56__DisclaimerToLanguageFields.scala @@ -0,0 +1,17 @@ +/* + * Part of NDLA article-api + * Copyright (C) 2025 NDLA + * + * See LICENSE + * + */ + +package no.ndla.articleapi.db.migration + +import no.ndla.database.LanguageFieldMigration + +class V56__DisclaimerToLanguageFields extends LanguageFieldMigration { + override val columnName: String = "document" + override val tableName: String = "contentdata" + override val fieldName: String = "disclaimer" +} diff --git a/article-api/src/main/scala/no/ndla/articleapi/service/ConverterService.scala b/article-api/src/main/scala/no/ndla/articleapi/service/ConverterService.scala index 6cce27e91e..d5526b65b5 100644 --- a/article-api/src/main/scala/no/ndla/articleapi/service/ConverterService.scala +++ b/article-api/src/main/scala/no/ndla/articleapi/service/ConverterService.scala @@ -260,8 +260,8 @@ trait ConverterService { val metaImage = findByLanguageOrBestEffort(article.metaImage, language).map(toApiArticleMetaImage) val copyright = toApiCopyright(article.copyright) val disclaimer = article.disclaimer - .flatMap(d => findByLanguageOrBestEffort(d, language)) - .map(d => DisclaimerDTO(d.disclaimer, d.language)) + .findByLanguageOrBestEffort(language) + .map(DisclaimerDTO.fromLanguageValue) Success( api.ArticleV2DTO( diff --git a/article-api/src/main/scala/no/ndla/articleapi/validation/ContentValidator.scala b/article-api/src/main/scala/no/ndla/articleapi/validation/ContentValidator.scala index 90fc36721d..cef746ff50 100644 --- a/article-api/src/main/scala/no/ndla/articleapi/validation/ContentValidator.scala +++ b/article-api/src/main/scala/no/ndla/articleapi/validation/ContentValidator.scala @@ -14,6 +14,7 @@ import no.ndla.common.errors.{ValidationException, ValidationMessage} import no.ndla.common.model.NDLADate import no.ndla.common.model.domain.article.{Article, Copyright} import no.ndla.common.model.domain.* +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.language.model.{Iso639, LanguageField} import no.ndla.mapping.License.getLicense import no.ndla.validation.HtmlTagRules.{allLegalTags, stringToJsoupDocument} @@ -50,7 +51,7 @@ trait ContentValidator { def validateArticle(article: Article, isImported: Boolean): Try[Article] = { val validationErrors = validateArticleContent(article.content) ++ article.introduction.flatMap(i => validateIntroduction(i)) ++ - validateArticleDisclaimer(article.disclaimer.getOrElse(Seq.empty)) ++ + validateArticleDisclaimer(article.disclaimer) ++ validateMetaDescription(article.metaDescription, isImported) ++ validateTitle(article.title) ++ validateCopyright(article.copyright) ++ @@ -90,12 +91,12 @@ trait ContentValidator { }) ++ validateNonEmpty("content", contents) } - private def validateArticleDisclaimer(disclaimers: Seq[Disclaimer]): Seq[ValidationMessage] = { - disclaimers.flatMap(disclaimer => { + private def validateArticleDisclaimer(disclaimers: OptLanguageFields[String]): Seq[ValidationMessage] = { + disclaimers.mapExisting { disclaimer => val field = s"disclaimer.${disclaimer.language}" - TextValidator.validate(field, disclaimer.disclaimer, allLegalTags).toList ++ - validateLanguage("content.language", disclaimer.language) - }) + TextValidator.validate(field, disclaimer.value, allLegalTags).toList ++ + validateLanguage("disclaimer.language", disclaimer.language) + }.flatten } private def rootElementContainsOnlySectionBlocks(field: String, html: String): Option[ValidationMessage] = { @@ -149,7 +150,7 @@ trait ContentValidator { private def validateTitle(titles: Seq[LanguageField[String]]): Seq[ValidationMessage] = { titles.flatMap(title => { - val field = s"title.$language" + val field = s"title.language" TextValidator.validate(field, title.value, inlineHtmlTags).toList ++ validateLanguage("title.language", title.language) ++ validateLength("title", title.value, 256) diff --git a/article-api/src/test/scala/no/ndla/articleapi/TestData.scala b/article-api/src/test/scala/no/ndla/articleapi/TestData.scala index 790eaaf635..1362a6db2f 100644 --- a/article-api/src/test/scala/no/ndla/articleapi/TestData.scala +++ b/article-api/src/test/scala/no/ndla/articleapi/TestData.scala @@ -15,6 +15,7 @@ import no.ndla.common.model import no.ndla.common.model.NDLADate import no.ndla.common.model.domain.article.{Article, Copyright} import no.ndla.common.model.domain.* +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.mapping.License trait TestData { @@ -155,7 +156,7 @@ trait TestData { relatedContent = Seq.empty, revisionDate = Some(NDLADate.now().withNano(0)), slug = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val sampleDomainArticle: Article = Article( @@ -181,7 +182,7 @@ trait TestData { relatedContent = Seq.empty, revisionDate = None, slug = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val sampleDomainArticle2: Article = Article( @@ -207,7 +208,7 @@ trait TestData { relatedContent = Seq.empty, revisionDate = None, slug = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val sampleArticleWithByNcSa: Article = sampleArticleWithPublicDomain.copy(copyright = byNcSaCopyright) @@ -245,7 +246,7 @@ trait TestData { relatedContent = Seq.empty, revisionDate = None, slug = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val apiArticleWithHtmlFaultV2: api.ArticleV2DTO = api.ArticleV2DTO( @@ -316,7 +317,7 @@ trait TestData { relatedContent = Seq.empty, revisionDate = None, slug = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) } diff --git a/article-api/src/test/scala/no/ndla/articleapi/db/migration/V55__DisclaimerToLanguageFieldsTest.scala b/article-api/src/test/scala/no/ndla/articleapi/db/migration/V55__DisclaimerToLanguageFieldsTest.scala new file mode 100644 index 0000000000..c9b3a60aaf --- /dev/null +++ b/article-api/src/test/scala/no/ndla/articleapi/db/migration/V55__DisclaimerToLanguageFieldsTest.scala @@ -0,0 +1,37 @@ +/* + * Part of NDLA article-api + * Copyright (C) 2024 NDLA + * + * See LICENSE + */ + +package no.ndla.articleapi.db.migration + +import no.ndla.articleapi.{TestEnvironment, UnitSuite} + +class V55__DisclaimerToLanguageFieldsTest extends UnitSuite with TestEnvironment { + test("That old disclaimers are migrated to new language fields") { + val oldDocument = + """{"disclaimer":[{"disclaimer":"Dette er bokmål","language":"nb"},{"disclaimer":"Dette er nynorsk","language":"nn"}]}""" + val expectedResult = """{"disclaimer":{"nb":"Dette er bokmål","nn":"Dette er nynorsk"}}""" + val migration = new V56__DisclaimerToLanguageFields + val result = migration.convertColumn(oldDocument) + result should be(expectedResult) + } + + test("That no old disclaimers are migrated to new language fields") { + val oldDocument = """{}""" + val expectedResult = """{"disclaimer":{}}""" + val migration = new V56__DisclaimerToLanguageFields + val result = migration.convertColumn(oldDocument) + result should be(expectedResult) + } + + test("That null disclaimers are migrated to new language fields") { + val oldDocument = """{"disclaimer":null}""" + val expectedResult = """{"disclaimer":{}}""" + val migration = new V56__DisclaimerToLanguageFields + val result = migration.convertColumn(oldDocument) + result should be(expectedResult) + } +} diff --git a/article-api/src/test/scala/no/ndla/articleapi/validation/ContentValidatorTest.scala b/article-api/src/test/scala/no/ndla/articleapi/validation/ContentValidatorTest.scala index e9e6298e59..5c131220d2 100644 --- a/article-api/src/test/scala/no/ndla/articleapi/validation/ContentValidatorTest.scala +++ b/article-api/src/test/scala/no/ndla/articleapi/validation/ContentValidatorTest.scala @@ -15,13 +15,13 @@ import no.ndla.common.model.domain.{ ArticleMetaImage, Author, Description, - Disclaimer, Introduction, RequiredLibrary, Tag, Title } import no.ndla.common.model.domain.article.Copyright +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.mapping.License.{CC_BY_SA, NA} import scala.util.Failure @@ -83,7 +83,7 @@ class ContentValidatorTest extends UnitSuite with TestEnvironment { test("validateArticle should throw an error if disclaimer contains illegal HTML tags") { val article = TestData.sampleArticleWithByNcSa.copy( content = Seq(ArticleContent(validDocument, "nb")), - disclaimer = Some(Seq(Disclaimer("

hei

", "nb"))) + disclaimer = OptLanguageFields.withValue("

hei

", "nb") ) val Failure(error: ValidationException) = contentValidator.validateArticle(article, false) @@ -98,14 +98,7 @@ class ContentValidatorTest extends UnitSuite with TestEnvironment { test("validateArticle should not throw an error if disclaimer contains legal HTML tags") { val article = TestData.sampleArticleWithByNcSa.copy( content = Seq(ArticleContent(validDocument, "nb")), - disclaimer = Some( - Seq( - Disclaimer( - validDisclaimer, - "nb" - ) - ) - ) + disclaimer = OptLanguageFields.withValue(validDisclaimer, "nb") ) contentValidator.validateArticle(article, false).isSuccess should be(true) } @@ -113,7 +106,7 @@ class ContentValidatorTest extends UnitSuite with TestEnvironment { test("validateArticle should not throw an error if disclaimer contains plain text") { val article = TestData.sampleArticleWithByNcSa.copy( content = Seq(ArticleContent(validDocument, "nb")), - disclaimer = Some(Seq(Disclaimer("disclaimer", "nb"))) + disclaimer = OptLanguageFields.withValue("disclaimer", "nb") ) contentValidator.validateArticle(article, false).isSuccess should be(true) } diff --git a/audio-api/src/test/scala/no/ndla/audioapi/service/WriteServiceTest.scala b/audio-api/src/test/scala/no/ndla/audioapi/service/WriteServiceTest.scala index e053e30dc4..66bd13e3df 100644 --- a/audio-api/src/test/scala/no/ndla/audioapi/service/WriteServiceTest.scala +++ b/audio-api/src/test/scala/no/ndla/audioapi/service/WriteServiceTest.scala @@ -404,7 +404,7 @@ class WriteServiceTest extends UnitSuite with TestEnvironment { val result = writeService.updateAudio(1, updatedAudioMeta, Some(mock[UploadedFile]), testUser) result.isFailure should be(true) - result.failed.get.getMessage should equal(new ValidationException(errors = Seq()).getMessage) + result.failed.get.getMessage should equal(new ValidationException(errors = Seq(validationMessage)).getMessage) } test("that updateAudio returns Failure when audio upload fails") { diff --git a/common/src/main/scala/no/ndla/common/errors/ValidationException.scala b/common/src/main/scala/no/ndla/common/errors/ValidationException.scala index 544341cd32..f6dcb1bbee 100644 --- a/common/src/main/scala/no/ndla/common/errors/ValidationException.scala +++ b/common/src/main/scala/no/ndla/common/errors/ValidationException.scala @@ -8,11 +8,18 @@ package no.ndla.common.errors +import no.ndla.common.errors.ValidationException.formatError + case class ValidationException( message: String = "Validation Error", errors: Seq[ValidationMessage] -) extends RuntimeException(message) +) extends RuntimeException(formatError(message, errors)) object ValidationException { def apply(path: String, msg: String) = new ValidationException(errors = Seq(ValidationMessage(path, msg))) + + def formatError(message: String, errors: Seq[ValidationMessage]): String = { + if (errors.nonEmpty) s"$message:\n${errors.map(e => s"\t${e.field}: ${e.message}").mkString("\n")}" + else message + } } diff --git a/common/src/main/scala/no/ndla/common/model/api/DisclaimerDTO.scala b/common/src/main/scala/no/ndla/common/model/api/DisclaimerDTO.scala index 750c1a57b7..60ef2514a0 100644 --- a/common/src/main/scala/no/ndla/common/model/api/DisclaimerDTO.scala +++ b/common/src/main/scala/no/ndla/common/model/api/DisclaimerDTO.scala @@ -10,6 +10,7 @@ package no.ndla.common.model.api import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import io.circe.{Decoder, Encoder} +import no.ndla.language.model.WithLanguageAndValue import sttp.tapir.Schema.annotations.description case class DisclaimerDTO( @@ -18,6 +19,8 @@ case class DisclaimerDTO( ) object DisclaimerDTO { + def fromLanguageValue(lv: WithLanguageAndValue[String]): DisclaimerDTO = DisclaimerDTO(lv.value, lv.language) + implicit def encoder: Encoder[DisclaimerDTO] = deriveEncoder implicit def decoder: Decoder[DisclaimerDTO] = deriveDecoder } diff --git a/common/src/main/scala/no/ndla/common/model/domain/article/Article.scala b/common/src/main/scala/no/ndla/common/model/domain/article/Article.scala index f41cf62204..cbb162c417 100644 --- a/common/src/main/scala/no/ndla/common/model/domain/article/Article.scala +++ b/common/src/main/scala/no/ndla/common/model/domain/article/Article.scala @@ -11,8 +11,9 @@ package no.ndla.common.model.domain.article import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import io.circe.{Decoder, Encoder} import no.ndla.common.model.{NDLADate, RelatedContentLink} -import no.ndla.common.model.domain._ -import no.ndla.common.implicits._ +import no.ndla.common.model.domain.* +import no.ndla.common.implicits.* +import no.ndla.common.model.domain.language.OptLanguageFields case class Article( id: Option[Long], @@ -37,7 +38,7 @@ case class Article( relatedContent: Seq[RelatedContent], revisionDate: Option[NDLADate], slug: Option[String], - disclaimer: Option[Seq[Disclaimer]] + disclaimer: OptLanguageFields[String] ) extends Content object Article { diff --git a/common/src/main/scala/no/ndla/common/model/domain/draft/Draft.scala b/common/src/main/scala/no/ndla/common/model/domain/draft/Draft.scala index e4f3cce33a..a348d7e201 100644 --- a/common/src/main/scala/no/ndla/common/model/domain/draft/Draft.scala +++ b/common/src/main/scala/no/ndla/common/model/domain/draft/Draft.scala @@ -9,9 +9,10 @@ package no.ndla.common.model.domain.draft import io.circe.{Decoder, Encoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} -import no.ndla.common.implicits._ +import no.ndla.common.implicits.* import no.ndla.common.model.{NDLADate, RelatedContentLink} -import no.ndla.common.model.domain._ +import no.ndla.common.model.domain.* +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.language.Language.getSupportedLanguages case class Draft( @@ -46,7 +47,7 @@ case class Draft( priority: Priority, started: Boolean, qualityEvaluation: Option[QualityEvaluation], - disclaimer: Option[Seq[Disclaimer]] + disclaimer: OptLanguageFields[String] ) extends Content { def supportedLanguages: Seq[String] = diff --git a/common/src/main/scala/no/ndla/common/model/domain/language/LanguageFields.scala b/common/src/main/scala/no/ndla/common/model/domain/language/LanguageFields.scala index ba4fb89f7c..2d9080f80b 100644 --- a/common/src/main/scala/no/ndla/common/model/domain/language/LanguageFields.scala +++ b/common/src/main/scala/no/ndla/common/model/domain/language/LanguageFields.scala @@ -1,5 +1,5 @@ /* - * Part of NDLA backend.common.main + * Part of NDLA common * Copyright (C) 2025 NDLA * * See LICENSE @@ -13,7 +13,7 @@ import io.circe.syntax.EncoderOps import no.ndla.language.Language import no.ndla.language.model.{BaseWithLanguageAndValue, WithLanguageAndValue} -case class LanguageFields[T: Encoder: Decoder](internal: Map[String, T]) { +case class LanguageFields[T](internal: Map[String, T]) { def getWithLanguageFields: Seq[WithLanguageAndValue[T]] = internal.map { case (language, value) => BaseWithLanguageAndValue(language, value) }.toSeq @@ -24,16 +24,14 @@ case class LanguageFields[T: Encoder: Decoder](internal: Map[String, T]) { } object LanguageFields { - def empty[T: Encoder: Decoder]: LanguageFields[T] = LanguageFields(Map.empty) - def fromFields[T]( - fields: Seq[WithLanguageAndValue[T]] - )(implicit encoder: Encoder[T], decoder: Decoder[T]): LanguageFields[T] = { + def empty[T]: LanguageFields[T] = LanguageFields(Map.empty) + def fromFields[T](fields: Seq[WithLanguageAndValue[T]]): LanguageFields[T] = { val underlyingMap = fields.map(f => f.language -> f.value).toMap LanguageFields(underlyingMap) } implicit def encoder[T: Encoder]: Encoder[LanguageFields[T]] = Encoder.instance { lf => lf.internal.asJson } - implicit def decoder[T: Decoder: Encoder]: Decoder[LanguageFields[T]] = Decoder.instance { json => + implicit def decoder[T: Decoder]: Decoder[LanguageFields[T]] = Decoder.instance { json => json.as[Map[String, T]].map { m => LanguageFields(m) } } diff --git a/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala b/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala index ea27e0bc9b..65b340119a 100644 --- a/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala +++ b/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala @@ -1,5 +1,5 @@ /* - * Part of NDLA backend.common.main + * Part of NDLA common * Copyright (C) 2025 NDLA * * See LICENSE @@ -11,11 +11,14 @@ package no.ndla.common.model.domain.language import io.circe.syntax.EncoderOps import io.circe.{Decoder, Encoder, Json} import no.ndla.common.model.domain.language.OptionalLanguageValue.{NotWantedKey, NotWantedKeyT} -import no.ndla.language.model.WithLanguageAndValue +import no.ndla.language.Language +import no.ndla.language.model.{BaseWithLanguageAndValue, WithLanguageAndValue} +import sttp.tapir.Schema case class OptLanguageFields[T: Encoder: Decoder]( internal: Map[String, Either[NotWantedKeyT, Option[T]]] ) { + def get(language: String): Option[OptionalLanguageValue[T]] = { val res = internal.get(language) res match { @@ -26,13 +29,81 @@ case class OptLanguageFields[T: Encoder: Decoder]( } } + def map[R](f: WithLanguageAndValue[Option[T]] => R): Seq[R] = internal.map { case (language, value) => + value match { + case Right(Some(value)) => f(BaseWithLanguageAndValue(language, Some(value))) + case _ => f(BaseWithLanguageAndValue(language, None)) + } + }.toSeq + + def mapExisting[R](f: WithLanguageAndValue[T] => R): Seq[R] = internal.flatMap { case (language, value) => + value match { + case Right(Some(value)) => Some(f(BaseWithLanguageAndValue(language, value))) + case _ => None + } + }.toSeq + + def getWithLanguageFields: Seq[WithLanguageAndValue[T]] = internal.flatMap { case (language, value) => + value match { + case Right(Some(value)) => Some(BaseWithLanguageAndValue(language, value)) + case _ => None + } + }.toSeq + + def findByLanguageOrBestEffort(language: String): Option[WithLanguageAndValue[T]] = { + get(language) match { + case Some(Exists(value)) => Some(BaseWithLanguageAndValue(language, value)) + case Some(NotWanted()) => None + case None => Language.findByLanguageOrBestEffort(getWithLanguageFields, language) + } + } + def withUnwanted(language: String): OptLanguageFields[T] = { val updated: Map[String, Either[NotWantedKeyT, Option[T]]] = internal.updated(language, Left(NotWantedKey)) OptLanguageFields(updated) } + + def withValue(value: T, language: String): OptLanguageFields[T] = { + val updated: Map[String, Either[NotWantedKeyT, Option[T]]] = internal.updated(language, Right(Some(value))) + OptLanguageFields(updated) + } } object OptLanguageFields { + implicit val s1: Schema[OptLanguageFields[String]] = Schema.any + + def withUnwanted[T: Encoder: Decoder](language: String): OptLanguageFields[T] = { + val underlyingMap = Map(language -> Left(NotWantedKey)) + OptLanguageFields(underlyingMap) + } + + def withValue[T: Encoder: Decoder](value: T, language: String): OptLanguageFields[T] = { + val underlyingMap = Map(language -> Right(Some(value))) + OptLanguageFields(underlyingMap) + } + + implicit class optStringLanguageFields(s: OptLanguageFields[String]) { + def withOptValue(value: Option[String], language: String): OptLanguageFields[String] = { + value match { + case Some("") => s.withUnwanted(language) + case Some(v) => s.withValue(v, language) + case None => this.s + } + } + + def withOptValue(value: Option[String], language: Option[String]): OptLanguageFields[String] = language match { + case None => this.s + case Some(lang) => this.withOptValue(value, lang) + } + } + + def fromMaybeString(value: Option[String], language: String): OptLanguageFields[String] = { + value match { + case Some("") => withUnwanted(language) + case Some(v) => withValue(v, language) + case None => empty + } + } def fromFields[T]( fields: Seq[WithLanguageAndValue[T]] @@ -41,6 +112,8 @@ object OptLanguageFields { OptLanguageFields(underlyingMap) } + def empty[T: Encoder: Decoder]: OptLanguageFields[T] = OptLanguageFields(Map.empty) + implicit def eitherEncoder[T](implicit e: Encoder[T]): Encoder[Either[NotWantedKeyT, Option[T]]] = Encoder.instance { case Right(value) => value.asJson case Left(_) => Json.obj(NotWantedKey -> Json.True) diff --git a/common/src/main/scala/no/ndla/common/model/domain/language/OptionalLanguageValue.scala b/common/src/main/scala/no/ndla/common/model/domain/language/OptionalLanguageValue.scala index 0904023d01..ccb34279c6 100644 --- a/common/src/main/scala/no/ndla/common/model/domain/language/OptionalLanguageValue.scala +++ b/common/src/main/scala/no/ndla/common/model/domain/language/OptionalLanguageValue.scala @@ -10,20 +10,22 @@ package no.ndla.common.model.domain.language import io.circe.syntax.EncoderOps import io.circe.{Decoder, Encoder, HCursor, Json} +import sttp.tapir.Schema sealed trait OptionalLanguageValue[T] -case class Exists[T: Encoder: Decoder](value: T) extends OptionalLanguageValue[T] -case class NotWanted[T]() extends OptionalLanguageValue[T] +case class Exists[T](value: T) extends OptionalLanguageValue[T] +case class NotWanted[T]() extends OptionalLanguageValue[T] object OptionalLanguageValue { type NotWantedKeyT = "__notwanted__" - final val NotWantedKey = "__notwanted__" + final val NotWantedKey: NotWantedKeyT = "__notwanted__" + implicit val NotWantedSchema: Schema[NotWantedKeyT] = Schema.string implicit def encoder[T](implicit valueEncoder: Encoder[T]): Encoder[OptionalLanguageValue[T]] = Encoder.instance { case Exists(value) => Json.obj("value" -> value.asJson) case NotWanted() => Json.obj(NotWantedKey -> Json.True) } - implicit def decoder[T: Encoder: Decoder]: Decoder[OptionalLanguageValue[T]] = + implicit def decoder[T: Decoder]: Decoder[OptionalLanguageValue[T]] = (c: HCursor) => { c.downField(NotWantedKey).as[Option[Boolean]].flatMap { case Some(true) => Right(NotWanted()) diff --git a/common/src/test/scala/no/ndla/common/model/domain/LanguageFieldsTest.scala b/common/src/test/scala/no/ndla/common/model/domain/LanguageFieldsTest.scala index 4a8d474b31..474d359c39 100644 --- a/common/src/test/scala/no/ndla/common/model/domain/LanguageFieldsTest.scala +++ b/common/src/test/scala/no/ndla/common/model/domain/LanguageFieldsTest.scala @@ -44,6 +44,20 @@ class LanguageFieldsTest extends UnitTestSuite { languageFields.findByLanguageOrBestEffort("nn") should be(Some(BaseWithLanguageAndValue("nn", "nynorsk"))) } + test("That language fields are found by language or best effort according to language priority when opt") { + val fields = Seq( + BaseWithLanguageAndValue("nb", "bokmål"), + BaseWithLanguageAndValue("en", "english") + ) + + val languageFields = OptLanguageFields.fromFields(fields).withUnwanted("nn") + + languageFields.findByLanguageOrBestEffort("nb") should be(Some(BaseWithLanguageAndValue("nb", "bokmål"))) + languageFields.findByLanguageOrBestEffort("en") should be(Some(BaseWithLanguageAndValue("en", "english"))) + languageFields.findByLanguageOrBestEffort("sma") should be(Some(BaseWithLanguageAndValue("nb", "bokmål"))) + languageFields.findByLanguageOrBestEffort("nn") should be(None) + } + test("That the LanguageFields type is able to differentiate between a missing and not needed field") { val fields = Seq( @@ -53,26 +67,20 @@ class LanguageFieldsTest extends UnitTestSuite { val languageFields = LanguageFields.fromFields(fields) val jsonString = CirceUtil.toJsonString(languageFields) + val result = CirceUtil.unsafeParseAs[LanguageFields[OptionalLanguageValue[String]]](jsonString) - val result = CirceUtil.unsafeParseAs[LanguageFields[OptionalLanguageValue[String]]](jsonString) result should be(languageFields) - result.get("nb") should be(Some(BaseWithLanguageAndValue("nb", Exists("bokmål")))) result.get("nn") should be(Some(BaseWithLanguageAndValue("nn", NotWanted()))) } test("That the OptLanguageFields type is able to differentiate between a missing and not needed field") { - - val fields = Seq( - BaseWithLanguageAndValue[String]("nb", "bokmål") - ) - + val fields = Seq(BaseWithLanguageAndValue[String]("nb", "bokmål")) val languageFields = OptLanguageFields.fromFields(fields).withUnwanted("en") val jsonString = CirceUtil.toJsonString(languageFields) + val result = CirceUtil.unsafeParseAs[OptLanguageFields[String]](jsonString) - val result = CirceUtil.unsafeParseAs[OptLanguageFields[String]](jsonString) result should be(languageFields) - result.get("nb") should be(Some(Exists("bokmål"))) result.get("nn") should be(None) result.get("en") should be(Some(NotWanted())) diff --git a/database/src/main/scala/no/ndla/database/DocumentMigration.scala b/database/src/main/scala/no/ndla/database/DocumentMigration.scala index e3820532db..4f266db260 100644 --- a/database/src/main/scala/no/ndla/database/DocumentMigration.scala +++ b/database/src/main/scala/no/ndla/database/DocumentMigration.scala @@ -32,5 +32,5 @@ abstract class DocumentMigration extends TableMigration[DocumentRow] { .update() } - protected def convertColumn(value: String): String + def convertColumn(value: String): String } diff --git a/database/src/main/scala/no/ndla/database/LanguageFieldMigration.scala b/database/src/main/scala/no/ndla/database/LanguageFieldMigration.scala new file mode 100644 index 0000000000..0e05130dd7 --- /dev/null +++ b/database/src/main/scala/no/ndla/database/LanguageFieldMigration.scala @@ -0,0 +1,41 @@ +/* + * Part of NDLA database + * Copyright (C) 2025 NDLA + * + * See LICENSE + * + */ + +package no.ndla.database + +import io.circe.{Json, parser} + +abstract class LanguageFieldMigration extends DocumentMigration { + protected def fieldName: String + protected def oldSubfieldName: String = fieldName + + private def convertOldLanguageField(fields: Vector[Json]): Json = { + fields.foldLeft(Json.obj()) { (acc, disclaimer) => + val language = disclaimer.hcursor.downField("language").as[String].toTry.get + val text = disclaimer.hcursor.downField(oldSubfieldName).as[String].toTry.get + acc.mapObject(_.add(language, Json.fromString(text))) + } + } + + private def addEmptyLanguageField(obj: Json): String = { + obj.withObject(_.add(fieldName, Json.obj()).toJson).noSpaces + } + + override def convertColumn(document: String): String = { + val oldArticle = parser.parse(document).toTry.get + oldArticle.hcursor.downField(fieldName).focus match { + case None => addEmptyLanguageField(oldArticle) + case Some(f) if f.isNull => addEmptyLanguageField(oldArticle) + case Some(disclaimers) => + val disclaimerVector = disclaimers.asArray.get + val converted = convertOldLanguageField(disclaimerVector) + val newArticle = oldArticle.withObject(_.remove(fieldName).add(fieldName, converted).toJson) + newArticle.noSpaces + } + } +} diff --git a/draft-api/src/main/scala/no/ndla/draftapi/db/HtmlMigration.scala b/draft-api/src/main/scala/no/ndla/draftapi/db/HtmlMigration.scala index 2433d1aab4..7706bb81c5 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/db/HtmlMigration.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/db/HtmlMigration.scala @@ -28,7 +28,7 @@ abstract class HtmlMigration extends DocumentMigration { jsoupDocumentToString(converted) } - protected def convertColumn(document: String): String = { + def convertColumn(document: String): String = { val oldArticle = parser.parse(document).flatMap(_.as[Draft]).toTry.get val convertedContent = oldArticle.content.map(c => { val converted = convertContent(c.content, c.language) diff --git a/draft-api/src/main/scala/no/ndla/draftapi/db/migration/V67__DisclaimerToLanguageFields.scala b/draft-api/src/main/scala/no/ndla/draftapi/db/migration/V67__DisclaimerToLanguageFields.scala new file mode 100644 index 0000000000..bae99a2143 --- /dev/null +++ b/draft-api/src/main/scala/no/ndla/draftapi/db/migration/V67__DisclaimerToLanguageFields.scala @@ -0,0 +1,17 @@ +/* + * Part of NDLA article-api + * Copyright (C) 2025 NDLA + * + * See LICENSE + * + */ + +package no.ndla.draftapi.db.migration + +import no.ndla.database.LanguageFieldMigration + +class V67__DisclaimerToLanguageFields extends LanguageFieldMigration { + override val columnName: String = "document" + override val tableName: String = "articledata" + override val fieldName: String = "disclaimer" +} diff --git a/draft-api/src/main/scala/no/ndla/draftapi/service/ConverterService.scala b/draft-api/src/main/scala/no/ndla/draftapi/service/ConverterService.scala index 31057106b0..f9cf6c1129 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/service/ConverterService.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/service/ConverterService.scala @@ -16,6 +16,7 @@ import no.ndla.common.model.api.{Delete, DisclaimerDTO, DraftCopyrightDTO, Missi import no.ndla.common.model.domain.{ArticleContent, Priority, Responsible} import no.ndla.common.model.domain.draft.DraftStatus.{IMPORTED, PLANNED} import no.ndla.common.model.domain.draft.{Comment, Draft, DraftStatus} +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.common.model.{NDLADate, RelatedContentLink, api as commonApi, domain as common} import no.ndla.common.{Clock, UUIDUtil, model} import no.ndla.draftapi.Props @@ -55,7 +56,7 @@ trait ConverterService { val domainContent = newArticle.content .map(content => common.ArticleContent(removeUnknownEmbedTagAttribute(content), newArticle.language)) .toSeq - val domainDisclaimer = newArticle.disclaimer.map { d => Seq(common.Disclaimer(d, newArticle.language)) } + val domainDisclaimer = OptLanguageFields.fromMaybeString(newArticle.disclaimer, newArticle.language) val status = externalIds match { case Nil => common.Status(PLANNED, Set.empty) @@ -259,9 +260,6 @@ trait ConverterService { private def toDomainTitle(articleTitle: api.ArticleTitleDTO): common.Title = common.Title(articleTitle.title, articleTitle.language) - private def toDomainDisclaimer(articleDisclaimer: DisclaimerDTO): common.Disclaimer = - common.Disclaimer(articleDisclaimer.disclaimer, articleDisclaimer.language) - private def toDomainContent(articleContent: api.ArticleContentDTO): common.ArticleContent = { common.ArticleContent(removeUnknownEmbedTagAttribute(articleContent.content), articleContent.language) } @@ -391,8 +389,7 @@ trait ConverterService { val metaImage = findByLanguageOrBestEffort(article.metaImage, language).map(toApiArticleMetaImage) val revisionMetas = article.revisionMeta.map(toApiRevisionMeta) val responsible = article.responsible.map(toApiResponsible) - val disclaimer = - article.disclaimer.flatMap { d => findByLanguageOrBestEffort(d, language).map(toApiArticleDisclaimer) } + val disclaimer = article.disclaimer.findByLanguageOrBestEffort(language).map(DisclaimerDTO.fromLanguageValue) Success( api.ArticleDTO( @@ -461,9 +458,6 @@ trait ConverterService { def toApiArticleTitle(title: common.Title): api.ArticleTitleDTO = api.ArticleTitleDTO(Jsoup.parseBodyFragment(title.title).body().text(), title.title, title.language) - private def toApiArticleDisclaimer(disclaimer: common.Disclaimer): DisclaimerDTO = - DisclaimerDTO(disclaimer.disclaimer, disclaimer.language) - private def toApiArticleContent(content: common.ArticleContent): api.ArticleContentDTO = api.ArticleContentDTO(content.content, content.language) @@ -650,7 +644,8 @@ trait ConverterService { article.tags, article.introduction, article.metaDescription, - article.visualElement + article.visualElement, + article.disclaimer ) langFields.foldRight(false)((curr, res) => res || curr.isDefined || metaImageExists) @@ -794,15 +789,7 @@ trait ConverterService { .flatten ) - val updatedDisclaimer = articleWithNewContent.disclaimer match { - case None => toMergeInto.disclaimer - case Some(newDisclaimer) => - val updated = mergeLanguageFields( - toMergeInto.disclaimer.getOrElse(Seq.empty), - maybeLang.map(lang => toDomainDisclaimer(DisclaimerDTO(newDisclaimer, lang))).toSeq - ) - Option.when(updated.nonEmpty)(updated) - } + val updatedDisclaimer = toMergeInto.disclaimer.withOptValue(articleWithNewContent.disclaimer, maybeLang) val updatedContents = mergeLanguageFields( toMergeInto.content, @@ -958,7 +945,7 @@ trait ConverterService { priority = priority, started = false, qualityEvaluation = qualityEvaluationToDomain(article.qualityEvaluation), - disclaimer = article.disclaimer.map { d => Seq(common.Disclaimer(d, lang)) } + disclaimer = OptLanguageFields.fromMaybeString(article.disclaimer, lang) ) } diff --git a/draft-api/src/main/scala/no/ndla/draftapi/validation/ContentValidator.scala b/draft-api/src/main/scala/no/ndla/draftapi/validation/ContentValidator.scala index c9ff0c12bf..917de37f20 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/validation/ContentValidator.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/validation/ContentValidator.scala @@ -11,6 +11,7 @@ import no.ndla.common.errors.{ValidationException, ValidationMessage} import no.ndla.common.model.NDLADate import no.ndla.common.model.domain.* import no.ndla.common.model.domain.draft.* +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.draftapi.Props import no.ndla.draftapi.integration.ArticleApiClient import no.ndla.draftapi.model.api.{ContentIdDTO, NotFoundException, UpdatedArticleDTO} @@ -84,7 +85,7 @@ trait ContentValidator { if (shouldValidateEntireArticle) article.content.flatMap(c => validateArticleContent(c)) ++ article.introduction.flatMap(i => validateIntroduction(i)) ++ - validateArticleDisclaimer(article.disclaimer.getOrElse(Seq.empty)) ++ + validateArticleDisclaimer(article.disclaimer) ++ article.metaDescription.flatMap(m => validateMetaDescription(m)) ++ validateTitles(article.title) ++ article.copyright.map(x => validateCopyright(x)).toSeq.flatten ++ @@ -165,11 +166,12 @@ trait ContentValidator { validateLanguage("content.language", content.language) } - private def validateArticleDisclaimer(disclaimers: Seq[Disclaimer]): Seq[ValidationMessage] = { - disclaimers.flatMap(disclaimer => { - TextValidator.validate("disclaimer", disclaimer.disclaimer, allLegalTags).toList ++ + private def validateArticleDisclaimer(disclaimers: OptLanguageFields[String]): Seq[ValidationMessage] = { + disclaimers.mapExisting { disclaimer => + val field = s"disclaimer.${disclaimer.language}" + TextValidator.validate(field, disclaimer.value, allLegalTags).toList ++ validateLanguage("disclaimer.language", disclaimer.language) - }) + }.flatten } private def rootElementContainsOnlySectionBlocks(field: String, html: String): Option[ValidationMessage] = { diff --git a/draft-api/src/test/scala/no/ndla/draftapi/TestData.scala b/draft-api/src/test/scala/no/ndla/draftapi/TestData.scala index 27054f7a17..311a38c7a9 100644 --- a/draft-api/src/test/scala/no/ndla/draftapi/TestData.scala +++ b/draft-api/src/test/scala/no/ndla/draftapi/TestData.scala @@ -13,6 +13,7 @@ import no.ndla.common.model.api.{DraftCopyrightDTO, Missing} import no.ndla.common.model.domain.Priority import no.ndla.common.model.domain.draft.Draft import no.ndla.common.model.domain.draft.DraftStatus.* +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.common.model.{NDLADate, api as commonApi, domain as common} import no.ndla.draftapi.integration.{LearningPath, Title} import no.ndla.draftapi.model.api.* @@ -322,7 +323,7 @@ object TestData { Priority.Unspecified, false, None, - None + disclaimer = OptLanguageFields.empty ) val sampleArticleWithPublicDomain: Draft = Draft( @@ -357,7 +358,7 @@ object TestData { Priority.Unspecified, false, None, - None + disclaimer = OptLanguageFields.empty ) val sampleDomainArticle: Draft = Draft( @@ -394,7 +395,7 @@ object TestData { Priority.Unspecified, false, None, - None + disclaimer = OptLanguageFields.empty ) val newArticle: NewArticleDTO = api.NewArticleDTO( @@ -485,7 +486,7 @@ object TestData { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val apiArticleWithHtmlFaultV2: api.ArticleDTO = api.ArticleDTO( diff --git a/draft-api/src/test/scala/no/ndla/draftapi/service/ConverterServiceTest.scala b/draft-api/src/test/scala/no/ndla/draftapi/service/ConverterServiceTest.scala index 3b4c02ecf5..0fa6a06019 100644 --- a/draft-api/src/test/scala/no/ndla/draftapi/service/ConverterServiceTest.scala +++ b/draft-api/src/test/scala/no/ndla/draftapi/service/ConverterServiceTest.scala @@ -14,6 +14,7 @@ import no.ndla.common.model.api.{Delete, Missing, UpdateWith} import no.ndla.common.model.domain.* import no.ndla.common.model.domain.draft.DraftStatus.* import no.ndla.common.model.domain.draft.{Comment, Draft, DraftCopyright, DraftStatus} +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.common.model.{NDLADate, api as commonApi} import no.ndla.draftapi.model.api import no.ndla.draftapi.model.api.{NewCommentDTO, UpdatedCommentDTO} @@ -333,7 +334,7 @@ class ConverterServiceTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = Some(Seq(Disclaimer("Disclaimer test", "nb"))) + disclaimer = OptLanguageFields.withValue("Disclaimer test", "nb") ) val updatedNothing = TestData.blankUpdatedArticle.copy( @@ -381,7 +382,7 @@ class ConverterServiceTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = Some(Seq(Disclaimer("Disclaimer test", "nb"))) + disclaimer = OptLanguageFields.withValue("Disclaimer test", "nb") ) val expectedArticle = Draft( @@ -416,7 +417,7 @@ class ConverterServiceTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = Some(Seq(Disclaimer("NyDisclaimer test", "nb"))) + disclaimer = OptLanguageFields.withValue("NyDisclaimer test", "nb") ) val updatedEverything = TestData.blankUpdatedArticle.copy( @@ -482,7 +483,7 @@ class ConverterServiceTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val expectedArticle = Draft( @@ -525,7 +526,7 @@ class ConverterServiceTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val updatedEverything = TestData.blankUpdatedArticle.copy( @@ -1127,7 +1128,7 @@ class ConverterServiceTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = Some(Seq(Disclaimer("articleDisclaimer", "nb"))) + disclaimer = OptLanguageFields.withValue("articleDisclaimer", "nb") ) val article = common.model.domain.article.Article( id = Some(articleId), @@ -1153,7 +1154,7 @@ class ConverterServiceTest extends UnitSuite with TestEnvironment { relatedContent = Seq.empty, revisionDate = None, slug = Some("kjempe-slug"), - disclaimer = Some(Seq(Disclaimer("articleDisclaimer", "nb"))) + disclaimer = OptLanguageFields.withValue("articleDisclaimer", "nb") ) val result = service.toArticleApiArticle(draft) diff --git a/draft-api/src/test/scala/no/ndla/draftapi/service/StateTransitionRulesTest.scala b/draft-api/src/test/scala/no/ndla/draftapi/service/StateTransitionRulesTest.scala index bbad36f927..832324fab2 100644 --- a/draft-api/src/test/scala/no/ndla/draftapi/service/StateTransitionRulesTest.scala +++ b/draft-api/src/test/scala/no/ndla/draftapi/service/StateTransitionRulesTest.scala @@ -11,6 +11,7 @@ import no.ndla.common.errors.{ValidationException, ValidationMessage} import no.ndla.common.model.domain.{Priority, Responsible, Status} import no.ndla.common.model.domain.draft.Draft import no.ndla.common.model.domain.draft.DraftStatus.* +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.common.model.{NDLADate, domain as common} import no.ndla.draftapi.integration.{SearchHit, Title} import no.ndla.draftapi.model.domain.StateTransition @@ -356,7 +357,7 @@ class StateTransitionRulesTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val article = common.article.Article( id = Some(articleId), @@ -381,7 +382,7 @@ class StateTransitionRulesTest extends UnitSuite with TestEnvironment { relatedContent = Seq.empty, revisionDate = None, slug = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val status = common.Status(END_CONTROL, Set.empty) @@ -479,7 +480,7 @@ class StateTransitionRulesTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val status = common.Status(PLANNED, Set.empty) val transitionsToTest = StateTransitionRules.StateTransitions.filter(_.to == PUBLISHED) @@ -536,7 +537,7 @@ class StateTransitionRulesTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val status = common.Status(PLANNED, Set.empty) val transitionsToTest = StateTransitionRules.StateTransitions.filter(_.to == ARCHIVED) @@ -597,7 +598,7 @@ class StateTransitionRulesTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val status = common.Status(PLANNED, Set.empty) val transitionsToTest = StateTransitionRules.StateTransitions.filter(_.to == UNPUBLISHED) @@ -659,7 +660,7 @@ class StateTransitionRulesTest extends UnitSuite with TestEnvironment { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val status = common.Status(PUBLISHED, Set.empty) val transitionToTest: StateTransition = PUBLISHED -> IN_PROGRESS diff --git a/draft-api/src/test/scala/no/ndla/draftapi/validation/ContentValidatorTest.scala b/draft-api/src/test/scala/no/ndla/draftapi/validation/ContentValidatorTest.scala index f75a742335..d7e1d6f1fe 100644 --- a/draft-api/src/test/scala/no/ndla/draftapi/validation/ContentValidatorTest.scala +++ b/draft-api/src/test/scala/no/ndla/draftapi/validation/ContentValidatorTest.scala @@ -10,6 +10,7 @@ package no.ndla.draftapi.validation import no.ndla.common.errors.{ValidationException, ValidationMessage} import no.ndla.common.model.domain.* import no.ndla.common.model.domain.draft.{Comment, Draft, DraftCopyright, RevisionMeta} +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.draftapi.{TestData, TestEnvironment, UnitSuite} import no.ndla.mapping.License.CC_BY_SA @@ -63,21 +64,20 @@ class ContentValidatorTest extends UnitSuite with TestEnvironment { test("validateArticle should throw an error if disclaimer contains illegal HTML tags") { val article = articleToValidate.copy( content = Seq(ArticleContent(validDocument, "nb")), - disclaimer = Some(Seq(Disclaimer("

hei

", "nb"))) + disclaimer = OptLanguageFields.withValue("

hei

", "nb") ) val Failure(error: ValidationException) = contentValidator.validateArticle(article) - error should be( - ValidationException( - "disclaimer", - "The content contains illegal tags and/or attributes. Allowed HTML tags are: h3, msgroup, a, article, sub, sup, mtext, msrow, tbody, mtd, pre, thead, figcaption, mover, msup, semantics, ol, span, mroot, munder, h4, mscarries, dt, nav, mtr, ndlaembed, li, br, mrow, merror, mphantom, u, audio, ul, maligngroup, mfenced, annotation, div, strong, section, i, mspace, malignmark, mfrac, code, h2, td, aside, em, mstack, button, dl, th, tfoot, math, tr, b, blockquote, msline, col, annotation-xml, mstyle, caption, mpadded, mo, mlongdiv, msubsup, p, munderover, maction, menclose, h1, details, mmultiscripts, msqrt, mscarry, mstac, mi, mglyph, mlabeledtr, mtable, mprescripts, summary, mn, msub, ms, table, colgroup, dd" - ) + val expected = ValidationException( + "disclaimer.nb", + "The content contains illegal tags and/or attributes. Allowed HTML tags are: h3, msgroup, a, article, sub, sup, mtext, msrow, tbody, mtd, pre, thead, figcaption, mover, msup, semantics, ol, span, mroot, munder, h4, mscarries, dt, nav, mtr, ndlaembed, li, br, mrow, merror, mphantom, u, audio, ul, maligngroup, mfenced, annotation, div, strong, section, i, mspace, malignmark, mfrac, code, h2, td, aside, em, mstack, button, dl, th, tfoot, math, tr, b, blockquote, msline, col, annotation-xml, mstyle, caption, mpadded, mo, mlongdiv, msubsup, p, munderover, maction, menclose, h1, details, mmultiscripts, msqrt, mscarry, mstac, mi, mglyph, mlabeledtr, mtable, mprescripts, summary, mn, msub, ms, table, colgroup, dd" ) + error should be(expected) } test("validateArticle should not throw an error if disclaimer contains legal HTML tags") { val article = articleToValidate.copy( content = Seq(ArticleContent(validDocument, "nb")), - disclaimer = Some(Seq(Disclaimer(validDisclaimer, "nb"))) + disclaimer = OptLanguageFields.withValue(validDisclaimer, "nb") ) contentValidator.validateArticle(article).isSuccess should be(true) diff --git a/integration-tests/src/test/scala/no/ndla/integrationtests/draftapi/articleapi/ArticleApiClientTest.scala b/integration-tests/src/test/scala/no/ndla/integrationtests/draftapi/articleapi/ArticleApiClientTest.scala index a370e954b0..4740350f0f 100644 --- a/integration-tests/src/test/scala/no/ndla/integrationtests/draftapi/articleapi/ArticleApiClientTest.scala +++ b/integration-tests/src/test/scala/no/ndla/integrationtests/draftapi/articleapi/ArticleApiClientTest.scala @@ -10,6 +10,7 @@ package no.ndla.integrationtests.draftapi.articleapi import no.ndla.articleapi.ArticleApiProperties import no.ndla.common.model.domain.Priority import no.ndla.common.model.domain.draft.Draft +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.common.model.{NDLADate, domain as common} import no.ndla.draftapi.model.api.ContentIdDTO import no.ndla.integrationtests.UnitSuite @@ -126,7 +127,7 @@ class ArticleApiClientTest priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val exampleToken = diff --git a/language/src/main/scala/no/ndla/language/model/WithLanguageAndValue.scala b/language/src/main/scala/no/ndla/language/model/WithLanguageAndValue.scala index 64c4b823e1..e3787e3604 100644 --- a/language/src/main/scala/no/ndla/language/model/WithLanguageAndValue.scala +++ b/language/src/main/scala/no/ndla/language/model/WithLanguageAndValue.scala @@ -1,5 +1,5 @@ /* - * Part of NDLA backend.language.main + * Part of NDLA language * Copyright (C) 2025 NDLA * * See LICENSE diff --git a/learningpath-api/src/main/scala/no/ndla/learningpathapi/db/migration/V39__MadeAvailableForThePublished.scala b/learningpath-api/src/main/scala/no/ndla/learningpathapi/db/migration/V39__MadeAvailableForThePublished.scala index 82d4f17d82..fd0f7dc720 100644 --- a/learningpath-api/src/main/scala/no/ndla/learningpathapi/db/migration/V39__MadeAvailableForThePublished.scala +++ b/learningpath-api/src/main/scala/no/ndla/learningpathapi/db/migration/V39__MadeAvailableForThePublished.scala @@ -17,7 +17,7 @@ class V39__MadeAvailableForThePublished extends DocumentMigration { override val columnName: String = "document" override val tableName: String = "learningpaths" - protected def convertColumn(document: String): String = { + def convertColumn(document: String): String = { val oldLp = CirceUtil.unsafeParseAs[LearningPath](document) val madeAvailable = oldLp.status match { case UNLISTED | PUBLISHED => Some(oldLp.lastUpdated) diff --git a/learningpath-api/src/test/scala/no/ndla/learningpathapi/service/ConverterServiceTest.scala b/learningpath-api/src/test/scala/no/ndla/learningpathapi/service/ConverterServiceTest.scala index 0db88c0bf8..fff8b2430b 100644 --- a/learningpath-api/src/test/scala/no/ndla/learningpathapi/service/ConverterServiceTest.scala +++ b/learningpath-api/src/test/scala/no/ndla/learningpathapi/service/ConverterServiceTest.scala @@ -427,7 +427,7 @@ class ConverterServiceTest extends UnitSuite with UnitTestEnvironment { } test("asEmbedUrl throws error if an not allowed value for embedType is used") { - assertResult("Validation Error") { + assertResult("Validation Error:\n\tembedType: 'test' is not a valid embed type.") { intercept[ValidationException] { service.asEmbedUrlV2(api.EmbedUrlV2DTO("http://test.no/2/oembed/", "test"), "nb") }.getMessage diff --git a/search-api/src/main/scala/no/ndla/searchapi/controller/parameters/GetSearchQueryParams.scala b/search-api/src/main/scala/no/ndla/searchapi/controller/parameters/GetSearchQueryParams.scala index 97d123830e..fd951e3c67 100644 --- a/search-api/src/main/scala/no/ndla/searchapi/controller/parameters/GetSearchQueryParams.scala +++ b/search-api/src/main/scala/no/ndla/searchapi/controller/parameters/GetSearchQueryParams.scala @@ -1,5 +1,5 @@ /* - * Part of NDLA backend.search-api.main + * Part of NDLA search-api * Copyright (C) 2024 NDLA * * See LICENSE diff --git a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala index 3bf3d01477..2613dfb6cc 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/TestData.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/TestData.scala @@ -38,6 +38,7 @@ import no.ndla.common.model.domain.concept.{ WordClass } import no.ndla.common.model.domain.draft.{Draft, DraftCopyright, DraftStatus, RevisionMeta, RevisionStatus} +import no.ndla.common.model.domain.language.OptLanguageFields import no.ndla.common.model.domain.learningpath.{ LearningPath, LearningPathStatus, @@ -56,9 +57,9 @@ import no.ndla.searchapi.model.grep.{ GrepKjerneelement, GrepKompetansemaal, GrepLaererplan, + GrepTextObj, GrepTitle, - GrepTverrfagligTema, - GrepTextObj + GrepTverrfagligTema } import no.ndla.searchapi.model.search.* import no.ndla.searchapi.model.search.settings.{MultiDraftSearchSettings, SearchSettings} @@ -211,7 +212,7 @@ object TestData { Seq.empty, None, slug = None, - None + disclaimer = OptLanguageFields.empty ) val sampleDomainArticle: Article = Article( @@ -237,7 +238,7 @@ object TestData { Seq.empty, None, slug = None, - None + disclaimer = OptLanguageFields.empty ) val sampleDomainArticle2: Article = Article( @@ -263,7 +264,7 @@ object TestData { Seq.empty, None, slug = None, - None + disclaimer = OptLanguageFields.empty ) val sampleArticleWithByNcSa: Article = @@ -544,9 +545,9 @@ object TestData { conceptIds = Seq.empty, availability = Availability.everyone, relatedContent = Seq.empty, - None, + revisionDate = None, slug = None, - None + disclaimer = OptLanguageFields.empty ) val emptyDomainDraft: Draft = Draft( @@ -581,7 +582,7 @@ object TestData { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - None + disclaimer = OptLanguageFields.empty ) val draftStatus: Status = Status(DraftStatus.PLANNED, Set.empty) @@ -644,7 +645,7 @@ object TestData { priority = Priority.Unspecified, started = false, qualityEvaluation = None, - disclaimer = None + disclaimer = OptLanguageFields.empty ) val sampleDraftWithByNcSa: Draft = sampleDraftWithPublicDomain.copy(copyright = Some(draftByNcSaCopyright)) diff --git a/search-api/src/test/scala/no/ndla/searchapi/model/api/grep/GrepResultDTOTest.scala b/search-api/src/test/scala/no/ndla/searchapi/model/api/grep/GrepResultDTOTest.scala index 38948027e9..3516ff6794 100644 --- a/search-api/src/test/scala/no/ndla/searchapi/model/api/grep/GrepResultDTOTest.scala +++ b/search-api/src/test/scala/no/ndla/searchapi/model/api/grep/GrepResultDTOTest.scala @@ -1,5 +1,5 @@ /* - * Part of NDLA backend.search-api.test + * Part of NDLA search-api * Copyright (C) 2025 NDLA * * See LICENSE From ab6e40a88234c6b8ceabe63b171c964f7db9ea95 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Thu, 16 Jan 2025 13:37:01 +0100 Subject: [PATCH 3/4] database: Use generic naming in `LanguageFieldMigration` --- .../database/LanguageFieldMigration.scala | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/database/src/main/scala/no/ndla/database/LanguageFieldMigration.scala b/database/src/main/scala/no/ndla/database/LanguageFieldMigration.scala index 0e05130dd7..02e368ff7b 100644 --- a/database/src/main/scala/no/ndla/database/LanguageFieldMigration.scala +++ b/database/src/main/scala/no/ndla/database/LanguageFieldMigration.scala @@ -15,9 +15,9 @@ abstract class LanguageFieldMigration extends DocumentMigration { protected def oldSubfieldName: String = fieldName private def convertOldLanguageField(fields: Vector[Json]): Json = { - fields.foldLeft(Json.obj()) { (acc, disclaimer) => - val language = disclaimer.hcursor.downField("language").as[String].toTry.get - val text = disclaimer.hcursor.downField(oldSubfieldName).as[String].toTry.get + fields.foldLeft(Json.obj()) { (acc, field) => + val language = field.hcursor.downField("language").as[String].toTry.get + val text = field.hcursor.downField(oldSubfieldName).as[String].toTry.get acc.mapObject(_.add(language, Json.fromString(text))) } } @@ -27,14 +27,14 @@ abstract class LanguageFieldMigration extends DocumentMigration { } override def convertColumn(document: String): String = { - val oldArticle = parser.parse(document).toTry.get - oldArticle.hcursor.downField(fieldName).focus match { - case None => addEmptyLanguageField(oldArticle) - case Some(f) if f.isNull => addEmptyLanguageField(oldArticle) - case Some(disclaimers) => - val disclaimerVector = disclaimers.asArray.get - val converted = convertOldLanguageField(disclaimerVector) - val newArticle = oldArticle.withObject(_.remove(fieldName).add(fieldName, converted).toJson) + val oldDocument = parser.parse(document).toTry.get + oldDocument.hcursor.downField(fieldName).focus match { + case None => addEmptyLanguageField(oldDocument) + case Some(f) if f.isNull => addEmptyLanguageField(oldDocument) + case Some(values) => + val valueVector = values.asArray.get + val converted = convertOldLanguageField(valueVector) + val newArticle = oldDocument.withObject(_.remove(fieldName).add(fieldName, converted).toJson) newArticle.noSpaces } } From 2b73d89c05a5faaee4c93ee39afed53b69ab1f04 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Thu, 16 Jan 2025 14:16:08 +0100 Subject: [PATCH 4/4] draft-api: Remember to delete disclaimer when deleting language --- .../no/ndla/common/model/domain/draft/Draft.scala | 11 ++++++++++- .../model/domain/language/OptLanguageFields.scala | 5 +++++ .../no/ndla/draftapi/service/ConverterService.scala | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/common/src/main/scala/no/ndla/common/model/domain/draft/Draft.scala b/common/src/main/scala/no/ndla/common/model/domain/draft/Draft.scala index a348d7e201..23d23be1aa 100644 --- a/common/src/main/scala/no/ndla/common/model/domain/draft/Draft.scala +++ b/common/src/main/scala/no/ndla/common/model/domain/draft/Draft.scala @@ -51,7 +51,16 @@ case class Draft( ) extends Content { def supportedLanguages: Seq[String] = - getSupportedLanguages(title, visualElement, introduction, metaDescription, tags, content, metaImage) + getSupportedLanguages( + title, + visualElement, + introduction, + metaDescription, + tags, + content, + metaImage, + disclaimer.getWithLanguageFields + ) } object Draft { diff --git a/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala b/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala index 65b340119a..fa7ff3581e 100644 --- a/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala +++ b/common/src/main/scala/no/ndla/common/model/domain/language/OptLanguageFields.scala @@ -67,6 +67,11 @@ case class OptLanguageFields[T: Encoder: Decoder]( val updated: Map[String, Either[NotWantedKeyT, Option[T]]] = internal.updated(language, Right(Some(value))) OptLanguageFields(updated) } + + def dropLanguage(language: String): OptLanguageFields[T] = { + val newInternal = internal.removed(language) + OptLanguageFields(newInternal) + } } object OptLanguageFields { diff --git a/draft-api/src/main/scala/no/ndla/draftapi/service/ConverterService.scala b/draft-api/src/main/scala/no/ndla/draftapi/service/ConverterService.scala index f9cf6c1129..e1f5ade5e9 100644 --- a/draft-api/src/main/scala/no/ndla/draftapi/service/ConverterService.scala +++ b/draft-api/src/main/scala/no/ndla/draftapi/service/ConverterService.scala @@ -558,6 +558,7 @@ trait ConverterService { val tags = article.tags.filter(_.language != language) val metaImage = article.metaImage.filter(_.language != language) val visualElement = article.visualElement.filter(_.language != language) + val disclaimers = article.disclaimer.dropLanguage(language) newNotes(Seq(s"Slettet språkvariant '$language'."), userInfo, article.status) match { case Failure(ex) => Failure(ex) case Success(newEditorNotes) => @@ -570,7 +571,8 @@ trait ConverterService { tags = tags, metaImage = metaImage, visualElement = visualElement, - notes = article.notes ++ newEditorNotes + notes = article.notes ++ newEditorNotes, + disclaimer = disclaimers ) ) }