From 7cec561ab66c4f4d1cad3fc33903623ee103ad75 Mon Sep 17 00:00:00 2001 From: Jonas Natten Date: Fri, 8 Dec 2023 12:15:58 +0100 Subject: [PATCH] migration --- .../learningpathapi/ComponentRegistry.scala | 4 - .../learningpathapi/TestEnvironment.scala | 4 - .../repository/FolderRepositoryTest.scala | 7 +- .../db/migration/V2__folder_tables.sql | 11 +- ...onfig_tables.sql => V3__config_tables.sql} | 0 ..._forum_tables.sql => V4__forum_tables.sql} | 0 .../no/ndla/myndlaapi/ComponentRegistry.scala | 5 +- .../scala/no/ndla/myndlaapi/LpMigration.scala | 121 ++++++++++++++++ .../scala/no/ndla/myndlaapi/MainClass.scala | 11 +- .../ndla/myndlaapi/MyNdlaApiProperties.scala | 2 +- .../myndlaapi/integration/DataSource.scala | 17 ++- .../no/ndla/myndlaapi/TestEnvironment.scala | 31 ++++- .../repository/FolderRepositoryTest.scala | 5 +- .../no/ndla/myndla/model/domain/Folder.scala | 83 ++++++----- .../myndla/model/domain/FolderResource.scala | 35 +++-- .../ndla/myndla/model/domain/Resource.scala | 69 +++++----- .../model/domain/config/ConfigMeta.scala | 50 ++++--- .../myndla/repository/ConfigRepository.scala | 11 +- .../myndla/repository/FolderRepository.scala | 129 +++++++++++++++++- .../myndla/repository/UserRepository.scala | 7 + .../no/ndla/myndla/TestEnvironment.scala | 4 - 21 files changed, 434 insertions(+), 172 deletions(-) rename myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/{V4__config_tables.sql => V3__config_tables.sql} (100%) rename myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/{V3__forum_tables.sql => V4__forum_tables.sql} (100%) create mode 100644 myndla-api/src/main/scala/no/ndla/myndlaapi/LpMigration.scala diff --git a/learningpath-api/src/main/scala/no/ndla/learningpathapi/ComponentRegistry.scala b/learningpath-api/src/main/scala/no/ndla/learningpathapi/ComponentRegistry.scala index c917cf3efb..20388c6062 100644 --- a/learningpath-api/src/main/scala/no/ndla/learningpathapi/ComponentRegistry.scala +++ b/learningpath-api/src/main/scala/no/ndla/learningpathapi/ComponentRegistry.scala @@ -92,16 +92,12 @@ class ComponentRegistry(properties: LearningpathApiProperties) with SearchApiClient with Props with DBMigrator - with DBFolder - with DBResource - with DBFolderResource with TextValidator with UrlValidator with ErrorHelpers with LearningpathApiInfo with DBLearningPath with DBLearningStep - with DBConfigMeta with NdlaController with RedisClient { override val props: LearningpathApiProperties = properties diff --git a/learningpath-api/src/test/scala/no/ndla/learningpathapi/TestEnvironment.scala b/learningpath-api/src/test/scala/no/ndla/learningpathapi/TestEnvironment.scala index 117a137612..5c24dd022b 100644 --- a/learningpath-api/src/test/scala/no/ndla/learningpathapi/TestEnvironment.scala +++ b/learningpath-api/src/test/scala/no/ndla/learningpathapi/TestEnvironment.scala @@ -82,10 +82,6 @@ trait TestEnvironment with UrlValidator with DBLearningPath with DBLearningStep - with DBConfigMeta - with DBFolder - with DBResource - with DBFolderResource with NdlaController with ErrorHelpers with Props diff --git a/learningpath-api/src/test/scala/no/ndla/learningpathapi/repository/FolderRepositoryTest.scala b/learningpath-api/src/test/scala/no/ndla/learningpathapi/repository/FolderRepositoryTest.scala index b1101c7dcd..5f7e113254 100644 --- a/learningpath-api/src/test/scala/no/ndla/learningpathapi/repository/FolderRepositoryTest.scala +++ b/learningpath-api/src/test/scala/no/ndla/learningpathapi/repository/FolderRepositoryTest.scala @@ -11,7 +11,7 @@ import cats.implicits._ import com.zaxxer.hikari.HikariDataSource import no.ndla.common.model.NDLADate import no.ndla.learningpathapi.{TestData, TestEnvironment} -import no.ndla.myndla.model.domain.{DBFolderResource, Folder, FolderStatus, NewFolderData, ResourceDocument} +import no.ndla.myndla.model.domain.{DBFolder, DBFolderResource, DBResource, Folder, FolderStatus, NewFolderData, ResourceDocument} import no.ndla.scalatestsuite.IntegrationSuite import org.scalatest.Outcome import scalikejdbc._ @@ -20,10 +20,7 @@ import java.net.Socket import java.util.UUID import scala.util.{Failure, Success, Try} -class FolderRepositoryTest - extends IntegrationSuite(EnablePostgresContainer = true) - with TestEnvironment - with DBFolderResource { +class FolderRepositoryTest extends IntegrationSuite(EnablePostgresContainer = true) with TestEnvironment { override val dataSource: HikariDataSource = testDataSource.get override val migrator: DBMigrator = new DBMigrator var repository: FolderRepository = _ diff --git a/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V2__folder_tables.sql b/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V2__folder_tables.sql index 5f72203ab3..87d4cb01c4 100644 --- a/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V2__folder_tables.sql +++ b/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V2__folder_tables.sql @@ -19,9 +19,10 @@ CREATE TABLE folders ( updated timestamp NOT NULL DEFAULT now(), shared timestamp NULL, description text NULL, - CONSTRAINT folders_pkey PRIMARY KEY (id), - CONSTRAINT fk_parent_id FOREIGN KEY (parent_id) REFERENCES folders(id) + CONSTRAINT folders_pkey PRIMARY KEY (id) +-- CONSTRAINT fk_parent_id FOREIGN KEY (parent_id) REFERENCES folders(id) ); + CREATE INDEX folders_feide_id_idx ON folders USING btree (feide_id); CREATE INDEX folders_parent_id_idx ON folders USING btree (parent_id); @@ -29,7 +30,7 @@ CREATE TABLE folder_resources ( folder_id uuid NOT NULL, resource_id uuid NOT NULL, "rank" int4 NULL, - CONSTRAINT folder_resource_pkey PRIMARY KEY (folder_id, resource_id), - CONSTRAINT folder_resources_folder_id_fkey FOREIGN KEY (folder_id) REFERENCES folders(id) ON DELETE CASCADE, - CONSTRAINT folder_resources_resource_id_fkey FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE + CONSTRAINT folder_resource_pkey PRIMARY KEY (folder_id, resource_id) +-- CONSTRAINT folder_resources_folder_id_fkey FOREIGN KEY (folder_id) REFERENCES folders(id) ON DELETE CASCADE, +-- CONSTRAINT folder_resources_resource_id_fkey FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE ); diff --git a/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V4__config_tables.sql b/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V3__config_tables.sql similarity index 100% rename from myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V4__config_tables.sql rename to myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V3__config_tables.sql diff --git a/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V3__forum_tables.sql b/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V4__forum_tables.sql similarity index 100% rename from myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V3__forum_tables.sql rename to myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V4__forum_tables.sql diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala index 015e0cb894..d01be734bb 100644 --- a/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala @@ -62,10 +62,6 @@ class ComponentRegistry(properties: MyNdlaApiProperties) with ConfigService with UserRepository with ConfigRepository - with DBConfigMeta - with DBFolder - with DBResource - with DBFolderResource with FeideApiClient with ConfigController with RedisClient @@ -75,6 +71,7 @@ class ComponentRegistry(properties: MyNdlaApiProperties) override val props: MyNdlaApiProperties = properties override val dataSource: HikariDataSource = DataSource.getHikariDataSource + override val lpDs = DataSource.getLpDs DataSource.connectToDatabase() lazy val healthController = new TapirHealthController[Eff] diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/LpMigration.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/LpMigration.scala new file mode 100644 index 0000000000..2ab954a9af --- /dev/null +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/LpMigration.scala @@ -0,0 +1,121 @@ +/* + * Part of NDLA myndla-api + * Copyright (C) 2023 NDLA + * + * See LICENSE + */ + +package no.ndla.myndlaapi + +import com.typesafe.scalalogging.StrictLogging +import com.zaxxer.hikari.HikariDataSource +import no.ndla.common.Clock +import no.ndla.myndla.model.domain.MyNDLAUserDocument +import no.ndla.myndla.repository.{ConfigRepository, FolderRepository, UserRepository} +import scalikejdbc.{ + ConnectionPool, + DBSession, + DataSourceConnectionPool, + NamedDB, + scalikejdbcSQLInterpolationImplicitDef +} + +// TODO: Delete this when migration is done +case class LpMigration(props: MyNdlaApiProperties, localDataSource: HikariDataSource, lpDataSource: HikariDataSource) + extends FolderRepository + with UserRepository + with ConfigRepository + with Clock + with StrictLogging { + override val folderRepository: FolderRepository = new FolderRepository + override val userRepository: UserRepository = new UserRepository + override val configRepository: ConfigRepository = new ConfigRepository + override val clock: SystemClock = new SystemClock + + def migrateConfig(lpSession: DBSession, myndlaSession: DBSession): Unit = { + val allLpConfigs = configRepository.getAllConfigs(lpSession) + allLpConfigs.foreach(x => configRepository.updateConfigParam(x)(myndlaSession)) + } + + def migrateUsers(lpSession: DBSession, myndlaSession: DBSession): Unit = { + val oldUsers = userRepository.getAllUsers(lpSession) + oldUsers.foreach(x => + userRepository.insertUser( + x.feideId, + MyNDLAUserDocument( + favoriteSubjects = x.favoriteSubjects, + userRole = x.userRole, + lastUpdated = x.lastUpdated, + organization = x.organization, + groups = x.groups, + username = x.username, + displayName = x.displayName, + email = x.email, + arenaEnabled = x.arenaEnabled, + shareName = x.shareName + ) + )(myndlaSession) + ) + } + + def migrateFolders(lpSession: DBSession, myndlaSession: DBSession): Unit = { + val folders = folderRepository.getAllFolderRows(lpSession) + val resources = folderRepository.getAllResourceRows(lpSession) + val folderResources = folderRepository.getAllFolderResourceRows(lpSession) + + folders.foreach(x => folderRepository.insertFolderRow(x)(myndlaSession)) + resources.foreach(x => folderRepository.insertResourceRow(x)(myndlaSession)) + folderResources.foreach(x => folderRepository.insertFolderResourceRow(x)(myndlaSession)) + } + + def detectExistingMigration(myndlaSession: DBSession): Boolean = { + configRepository.getAllConfigs(myndlaSession).nonEmpty + + } + + def addForeignKeys(myndlasession: DBSession): Unit = { + // Foreign keys constraints fails with migration so we need to add them after the migration is done :^) + sql""" + ALTER TABLE folders + ADD CONSTRAINT fk_parent_id FOREIGN KEY (parent_id) REFERENCES folders(id); + """.execute()(myndlasession): Unit + + sql""" + ALTER TABLE folder_resources + ADD CONSTRAINT folder_resources_folder_id_fkey + FOREIGN KEY (folder_id) + REFERENCES folders(id) + ON DELETE CASCADE; + """.execute()(myndlasession): Unit + + sql""" + ALTER TABLE folder_resources + ADD CONSTRAINT folder_resources_resource_id_fkey + FOREIGN KEY (resource_id) + REFERENCES resources(id) + ON DELETE CASCADE; + """.execute()(myndlasession): Unit + } + + def start(): Unit = { + val lpPool = new DataSourceConnectionPool(lpDataSource) + val myNdlaPool = new DataSourceConnectionPool(localDataSource) + ConnectionPool.add(Symbol("learningpath"), lpPool) + ConnectionPool.add(Symbol("myndla"), myNdlaPool) + + NamedDB(Symbol("learningpath")).readOnly { lpSession => + NamedDB(Symbol("myndla")).localTx { myndlaSession => + if (!detectExistingMigration(myndlaSession)) { + logger.info("Doing learningpath-api -> myndla-api migration!") + + migrateConfig(lpSession, myndlaSession) + migrateUsers(lpSession, myndlaSession) + migrateFolders(lpSession, myndlaSession) + addForeignKeys(myndlaSession) + } else { + logger.info("Migration already done! Please delete `LpMigration` from `myndla-api`.") + } + } + } + } +} diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/MainClass.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/MainClass.scala index d2724935db..a10b19b6d7 100644 --- a/myndla-api/src/main/scala/no/ndla/myndlaapi/MainClass.scala +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/MainClass.scala @@ -25,8 +25,15 @@ class MainClass(override val props: MyNdlaApiProperties) extends NdlaTapirMain[E override def beforeStart(): Unit = { logger.info("Starting the db migration...") val startDBMillis = System.currentTimeMillis() - componentRegistry.migrator.migrate() - logger.info(s"Done db migration, took ${System.currentTimeMillis() - startDBMillis}ms") + if (props.migrateToLocalDB) { + componentRegistry.migrator.migrate() + logger.info(s"Done db migration, took ${System.currentTimeMillis() - startDBMillis}ms") + LpMigration(props, componentRegistry.dataSource, componentRegistry.lpDs).start() + } else { + logger.info( + "Skipping db migration, because we're running against learningpath-api db for now... use `LP_MIGRATE=true` to use local db." + ) + } } override def startServer(name: String, port: Int)(warmupFunc: => Unit): Unit = { diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/MyNdlaApiProperties.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/MyNdlaApiProperties.scala index bc2aeb8492..fc7b179cc7 100644 --- a/myndla-api/src/main/scala/no/ndla/myndlaapi/MyNdlaApiProperties.scala +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/MyNdlaApiProperties.scala @@ -45,5 +45,5 @@ class MyNdlaApiProperties extends BaseProps { def LpMetaSchema: String = prop("LP_" + PropertyKeys.MetaSchemaKey) def LpMetaMaxConnections: Int = propOrElse("LP_" + PropertyKeys.MetaMaxConnections, "10").toInt - def migrateToLocalDB: Boolean = propOrNone("MIGRATE_LP").exists(x => x.toLowerCase == "true") + def migrateToLocalDB: Boolean = propOrNone("LP_MIGRATE").exists(x => x.toLowerCase == "true") } diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/DataSource.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/DataSource.scala index be54751fd8..2c81eaed4f 100644 --- a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/DataSource.scala +++ b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/DataSource.scala @@ -15,6 +15,8 @@ import scalikejdbc.{ConnectionPool, DataSourceConnectionPool} trait DataSource { this: Props => val dataSource: HikariDataSource + val lpDs: HikariDataSource + import props._ object DataSource { @@ -22,26 +24,29 @@ trait DataSource { val dataSourceConfig = new HikariConfig() dataSourceConfig.setUsername(MetaUserName) dataSourceConfig.setPassword(MetaPassword) - dataSourceConfig.setJdbcUrl(s"jdbc:postgresql://$MetaServer:$MetaPort/$MetaResource") + dataSourceConfig.setJdbcUrl(s"jdbc:postgresql://$MetaServer:$MetaPort/$MetaResource?ApplicationName=$ApplicationName") dataSourceConfig.setDriverClassName("org.postgresql.Driver") dataSourceConfig.setSchema(MetaSchema) dataSourceConfig.setMaximumPoolSize(MetaMaxConnections) new HikariDataSource(dataSourceConfig) } - def connectToDatabase(): Unit = ConnectionPool.singleton(new DataSourceConnectionPool(dataSource)) - } + def connectToDatabase(): Unit = { + val ds = if (migrateToLocalDB) { dataSource } + else { lpDs } + ConnectionPool.singleton(new DataSourceConnectionPool(ds)) + } - class LearningpathDataSource { - def getDs: HikariDataSource = { + def getLpDs: HikariDataSource = { val dataSourceConfig = new HikariConfig() dataSourceConfig.setUsername(LpMetaUserName) dataSourceConfig.setPassword(LpMetaPassword) - dataSourceConfig.setJdbcUrl(s"jdbc:postgresql://$LpMetaServer:$LpMetaPort/$LpMetaResource") + dataSourceConfig.setJdbcUrl(s"jdbc:postgresql://$LpMetaServer:$LpMetaPort/$LpMetaResource?ApplicationName=$ApplicationName") dataSourceConfig.setDriverClassName("org.postgresql.Driver") dataSourceConfig.setSchema(LpMetaSchema) dataSourceConfig.setMaximumPoolSize(LpMetaMaxConnections) new HikariDataSource(dataSourceConfig) } } + } diff --git a/myndla-api/src/test/scala/no/ndla/myndlaapi/TestEnvironment.scala b/myndla-api/src/test/scala/no/ndla/myndlaapi/TestEnvironment.scala index f3eeb402bb..def76b1b98 100644 --- a/myndla-api/src/test/scala/no/ndla/myndlaapi/TestEnvironment.scala +++ b/myndla-api/src/test/scala/no/ndla/myndlaapi/TestEnvironment.scala @@ -12,12 +12,32 @@ import no.ndla.common.Clock import no.ndla.myndla.model.domain.{DBFolder, DBFolderResource, DBMyNDLAUser, DBResource} import no.ndla.myndla.model.domain.config.DBConfigMeta import no.ndla.myndla.repository.{ConfigRepository, FolderRepository, UserRepository} -import no.ndla.myndla.service.{ConfigService, FolderConverterService, FolderReadService, FolderWriteService, UserService} -import no.ndla.myndlaapi.controller.{ConfigController, ErrorHelpers, FolderController, StatsController, SwaggerDocControllerConfig, UserController} +import no.ndla.myndla.service.{ + ConfigService, + FolderConverterService, + FolderReadService, + FolderWriteService, + UserService +} +import no.ndla.myndlaapi.controller.{ + ConfigController, + ErrorHelpers, + FolderController, + StatsController, + SwaggerDocControllerConfig, + UserController +} import no.ndla.myndlaapi.integration.DataSource import no.ndla.myndlaapi.service.ReadService import no.ndla.network.clients.{FeideApiClient, RedisClient} -import no.ndla.network.tapir.{NdlaMiddleware, Routes, Service, SwaggerControllerConfig, TapirErrorHelpers, TapirHealthController} +import no.ndla.network.tapir.{ + NdlaMiddleware, + Routes, + Service, + SwaggerControllerConfig, + TapirErrorHelpers, + TapirHealthController +} import org.mockito.MockitoSugar.{mock, reset} trait TestEnvironment @@ -37,10 +57,6 @@ trait TestEnvironment with ConfigService with UserRepository with ConfigRepository - with DBConfigMeta - with DBFolder - with DBResource - with DBFolderResource with FeideApiClient with ConfigController with RedisClient @@ -54,6 +70,7 @@ trait TestEnvironment val props = new MyNdlaApiProperties lazy val clock: SystemClock = mock[SystemClock] val dataSource: HikariDataSource = mock[HikariDataSource] + val lpDs: HikariDataSource = mock[HikariDataSource] val migrator: DBMigrator = mock[DBMigrator] val readService: ReadService = mock[ReadService] val folderRepository: FolderRepository = mock[FolderRepository] diff --git a/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala b/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala index 4aea7eae65..095318e2e3 100644 --- a/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala +++ b/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala @@ -20,10 +20,7 @@ import java.net.Socket import java.util.UUID import scala.util.{Failure, Success, Try} -class FolderRepositoryTest - extends IntegrationSuite(EnablePostgresContainer = true) - with TestEnvironment - with DBFolderResource { +class FolderRepositoryTest extends IntegrationSuite(EnablePostgresContainer = true) with TestEnvironment { override val dataSource: HikariDataSource = testDataSource.get override val migrator: DBMigrator = new DBMigrator var repository: FolderRepository = _ diff --git a/myndla/src/main/scala/no/ndla/myndla/model/domain/Folder.scala b/myndla/src/main/scala/no/ndla/myndla/model/domain/Folder.scala index c5807af8ef..5abf17e664 100644 --- a/myndla/src/main/scala/no/ndla/myndla/model/domain/Folder.scala +++ b/myndla/src/main/scala/no/ndla/myndla/model/domain/Folder.scala @@ -78,54 +78,51 @@ case class Folder( } } -trait DBFolder { +object DBFolder extends SQLSyntaxSupport[Folder] { + implicit val jsonEncoder: Formats = DefaultFormats + new EnumNameSerializer(FolderStatus) + override val tableName = "folders" - object DBFolder extends SQLSyntaxSupport[Folder] { - implicit val jsonEncoder: Formats = DefaultFormats + new EnumNameSerializer(FolderStatus) - override val tableName = "folders" + val repositorySerializer: Formats = jsonEncoder + FieldSerializer[Folder]( + ignore("id") orElse + ignore("feideId") orElse + ignore("parentId") + ) - val repositorySerializer: Formats = jsonEncoder + FieldSerializer[Folder]( - ignore("id") orElse - ignore("feideId") orElse - ignore("parentId") - ) - - def fromResultSet(lp: SyntaxProvider[Folder])(rs: WrappedResultSet): Try[Folder] = - fromResultSet((s: String) => lp.resultName.c(s))(rs) + def fromResultSet(lp: SyntaxProvider[Folder])(rs: WrappedResultSet): Try[Folder] = + fromResultSet((s: String) => lp.resultName.c(s))(rs) - def fromResultSet(rs: WrappedResultSet): Try[Folder] = fromResultSet((s: String) => s)(rs) + def fromResultSet(rs: WrappedResultSet): Try[Folder] = fromResultSet((s: String) => s)(rs) - def fromResultSet(colNameWrapper: String => String)(rs: WrappedResultSet): Try[Folder] = { - import no.ndla.myndla.{maybeUuidBinder, uuidBinder} + def fromResultSet(colNameWrapper: String => String)(rs: WrappedResultSet): Try[Folder] = { + import no.ndla.myndla.{maybeUuidBinder, uuidBinder} - val id = rs.get[Try[UUID]](colNameWrapper("id")) - val parentId = rs.get[Option[UUID]](colNameWrapper("parent_id")) - val feideId = rs.string(colNameWrapper("feide_id")) - val name = rs.string(colNameWrapper("name")) - val status = FolderStatus.valueOfOrError(rs.string(colNameWrapper("status"))) - val description = rs.stringOpt(colNameWrapper("description")) - val rank = rs.intOpt(colNameWrapper("rank")) - val created = NDLADate.fromUtcDate(rs.localDateTime(colNameWrapper("created"))) - val updated = NDLADate.fromUtcDate(rs.localDateTime(colNameWrapper("updated"))) - val shared = rs.localDateTimeOpt(colNameWrapper("shared")).map(NDLADate.fromUtcDate) + val id = rs.get[Try[UUID]](colNameWrapper("id")) + val parentId = rs.get[Option[UUID]](colNameWrapper("parent_id")) + val feideId = rs.string(colNameWrapper("feide_id")) + val name = rs.string(colNameWrapper("name")) + val status = FolderStatus.valueOfOrError(rs.string(colNameWrapper("status"))) + val description = rs.stringOpt(colNameWrapper("description")) + val rank = rs.intOpt(colNameWrapper("rank")) + val created = NDLADate.fromUtcDate(rs.localDateTime(colNameWrapper("created"))) + val updated = NDLADate.fromUtcDate(rs.localDateTime(colNameWrapper("updated"))) + val shared = rs.localDateTimeOpt(colNameWrapper("shared")).map(NDLADate.fromUtcDate) - for { - id <- id - status <- status - } yield Folder( - id = id, - parentId = parentId, - feideId = feideId, - name = name, - status = status, - description = description, - resources = List.empty, - subfolders = List.empty, - rank = rank, - created = created, - updated = updated, - shared = shared - ) - } + for { + id <- id + status <- status + } yield Folder( + id = id, + parentId = parentId, + feideId = feideId, + name = name, + status = status, + description = description, + resources = List.empty, + subfolders = List.empty, + rank = rank, + created = created, + updated = updated, + shared = shared + ) } } diff --git a/myndla/src/main/scala/no/ndla/myndla/model/domain/FolderResource.scala b/myndla/src/main/scala/no/ndla/myndla/model/domain/FolderResource.scala index 8d31bc9429..640ea28ef5 100644 --- a/myndla/src/main/scala/no/ndla/myndla/model/domain/FolderResource.scala +++ b/myndla/src/main/scala/no/ndla/myndla/model/domain/FolderResource.scala @@ -18,24 +18,21 @@ case class FolderResource(folderId: UUID, resourceId: UUID, rank: Int) extends R override val sortRank: Option[Int] = Some(rank) } -trait DBFolderResource { - - object DBFolderResource extends SQLSyntaxSupport[FolderResource] { - implicit val formats = DefaultFormats - override val tableName = "folder_resources" - // lazy override val schemaName = Some(props.MetaSchema) - - def fromResultSet(lp: SyntaxProvider[FolderResource])(rs: WrappedResultSet): Try[FolderResource] = - fromResultSet(s => lp.resultName.c(s), rs) - - def fromResultSet(colNameWrapper: String => String, rs: WrappedResultSet): Try[FolderResource] = { - import no.ndla.myndla.uuidBinder - for { - folderId <- rs.get[Try[UUID]](colNameWrapper("folder_id")) - resourceId <- rs.get[Try[UUID]](colNameWrapper("resource_id")) - rank <- Try(rs.int(colNameWrapper("rank"))) - } yield FolderResource(folderId = folderId, resourceId = resourceId, rank = rank) - } - +object DBFolderResource extends SQLSyntaxSupport[FolderResource] { + implicit val formats = DefaultFormats + override val tableName = "folder_resources" + // lazy override val schemaName = Some(props.MetaSchema) + + def fromResultSet(lp: SyntaxProvider[FolderResource])(rs: WrappedResultSet): Try[FolderResource] = + fromResultSet(s => lp.resultName.c(s), rs) + + def fromResultSet(colNameWrapper: String => String, rs: WrappedResultSet): Try[FolderResource] = { + import no.ndla.myndla.uuidBinder + for { + folderId <- rs.get[Try[UUID]](colNameWrapper("folder_id")) + resourceId <- rs.get[Try[UUID]](colNameWrapper("resource_id")) + rank <- Try(rs.int(colNameWrapper("rank"))) + } yield FolderResource(folderId = folderId, resourceId = resourceId, rank = rank) } + } diff --git a/myndla/src/main/scala/no/ndla/myndla/model/domain/Resource.scala b/myndla/src/main/scala/no/ndla/myndla/model/domain/Resource.scala index eeb6e7b39d..f1afdde539 100644 --- a/myndla/src/main/scala/no/ndla/myndla/model/domain/Resource.scala +++ b/myndla/src/main/scala/no/ndla/myndla/model/domain/Resource.scala @@ -55,49 +55,46 @@ case class Resource( override val sortRank: Option[Int] = connection.map(_.rank) } -trait DBResource { - this: DBFolderResource => - object DBResource extends SQLSyntaxSupport[Resource] { - implicit val formats: Formats = DefaultFormats - override val tableName = "resources" +object DBResource extends SQLSyntaxSupport[Resource] { + implicit val formats: Formats = DefaultFormats + override val tableName = "resources" - val JSonSerializer: FieldSerializer[Resource] = FieldSerializer[Resource]( - ignore("id") orElse - ignore("feideId") orElse - ignore("created") - ) + val JSonSerializer: FieldSerializer[Resource] = FieldSerializer[Resource]( + ignore("id") orElse + ignore("feideId") orElse + ignore("created") + ) - def fromResultSet(lp: SyntaxProvider[Resource], withConnection: Boolean)(rs: WrappedResultSet): Try[Resource] = - fromResultSet(s => lp.resultName.c(s), withConnection)(rs) + def fromResultSet(lp: SyntaxProvider[Resource], withConnection: Boolean)(rs: WrappedResultSet): Try[Resource] = + fromResultSet(s => lp.resultName.c(s), withConnection)(rs) - def fromResultSetOpt(rs: WrappedResultSet, withConnection: Boolean): Try[Option[Resource]] = { - import no.ndla.myndla.maybeUuidBinder - rs.get[Option[UUID]]("resource_id").traverse(_ => fromResultSet(rs, withConnection)) - } + def fromResultSetOpt(rs: WrappedResultSet, withConnection: Boolean): Try[Option[Resource]] = { + import no.ndla.myndla.maybeUuidBinder + rs.get[Option[UUID]]("resource_id").traverse(_ => fromResultSet(rs, withConnection)) + } - private def fromResultSet(rs: WrappedResultSet, withConnection: Boolean): Try[Resource] = - fromResultSet(s => s, withConnection)(rs) + private def fromResultSet(rs: WrappedResultSet, withConnection: Boolean): Try[Resource] = + fromResultSet(s => s, withConnection)(rs) - private def fromResultSet( - colNameWrapper: String => String, - withConnection: Boolean - )(rs: WrappedResultSet): Try[Resource] = { - import no.ndla.myndla.uuidBinder + private def fromResultSet( + colNameWrapper: String => String, + withConnection: Boolean + )(rs: WrappedResultSet): Try[Resource] = { + import no.ndla.myndla.uuidBinder - val connection = - if (withConnection) DBFolderResource.fromResultSet(colNameWrapper, rs).toOption - else None + val connection = + if (withConnection) DBFolderResource.fromResultSet(colNameWrapper, rs).toOption + else None - for { - id <- rs.get[Try[UUID]](colNameWrapper("id")) - jsonString = rs.string(colNameWrapper("document")) - feideId = rs.string(colNameWrapper("feide_id")) - created = NDLADate.fromUtcDate(rs.localDateTime(colNameWrapper("created"))) - path = rs.string(colNameWrapper("path")) - resourceType = rs.string(colNameWrapper("resource_type")) - metaData <- Try(read[ResourceDocument](jsonString)) + for { + id <- rs.get[Try[UUID]](colNameWrapper("id")) + jsonString = rs.string(colNameWrapper("document")) + feideId = rs.string(colNameWrapper("feide_id")) + created = NDLADate.fromUtcDate(rs.localDateTime(colNameWrapper("created"))) + path = rs.string(colNameWrapper("path")) + resourceType = rs.string(colNameWrapper("resource_type")) + metaData <- Try(read[ResourceDocument](jsonString)) - } yield metaData.toFullResource(id, path, resourceType, feideId, created, connection) - } + } yield metaData.toFullResource(id, path, resourceType, feideId, created, connection) } } diff --git a/myndla/src/main/scala/no/ndla/myndla/model/domain/config/ConfigMeta.scala b/myndla/src/main/scala/no/ndla/myndla/model/domain/config/ConfigMeta.scala index e08e2a5444..ce50e52382 100644 --- a/myndla/src/main/scala/no/ndla/myndla/model/domain/config/ConfigMeta.scala +++ b/myndla/src/main/scala/no/ndla/myndla/model/domain/config/ConfigMeta.scala @@ -95,32 +95,30 @@ case class ConfigMeta( } } -trait DBConfigMeta { - object DBConfigMeta extends SQLSyntaxSupport[ConfigMeta] with StrictLogging { - implicit val formats: Formats = - org.json4s.DefaultFormats + - Json4s.serializer(ConfigKey) ++ - JavaTimeSerializers.all + - NDLADate.Json4sSerializer - - override val tableName = "configtable" - - def fromResultSet(c: SyntaxProvider[ConfigMeta])(rs: WrappedResultSet): ConfigMeta = fromResultSet(c.resultName)(rs) - import ConfigMetaValue._ - - implicit val enc: Encoder[ConfigMeta] = deriveEncoder[ConfigMeta] - implicit val dec: Decoder[ConfigMeta] = deriveDecoder[ConfigMeta] - - def fromResultSet(c: ResultName[ConfigMeta])(rs: WrappedResultSet): ConfigMeta = { - val dbStr = rs.string(c.column("value")) - val parsed = parse(dbStr) - parsed.flatMap(_.as[ConfigMeta]) match { - case Right(json) => json - case Left(err) => - logger.error(s"Could not parse json from database: '$dbStr'", err) - // TODO: Would love to propagate these errors via `Try/Either` instead of throwing an exception - throw new RuntimeException("Could not parse json from database") - } +object DBConfigMeta extends SQLSyntaxSupport[ConfigMeta] with StrictLogging { + implicit val formats: Formats = + org.json4s.DefaultFormats + + Json4s.serializer(ConfigKey) ++ + JavaTimeSerializers.all + + NDLADate.Json4sSerializer + + override val tableName = "configtable" + + def fromResultSet(c: SyntaxProvider[ConfigMeta])(rs: WrappedResultSet): ConfigMeta = fromResultSet(c.resultName)(rs) + import ConfigMetaValue._ + + implicit val enc: Encoder[ConfigMeta] = deriveEncoder[ConfigMeta] + implicit val dec: Decoder[ConfigMeta] = deriveDecoder[ConfigMeta] + + def fromResultSet(c: ResultName[ConfigMeta])(rs: WrappedResultSet): ConfigMeta = { + val dbStr = rs.string(c.column("value")) + val parsed = parse(dbStr) + parsed.flatMap(_.as[ConfigMeta]) match { + case Right(json) => json + case Left(err) => + logger.error(s"Could not parse json from database: '$dbStr'", err) + // TODO: Would love to propagate these errors via `Try/Either` instead of throwing an exception + throw new RuntimeException("Could not parse json from database") } } diff --git a/myndla/src/main/scala/no/ndla/myndla/repository/ConfigRepository.scala b/myndla/src/main/scala/no/ndla/myndla/repository/ConfigRepository.scala index 682d853a29..787e898b3c 100644 --- a/myndla/src/main/scala/no/ndla/myndla/repository/ConfigRepository.scala +++ b/myndla/src/main/scala/no/ndla/myndla/repository/ConfigRepository.scala @@ -17,7 +17,6 @@ import sqls.count import scala.util.{Success, Try} trait ConfigRepository { - this: DBConfigMeta => val configRepository: ConfigRepository import DBConfigMeta._ @@ -74,5 +73,15 @@ trait ConfigRepository { .map(DBConfigMeta.fromResultSet(c)) .single() } + + def getAllConfigs(implicit session: DBSession): List[ConfigMeta] = { + val c = DBConfigMeta.syntax("c") + sql""" + select ${c.result.*} + from ${DBConfigMeta.as(c)} + """ + .map(DBConfigMeta.fromResultSet(c)) + .list() + } } } diff --git a/myndla/src/main/scala/no/ndla/myndla/repository/FolderRepository.scala b/myndla/src/main/scala/no/ndla/myndla/repository/FolderRepository.scala index 72a7a399cf..f5d31b7b16 100644 --- a/myndla/src/main/scala/no/ndla/myndla/repository/FolderRepository.scala +++ b/myndla/src/main/scala/no/ndla/myndla/repository/FolderRepository.scala @@ -37,7 +37,7 @@ import java.util.UUID import scala.util.{Failure, Success, Try} trait FolderRepository { - this: DBFolder with DBResource with DBFolderResource with Clock => + this: Clock => val folderRepository: FolderRepository class FolderRepository extends StrictLogging { @@ -614,5 +614,132 @@ trait FolderRepository { .map(rs => (rs.long("antall"), rs.string("resource_type"))) .list() } + + case class FolderRow( + id: UUID, + parent_id: Option[UUID], + feide_id: Option[String], + rank: Option[Long], + name: String, + status: String, + created: NDLADate, + updated: NDLADate, + shared: Option[NDLADate], + description: Option[String] + ) + def getAllFolderRows(implicit session: DBSession): List[FolderRow] = { + sql"select * from ${DBFolder.table}" + .map(rs => { + FolderRow( + id = UUID.fromString(rs.string("id")), + parent_id = rs.stringOpt("parent_id").map(x => UUID.fromString(x)), + feide_id = rs.stringOpt("feide_id"), + rank = rs.longOpt("rank"), + name = rs.string("name"), + status = rs.string("status"), + created = NDLADate.fromUtcDate(rs.localDateTime("created")), + updated = NDLADate.fromUtcDate(rs.localDateTime("updated")), + shared = rs.localDateTimeOpt("shared").map(d => NDLADate.fromUtcDate(d)), + description = rs.stringOpt("description") + ) + }) + .list() + } + + case class ResourceRow( + id: UUID, + feide_id: String, + path: String, + resource_type: String, + created: NDLADate, + document: String + ) + def getAllResourceRows(implicit session: DBSession): List[ResourceRow] = { + sql"select * from ${DBResource.table}" + .map(rs => { + ResourceRow( + id = UUID.fromString(rs.string("id")), + feide_id = rs.string("feide_id"), + path = rs.string("path"), + resource_type = rs.string("resource_type"), + created = NDLADate.fromUtcDate(rs.localDateTime("created")), + document = rs.string("document") + ) + }) + .list() + } + + case class FolderResourceRow(folder_id: UUID, resource_id: UUID, rank: Int) + def getAllFolderResourceRows(implicit session: DBSession): List[FolderResourceRow] = { + sql"select * from ${DBFolderResource.table}" + .map(rs => { + FolderResourceRow( + folder_id = UUID.fromString(rs.string("folder_id")), + resource_id = UUID.fromString(rs.string("resource_id")), + rank = rs.int("rank") + ) + }) + .list() + } + + def insertFolderRow(folderRow: FolderRow)(implicit session: DBSession): Unit = { + val column = DBFolder.column.c _ + withSQL { + insert + .into(DBFolder) + .namedValues( + column("id") -> folderRow.id, + column("parent_id") -> folderRow.parent_id, + column("feide_id") -> folderRow.feide_id, + column("rank") -> folderRow.rank, + column("name") -> folderRow.name, + column("status") -> folderRow.status, + column("created") -> folderRow.created, + column("updated") -> folderRow.updated, + column("shared") -> folderRow.shared, + column("description") -> folderRow.description + ) + }.update(): Unit + logger.info(s"Inserted new folder with id ${folderRow.id}") + } + + def insertResourceRow(resourceRow: ResourceRow)(implicit session: DBSession): Unit = { + val jsonDocument = { + val dataObject = new PGobject() + dataObject.setType("jsonb") + dataObject.setValue(resourceRow.document) + ParameterBinder(dataObject, (ps, idx) => ps.setObject(idx, dataObject)) + } + val column = DBResource.column.c _ + withSQL { + insert + .into(DBResource) + .namedValues( + column("id") -> resourceRow.id, + column("feide_id") -> resourceRow.feide_id, + column("path") -> resourceRow.path, + column("resource_type") -> resourceRow.resource_type, + column("created") -> resourceRow.created, + column("document") -> jsonDocument + ) + }.update(): Unit + logger.info(s"Inserted new resource with id ${resourceRow.id}") + } + + def insertFolderResourceRow(folderResourceRow: FolderResourceRow)(implicit session: DBSession): Unit = { + val column = DBFolderResource.column.c _ + withSQL { + insert + .into(DBFolderResource) + .namedValues( + column("folder_id") -> folderResourceRow.folder_id, + column("resource_id") -> folderResourceRow.resource_id, + column("rank") -> folderResourceRow.rank + ) + }.update(): Unit + logger.info( + s"Inserted new folder resource with id ${folderResourceRow.folder_id}_${folderResourceRow.resource_id}" + ) + } } } diff --git a/myndla/src/main/scala/no/ndla/myndla/repository/UserRepository.scala b/myndla/src/main/scala/no/ndla/myndla/repository/UserRepository.scala index 6e7c0c7346..3dba17a7bd 100644 --- a/myndla/src/main/scala/no/ndla/myndla/repository/UserRepository.scala +++ b/myndla/src/main/scala/no/ndla/myndla/repository/UserRepository.scala @@ -105,5 +105,12 @@ trait UserRepository { .single() } + def getAllUsers(implicit session: DBSession): List[MyNDLAUser] = { + val u = DBMyNDLAUser.syntax("u") + sql"select ${u.result.*} from ${DBMyNDLAUser.as(u)}" + .map(DBMyNDLAUser.fromResultSet(u)) + .list() + } + } } diff --git a/myndla/src/test/scala/no/ndla/myndla/TestEnvironment.scala b/myndla/src/test/scala/no/ndla/myndla/TestEnvironment.scala index 92f6c34834..12183b50e9 100644 --- a/myndla/src/test/scala/no/ndla/myndla/TestEnvironment.scala +++ b/myndla/src/test/scala/no/ndla/myndla/TestEnvironment.scala @@ -29,13 +29,9 @@ trait TestEnvironment with RedisClient with ConfigService with ConfigRepository - with DBConfigMeta with UserRepository with FolderConverterService with FolderRepository - with DBFolder - with DBFolderResource - with DBResource with FolderReadService with FolderWriteService {