migration

NDLANO · Dec 8, 2023 · 7cec561 · 7cec561
1 parent 1aff677
commit 7cec561
Show file tree

Hide file tree

Showing 21 changed files with 434 additions and 172 deletions.
diff --git a/learningpath-api/src/main/scala/no/ndla/learningpathapi/ComponentRegistry.scala b/learningpath-api/src/main/scala/no/ndla/learningpathapi/ComponentRegistry.scala
@@ -92,16 +92,12 @@ class ComponentRegistry(properties: LearningpathApiProperties)
     with SearchApiClient
     with Props
     with DBMigrator
-    with DBFolder
-    with DBResource
-    with DBFolderResource
     with TextValidator
     with UrlValidator
     with ErrorHelpers
     with LearningpathApiInfo
     with DBLearningPath
     with DBLearningStep
-    with DBConfigMeta
     with NdlaController
     with RedisClient {
   override val props: LearningpathApiProperties = properties

diff --git a/learningpath-api/src/test/scala/no/ndla/learningpathapi/TestEnvironment.scala b/learningpath-api/src/test/scala/no/ndla/learningpathapi/TestEnvironment.scala
@@ -82,10 +82,6 @@ trait TestEnvironment
     with UrlValidator
     with DBLearningPath
     with DBLearningStep
-    with DBConfigMeta
-    with DBFolder
-    with DBResource
-    with DBFolderResource
     with NdlaController
     with ErrorHelpers
     with Props

diff --git a/...ningpath-api/src/test/scala/no/ndla/learningpathapi/repository/FolderRepositoryTest.scala b/...ningpath-api/src/test/scala/no/ndla/learningpathapi/repository/FolderRepositoryTest.scala
@@ -11,7 +11,7 @@ import cats.implicits._
 import com.zaxxer.hikari.HikariDataSource
 import no.ndla.common.model.NDLADate
 import no.ndla.learningpathapi.{TestData, TestEnvironment}
-import no.ndla.myndla.model.domain.{DBFolderResource, Folder, FolderStatus, NewFolderData, ResourceDocument}
+import no.ndla.myndla.model.domain.{DBFolder, DBFolderResource, DBResource, Folder, FolderStatus, NewFolderData, ResourceDocument}
 import no.ndla.scalatestsuite.IntegrationSuite
 import org.scalatest.Outcome
 import scalikejdbc._
@@ -20,10 +20,7 @@ import java.net.Socket
 import java.util.UUID
 import scala.util.{Failure, Success, Try}
 
-class FolderRepositoryTest
-    extends IntegrationSuite(EnablePostgresContainer = true)
-    with TestEnvironment
-    with DBFolderResource {
+class FolderRepositoryTest extends IntegrationSuite(EnablePostgresContainer = true) with TestEnvironment {
   override val dataSource: HikariDataSource = testDataSource.get
   override val migrator: DBMigrator         = new DBMigrator
   var repository: FolderRepository          = _

diff --git a/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V2__folder_tables.sql b/myndla-api/src/main/resources/no/ndla/myndlaapi/db/migration/V2__folder_tables.sql
@@ -19,17 +19,18 @@ CREATE TABLE folders (
 	updated timestamp NOT NULL DEFAULT now(),
 	shared timestamp NULL,
 	description text NULL,
-	CONSTRAINT folders_pkey PRIMARY KEY (id),
-	CONSTRAINT fk_parent_id FOREIGN KEY (parent_id) REFERENCES folders(id)
+	CONSTRAINT folders_pkey PRIMARY KEY (id)
+-- 	CONSTRAINT fk_parent_id FOREIGN KEY (parent_id) REFERENCES folders(id)
 );
+
 CREATE INDEX folders_feide_id_idx ON folders USING btree (feide_id);
 CREATE INDEX folders_parent_id_idx ON folders USING btree (parent_id);
 
 CREATE TABLE folder_resources (
 	folder_id uuid NOT NULL,
 	resource_id uuid NOT NULL,
 	"rank" int4 NULL,
-	CONSTRAINT folder_resource_pkey PRIMARY KEY (folder_id, resource_id),
-	CONSTRAINT folder_resources_folder_id_fkey FOREIGN KEY (folder_id) REFERENCES folders(id) ON DELETE CASCADE,
-	CONSTRAINT folder_resources_resource_id_fkey FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE
+	CONSTRAINT folder_resource_pkey PRIMARY KEY (folder_id, resource_id)
+-- 	CONSTRAINT folder_resources_folder_id_fkey FOREIGN KEY (folder_id) REFERENCES folders(id) ON DELETE CASCADE,
+-- 	CONSTRAINT folder_resources_resource_id_fkey FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE
 );
diff --git a/...dlaapi/db/migration/V4__config_tables.sql → ...dlaapi/db/migration/V3__config_tables.sql b/...dlaapi/db/migration/V4__config_tables.sql → ...dlaapi/db/migration/V3__config_tables.sql
diff --git a/...ndlaapi/db/migration/V3__forum_tables.sql → ...ndlaapi/db/migration/V4__forum_tables.sql b/...ndlaapi/db/migration/V3__forum_tables.sql → ...ndlaapi/db/migration/V4__forum_tables.sql
diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/ComponentRegistry.scala
@@ -62,10 +62,6 @@ class ComponentRegistry(properties: MyNdlaApiProperties)
     with ConfigService
     with UserRepository
     with ConfigRepository
-    with DBConfigMeta
-    with DBFolder
-    with DBResource
-    with DBFolderResource
     with FeideApiClient
     with ConfigController
     with RedisClient
@@ -75,6 +71,7 @@ class ComponentRegistry(properties: MyNdlaApiProperties)
   override val props: MyNdlaApiProperties = properties
 
   override val dataSource: HikariDataSource = DataSource.getHikariDataSource
+  override val lpDs                         = DataSource.getLpDs
   DataSource.connectToDatabase()
 
   lazy val healthController                               = new TapirHealthController[Eff]

diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/LpMigration.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/LpMigration.scala
@@ -0,0 +1,121 @@
+/*
+ * Part of NDLA myndla-api
+ * Copyright (C) 2023 NDLA
+ *
+ * See LICENSE
+ */
+
+package no.ndla.myndlaapi
+
+import com.typesafe.scalalogging.StrictLogging
+import com.zaxxer.hikari.HikariDataSource
+import no.ndla.common.Clock
+import no.ndla.myndla.model.domain.MyNDLAUserDocument
+import no.ndla.myndla.repository.{ConfigRepository, FolderRepository, UserRepository}
+import scalikejdbc.{
+  ConnectionPool,
+  DBSession,
+  DataSourceConnectionPool,
+  NamedDB,
+  scalikejdbcSQLInterpolationImplicitDef
+}
+
+// TODO: Delete this when migration is done
+case class LpMigration(props: MyNdlaApiProperties, localDataSource: HikariDataSource, lpDataSource: HikariDataSource)
+    extends FolderRepository
+    with UserRepository
+    with ConfigRepository
+    with Clock
+    with StrictLogging {
+  override val folderRepository: FolderRepository = new FolderRepository
+  override val userRepository: UserRepository     = new UserRepository
+  override val configRepository: ConfigRepository = new ConfigRepository
+  override val clock: SystemClock                 = new SystemClock
+
+  def migrateConfig(lpSession: DBSession, myndlaSession: DBSession): Unit = {
+    val allLpConfigs = configRepository.getAllConfigs(lpSession)
+    allLpConfigs.foreach(x => configRepository.updateConfigParam(x)(myndlaSession))
+  }
+
+  def migrateUsers(lpSession: DBSession, myndlaSession: DBSession): Unit = {
+    val oldUsers = userRepository.getAllUsers(lpSession)
+    oldUsers.foreach(x =>
+      userRepository.insertUser(
+        x.feideId,
+        MyNDLAUserDocument(
+          favoriteSubjects = x.favoriteSubjects,
+          userRole = x.userRole,
+          lastUpdated = x.lastUpdated,
+          organization = x.organization,
+          groups = x.groups,
+          username = x.username,
+          displayName = x.displayName,
+          email = x.email,
+          arenaEnabled = x.arenaEnabled,
+          shareName = x.shareName
+        )
+      )(myndlaSession)
+    )
+  }
+
+  def migrateFolders(lpSession: DBSession, myndlaSession: DBSession): Unit = {
+    val folders         = folderRepository.getAllFolderRows(lpSession)
+    val resources       = folderRepository.getAllResourceRows(lpSession)
+    val folderResources = folderRepository.getAllFolderResourceRows(lpSession)
+
+    folders.foreach(x => folderRepository.insertFolderRow(x)(myndlaSession))
+    resources.foreach(x => folderRepository.insertResourceRow(x)(myndlaSession))
+    folderResources.foreach(x => folderRepository.insertFolderResourceRow(x)(myndlaSession))
+  }
+
+  def detectExistingMigration(myndlaSession: DBSession): Boolean = {
+    configRepository.getAllConfigs(myndlaSession).nonEmpty
+
+  }
+
+  def addForeignKeys(myndlasession: DBSession): Unit = {
+    // Foreign keys constraints fails with migration so we need to add them after the migration is done :^)
+    sql"""
+      ALTER TABLE folders
+      ADD CONSTRAINT fk_parent_id FOREIGN KEY (parent_id) REFERENCES folders(id);
+    """.execute()(myndlasession): Unit
+
+    sql"""
+      ALTER TABLE folder_resources
+      ADD CONSTRAINT folder_resources_folder_id_fkey
+      FOREIGN KEY (folder_id)
+      REFERENCES folders(id)
+      ON DELETE CASCADE;
+    """.execute()(myndlasession): Unit
+
+    sql"""
+      ALTER TABLE folder_resources
+      ADD CONSTRAINT folder_resources_resource_id_fkey
+      FOREIGN KEY (resource_id)
+      REFERENCES resources(id)
+      ON DELETE CASCADE;
+    """.execute()(myndlasession): Unit
+  }
+
+  def start(): Unit = {
+    val lpPool     = new DataSourceConnectionPool(lpDataSource)
+    val myNdlaPool = new DataSourceConnectionPool(localDataSource)
+    ConnectionPool.add(Symbol("learningpath"), lpPool)
+    ConnectionPool.add(Symbol("myndla"), myNdlaPool)
+
+    NamedDB(Symbol("learningpath")).readOnly { lpSession =>
+      NamedDB(Symbol("myndla")).localTx { myndlaSession =>
+        if (!detectExistingMigration(myndlaSession)) {
+          logger.info("Doing learningpath-api -> myndla-api migration!")
+
+          migrateConfig(lpSession, myndlaSession)
+          migrateUsers(lpSession, myndlaSession)
+          migrateFolders(lpSession, myndlaSession)
+          addForeignKeys(myndlaSession)
+        } else {
+          logger.info("Migration already done! Please delete `LpMigration` from `myndla-api`.")
+        }
+      }
+    }
+  }
+}
diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/MainClass.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/MainClass.scala
@@ -25,8 +25,15 @@ class MainClass(override val props: MyNdlaApiProperties) extends NdlaTapirMain[E
   override def beforeStart(): Unit = {
     logger.info("Starting the db migration...")
     val startDBMillis = System.currentTimeMillis()
-    componentRegistry.migrator.migrate()
-    logger.info(s"Done db migration, took ${System.currentTimeMillis() - startDBMillis}ms")
+    if (props.migrateToLocalDB) {
+      componentRegistry.migrator.migrate()
+      logger.info(s"Done db migration, took ${System.currentTimeMillis() - startDBMillis}ms")
+      LpMigration(props, componentRegistry.dataSource, componentRegistry.lpDs).start()
+    } else {
+      logger.info(
+        "Skipping db migration, because we're running against learningpath-api db for now... use `LP_MIGRATE=true` to use local db."
+      )
+    }
   }
 
   override def startServer(name: String, port: Int)(warmupFunc: => Unit): Unit = {

diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/MyNdlaApiProperties.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/MyNdlaApiProperties.scala
@@ -45,5 +45,5 @@ class MyNdlaApiProperties extends BaseProps {
   def LpMetaSchema: String      = prop("LP_" + PropertyKeys.MetaSchemaKey)
   def LpMetaMaxConnections: Int = propOrElse("LP_" + PropertyKeys.MetaMaxConnections, "10").toInt
 
-  def migrateToLocalDB: Boolean = propOrNone("MIGRATE_LP").exists(x => x.toLowerCase == "true")
+  def migrateToLocalDB: Boolean = propOrNone("LP_MIGRATE").exists(x => x.toLowerCase == "true")
 }
diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/DataSource.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/DataSource.scala
@@ -15,33 +15,38 @@ import scalikejdbc.{ConnectionPool, DataSourceConnectionPool}
 trait DataSource {
   this: Props =>
   val dataSource: HikariDataSource
+  val lpDs: HikariDataSource
+
   import props._
   object DataSource {
 
     def getHikariDataSource: HikariDataSource = {
       val dataSourceConfig = new HikariConfig()
       dataSourceConfig.setUsername(MetaUserName)
       dataSourceConfig.setPassword(MetaPassword)
-      dataSourceConfig.setJdbcUrl(s"jdbc:postgresql://$MetaServer:$MetaPort/$MetaResource")
+      dataSourceConfig.setJdbcUrl(s"jdbc:postgresql://$MetaServer:$MetaPort/$MetaResource?ApplicationName=$ApplicationName")
       dataSourceConfig.setDriverClassName("org.postgresql.Driver")
       dataSourceConfig.setSchema(MetaSchema)
       dataSourceConfig.setMaximumPoolSize(MetaMaxConnections)
       new HikariDataSource(dataSourceConfig)
     }
 
-    def connectToDatabase(): Unit = ConnectionPool.singleton(new DataSourceConnectionPool(dataSource))
-  }
+    def connectToDatabase(): Unit = {
+      val ds = if (migrateToLocalDB) { dataSource }
+      else { lpDs }
+      ConnectionPool.singleton(new DataSourceConnectionPool(ds))
+    }
 
-  class LearningpathDataSource {
-    def getDs: HikariDataSource = {
+    def getLpDs: HikariDataSource = {
       val dataSourceConfig = new HikariConfig()
       dataSourceConfig.setUsername(LpMetaUserName)
       dataSourceConfig.setPassword(LpMetaPassword)
-      dataSourceConfig.setJdbcUrl(s"jdbc:postgresql://$LpMetaServer:$LpMetaPort/$LpMetaResource")
+      dataSourceConfig.setJdbcUrl(s"jdbc:postgresql://$LpMetaServer:$LpMetaPort/$LpMetaResource?ApplicationName=$ApplicationName")
       dataSourceConfig.setDriverClassName("org.postgresql.Driver")
       dataSourceConfig.setSchema(LpMetaSchema)
       dataSourceConfig.setMaximumPoolSize(LpMetaMaxConnections)
       new HikariDataSource(dataSourceConfig)
     }
   }
+
 }
diff --git a/myndla-api/src/test/scala/no/ndla/myndlaapi/TestEnvironment.scala b/myndla-api/src/test/scala/no/ndla/myndlaapi/TestEnvironment.scala
@@ -12,12 +12,32 @@ import no.ndla.common.Clock
 import no.ndla.myndla.model.domain.{DBFolder, DBFolderResource, DBMyNDLAUser, DBResource}
 import no.ndla.myndla.model.domain.config.DBConfigMeta
 import no.ndla.myndla.repository.{ConfigRepository, FolderRepository, UserRepository}
-import no.ndla.myndla.service.{ConfigService, FolderConverterService, FolderReadService, FolderWriteService, UserService}
-import no.ndla.myndlaapi.controller.{ConfigController, ErrorHelpers, FolderController, StatsController, SwaggerDocControllerConfig, UserController}
+import no.ndla.myndla.service.{
+  ConfigService,
+  FolderConverterService,
+  FolderReadService,
+  FolderWriteService,
+  UserService
+}
+import no.ndla.myndlaapi.controller.{
+  ConfigController,
+  ErrorHelpers,
+  FolderController,
+  StatsController,
+  SwaggerDocControllerConfig,
+  UserController
+}
 import no.ndla.myndlaapi.integration.DataSource
 import no.ndla.myndlaapi.service.ReadService
 import no.ndla.network.clients.{FeideApiClient, RedisClient}
-import no.ndla.network.tapir.{NdlaMiddleware, Routes, Service, SwaggerControllerConfig, TapirErrorHelpers, TapirHealthController}
+import no.ndla.network.tapir.{
+  NdlaMiddleware,
+  Routes,
+  Service,
+  SwaggerControllerConfig,
+  TapirErrorHelpers,
+  TapirHealthController
+}
 import org.mockito.MockitoSugar.{mock, reset}
 
 trait TestEnvironment
@@ -37,10 +57,6 @@ trait TestEnvironment
     with ConfigService
     with UserRepository
     with ConfigRepository
-    with DBConfigMeta
-    with DBFolder
-    with DBResource
-    with DBFolderResource
     with FeideApiClient
     with ConfigController
     with RedisClient
@@ -54,6 +70,7 @@ trait TestEnvironment
   val props                                          = new MyNdlaApiProperties
   lazy val clock: SystemClock                        = mock[SystemClock]
   val dataSource: HikariDataSource                   = mock[HikariDataSource]
+  val lpDs: HikariDataSource                         = mock[HikariDataSource]
   val migrator: DBMigrator                           = mock[DBMigrator]
   val readService: ReadService                       = mock[ReadService]
   val folderRepository: FolderRepository             = mock[FolderRepository]

diff --git a/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala b/myndla-api/src/test/scala/no/ndla/myndlaapi/repository/FolderRepositoryTest.scala
@@ -20,10 +20,7 @@ import java.net.Socket
 import java.util.UUID
 import scala.util.{Failure, Success, Try}
 
-class FolderRepositoryTest
-    extends IntegrationSuite(EnablePostgresContainer = true)
-    with TestEnvironment
-    with DBFolderResource {
+class FolderRepositoryTest extends IntegrationSuite(EnablePostgresContainer = true) with TestEnvironment {
   override val dataSource: HikariDataSource = testDataSource.get
   override val migrator: DBMigrator         = new DBMigrator
   var repository: FolderRepository          = _