Simpler migration code

NDLANO · Dec 9, 2024 · 8235ec9 · 8235ec9
1 parent 86f433e
commit 8235ec9
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 51 deletions.
diff --git a/database/src/main/scala/no/ndla/database/TableMigration.scala b/database/src/main/scala/no/ndla/database/TableMigration.scala
@@ -9,11 +9,12 @@
 package no.ndla.database
 
 import org.flywaydb.core.api.migration.{BaseJavaMigration, Context}
-import scalikejdbc.{DB, DBSession, *}
+import scalikejdbc.*
 
 abstract class TableMigration[ROW_DATA] extends BaseJavaMigration {
   val tableName: String
   val whereClause: SQLSyntax
+  val chunkSize: Int = 1000
   def extractRowData(rs: WrappedResultSet): ROW_DATA
   def updateRow(rowData: ROW_DATA)(implicit session: DBSession): Int
   lazy val tableNameSQL: SQLSyntax = SQLSyntax.createUnsafely(tableName)
@@ -25,7 +26,7 @@ abstract class TableMigration[ROW_DATA] extends BaseJavaMigration {
   }
 
   private def allRows(offset: Long)(implicit session: DBSession): Seq[ROW_DATA] = {
-    sql"select * from $tableNameSQL where $whereClause order by id limit 1000 offset $offset"
+    sql"select * from $tableNameSQL where $whereClause order by id limit $chunkSize offset $offset"
       .map(rs => extractRowData(rs))
       .list()
   }
@@ -36,11 +37,11 @@ abstract class TableMigration[ROW_DATA] extends BaseJavaMigration {
 
   private def migrateRows(implicit session: DBSession): Unit = {
     val count        = countAllRows.get
-    var numPagesLeft = (count / 1000) + 1
+    var numPagesLeft = (count / chunkSize) + 1
     var offset       = 0L
 
     while (numPagesLeft > 0) {
-      allRows(offset * 1000).map { rowData => updateRow(rowData) }: Unit
+      allRows(offset * chunkSize).map { rowData => updateRow(rowData) }: Unit
       numPagesLeft -= 1
       offset += 1
     }

diff --git a/...main/scala/no/ndla/myndlaapi/db/migrationwithdependencies/V16__MigrateResourcePaths.scala b/...main/scala/no/ndla/myndlaapi/db/migrationwithdependencies/V16__MigrateResourcePaths.scala
@@ -7,60 +7,36 @@
 
 package no.ndla.myndlaapi.db.migrationwithdependencies
 
+import no.ndla.database.TableMigration
 import no.ndla.myndlaapi.integration.TaxonomyApiClient
 import no.ndla.network.NdlaClient
-import org.flywaydb.core.api.migration.{BaseJavaMigration, Context}
-import scalikejdbc.{DB, DBSession, scalikejdbcSQLInterpolationImplicitDef}
+import scalikejdbc.{DBSession, WrappedResultSet, scalikejdbcSQLInterpolationImplicitDef}
 
 import java.util.UUID
 
 trait V16__MigrateResourcePaths {
   this: TaxonomyApiClient & NdlaClient =>
 
-  class V16__MigrateResourcePaths extends BaseJavaMigration {
-    private val chunkSize = 100;
-
-    override def migrate(context: Context): Unit = DB(context.getConnection)
-      .autoClose(false)
-      .withinTx { implicit session =>
-        migrateResources
-      }
-
-    private def migrateResources(implicit session: DBSession): Unit = {
-      val count        = countResources.get
-      var numPagesLeft = (count / chunkSize) + 1
-      var offset       = 0L
-
-      while (numPagesLeft > 0) {
-        allResources(offset * chunkSize).foreach { case (id, resourceType, path) =>
-          resourceType match {
-            case "article" | "learningpath" | "multidisciplinary" | "topic" =>
-              val updatedPath = taxonomyApiClient.resolveUrl(path)
-              updatedPath.map(updateResource(UUID.fromString(id), _))
-            case _ => ()
-          }
-        }
-        numPagesLeft -= 1
-        offset += 1
+  class V16__MigrateResourcePaths extends TableMigration[ResourceRow] {
+    override val tableName: String                  = "resources"
+    override val whereClause: scalikejdbc.SQLSyntax = sqls"path is not null"
+    override val chunkSize: Int                     = 1000
+    override def extractRowData(rs: WrappedResultSet): ResourceRow = ResourceRow(
+      UUID.fromString(rs.string("id")),
+      rs.string("resource_type"),
+      rs.string("path")
+    )
+    override def updateRow(rowData: ResourceRow)(implicit session: DBSession): Int = {
+      rowData.resourceType match {
+        case "article" | "learningpath" | "multidisciplinary" | "topic" =>
+          taxonomyApiClient
+            .resolveUrl(rowData.path)
+            .map { path => sql"update resources set path=$path where id = ${rowData.id}".update() }
+            .get
+        case _ => 0
       }
     }
-
-    private def updateResource(id: UUID, path: String)(implicit session: DBSession): Int = {
-      sql"update resources set path=$path where id = $id".update()
-    }
-
-    private def allResources(offset: Long)(implicit session: DBSession): Seq[(String, String, String)] = {
-      sql"select id, resource_type, path from resources order by id limit $chunkSize offset $offset"
-        .map(rs => {
-          (rs.string("id"), rs.string("resource_type"), rs.string("path"))
-        })
-        .list()
-    }
-
-    private def countResources(implicit session: DBSession): Option[Long] = {
-      sql"select count(*) from resources"
-        .map(rs => rs.long("count"))
-        .single()
-    }
   }
 }
+
+case class ResourceRow(id: UUID, resourceType: String, path: String)
diff --git a/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/TaxonomyApiClient.scala b/myndla-api/src/main/scala/no/ndla/myndlaapi/integration/TaxonomyApiClient.scala
@@ -14,7 +14,7 @@ import no.ndla.myndlaapi.Props
 import no.ndla.network.NdlaClient
 import sttp.client3.quick.*
 
-import scala.util.Try
+import scala.util.{Success, Try}
 
 trait TaxonomyApiClient {
   this: NdlaClient & Props =>
@@ -26,7 +26,7 @@ trait TaxonomyApiClient {
 
     def resolveUrl(path: String): Try[String] = {
       val req = quickRequest.get(uri"$resolveEndpoint?path=$path")
-      ndlaClient.fetch[ResolvePathResponse](req).map(resolved => resolved.url)
+      ndlaClient.fetch[ResolvePathResponse](req).map(resolved => resolved.url).orElse(Success(path))
     }
   }
 }