Fix register pagination

Signed-off-by: Elly Kitoto <[email protected]>

android/engine/src/main/java/org/smartregister/fhircore/engine/data/local/DefaultRepository.kt

@@ -47,6 +47,7 @@ import java.util.LinkedList
 import java.util.UUID
 import javax.inject.Inject
 import javax.inject.Singleton
+import kotlin.math.ceil
 import kotlinx.coroutines.withContext
 import kotlinx.serialization.json.JsonArray
 import kotlinx.serialization.json.JsonElement
@@ -608,7 +609,6 @@ constructor(
           }
         }
       }
-      repositoryResourceDataList.clear()
     }
   }
 
@@ -746,36 +746,51 @@ constructor(
     filterByRelatedEntityLocationMetaTag: Boolean,
     currentPage: Int?,
     pageSize: Int?,
-  ): MutableMap<String, RepositoryResourceData> {
-    val resultsDataMap = mutableMapOf<String, RepositoryResourceData>()
+  ): Map<String, RepositoryResourceData> {
+    // Use LinkedHashMap to maintain the order of insertion
+    val resultsDataMap = LinkedHashMap<String, RepositoryResourceData>()
     if (filterByRelatedEntityLocationMetaTag) {
-      val locationIds = retrieveRelatedEntitySyncLocationIds()
+      val syncLocationIds = retrieveRelatedEntitySyncLocationIds()
       if (currentPage != null && pageSize != null) {
-        for (ids in locationIds) {
-          if (resultsDataMap.size == pageSize) return resultsDataMap
+        val requiredSize = (currentPage + 1) * pageSize
+        var page = 0
+        val maxPages = ceil((syncLocationIds.size / pageSize).toDouble())
+        while ((resultsDataMap.size < requiredSize) && page <= maxPages) {
           val searchResults =
             searchResources(
-              baseResourceIds = ids,
-              baseResourceConfig = fhirResourceConfig.baseResource,
-              relatedResourcesConfigs = fhirResourceConfig.relatedResources,
-              activeResourceFilters = activeResourceFilters,
-              configComputedRuleValues = configComputedRuleValues,
-              currentPage = currentPage,
-              pageSize = pageSize,
-              relTagCodeSystem =
-                context.getString(R.string.sync_strategy_related_entity_location_system),
-            )
+                baseResourceIds = null,
+                baseResourceConfig = fhirResourceConfig.baseResource,
+                relatedResourcesConfigs = fhirResourceConfig.relatedResources,
+                activeResourceFilters = activeResourceFilters,
+                configComputedRuleValues = configComputedRuleValues,
+                currentPage = page,
+                pageSize = minOf(requiredSize, SQL_WHERE_CLAUSE_LIMIT),
+                relTagCodeSystem =
+                  context.getString(R.string.sync_strategy_related_entity_location_system),
+              )
+              .filter { resourceData ->
+                when (resourceData.resource.resourceType) {
+                  ResourceType.Location -> syncLocationIds.contains(resourceData.resource.logicalId)
+                  else ->
+                    resourceData.resource.meta.tag.any {
+                      it.system ==
+                        context.getString(R.string.sync_strategy_related_entity_location_system) &&
+                        syncLocationIds.contains(it.code)
+                    }
+                }
+              }
           processSearchResult(
             searchResults = searchResults,
             resultsDataMap = resultsDataMap,
             fhirResourceConfig = fhirResourceConfig,
             configComputedRuleValues = configComputedRuleValues,
             activeResourceFilters = activeResourceFilters,
-            pageSizeLimit = pageSize,
           )
+          page++
         }
+        return resultsDataMap
       } else {
-        for (ids in locationIds) {
+        for (ids in syncLocationIds.chunked(SQL_WHERE_CLAUSE_LIMIT)) {
           val searchResults =
             searchResources(
               baseResourceIds = ids,
@@ -794,7 +809,6 @@ constructor(
             fhirResourceConfig = fhirResourceConfig,
             configComputedRuleValues = configComputedRuleValues,
             activeResourceFilters = activeResourceFilters,
-            pageSizeLimit = null,
           )
         }
       }
@@ -822,11 +836,10 @@ constructor(
 
   private suspend fun processSearchResult(
     searchResults: List<SearchResult<Resource>>,
-    resultsDataMap: MutableMap<String, RepositoryResourceData>,
+    resultsDataMap: LinkedHashMap<String, RepositoryResourceData>,
     fhirResourceConfig: FhirResourceConfig,
     configComputedRuleValues: Map<String, Any>,
     activeResourceFilters: List<ActiveResourceFilterConfig>?,
-    pageSizeLimit: Int? = null,
   ) {
     val processedSearchResults =
       handleSearchResults(
@@ -836,7 +849,6 @@ constructor(
         relatedResourceConfigs = fhirResourceConfig.relatedResources,
         baseResourceConfigId = fhirResourceConfig.baseResource.id,
         configComputedRuleValues = configComputedRuleValues,
-        limit = pageSizeLimit,
       )
 
     while (processedSearchResults.isNotEmpty()) {
@@ -861,7 +873,6 @@ constructor(
           relatedResourceConfigs = newResourceConfig.relatedResources,
           baseResourceConfigId = fhirResourceConfig.baseResource.id,
           configComputedRuleValues = configComputedRuleValues,
-          limit = null,
         )
       processedSearchResults.addAll(newProcessedSearchResults)
     }
@@ -923,12 +934,11 @@ constructor(
 
   private suspend fun handleSearchResults(
     searchResults: List<SearchResult<Resource>>,
-    repositoryResourceDataResultMap: MutableMap<String, RepositoryResourceData>,
+    repositoryResourceDataResultMap: LinkedHashMap<String, RepositoryResourceData>,
     repositoryResourceDataMap: Map<String, String>?,
     relatedResourceConfigs: List<ResourceConfig>,
     baseResourceConfigId: String?,
     configComputedRuleValues: Map<String, Any>,
-    limit: Int?,
   ): SearchQueryResultQueue {
     val relatedResourcesQueue = RelatedResourcesQueue()
     val (forwardIncludes, reverseIncludes) =
@@ -993,10 +1003,7 @@ constructor(
           )
         }
       }
-      if (
-        repositoryResourceDataMap == null &&
-          (limit == null || repositoryResourceDataResultMap.size < limit)
-      ) {
+      if (repositoryResourceDataMap == null) {
         repositoryResourceDataResultMap[searchResult.resource.logicalId] = repositoryResourceData
       }
     }
@@ -1134,6 +1141,12 @@ constructor(
   ): Search {
     val search =
       Search(type = baseResourceConfig.resource).apply {
+        applyConfiguredSortAndFilters(
+          resourceConfig = baseResourceConfig,
+          filterActiveResources = filterActiveResources,
+          sortData = sortData,
+          configComputedRuleValues = configComputedRuleValues,
+        )
         if (!baseResourceIds.isNullOrEmpty()) {
           when (baseResourceConfig.resource) {
             ResourceType.Location ->
@@ -1149,16 +1162,9 @@ constructor(
               }
           }
         }
-        applyConfiguredSortAndFilters(
-          resourceConfig = baseResourceConfig,
-          filterActiveResources = filterActiveResources,
-          sortData = sortData,
-          configComputedRuleValues = configComputedRuleValues,
-        )
-        if (currentPage != null && count != null) {
-          this.count = count
-          from = currentPage * count
-        }
+
+        this.count = count
+        from = count?.let { currentPage?.times(it) }
       }
     return search
   }
@@ -1297,14 +1303,14 @@ constructor(
     return null
   }
 
-  protected suspend fun retrieveRelatedEntitySyncLocationIds(): List<List<String>> =
+  protected suspend fun retrieveRelatedEntitySyncLocationIds(): HashSet<String> =
     withContext(dispatcherProvider.io()) {
       context
         .retrieveRelatedEntitySyncLocationState(MultiSelectViewAction.FILTER_DATA)
+        .asSequence()
         .chunked(SQL_WHERE_CLAUSE_LIMIT)
         .map { it.map { state -> state.locationId } }
-        .flatMap { retrieveFlattenedSubLocationIds(it) }
-        .chunked(SQL_WHERE_CLAUSE_LIMIT)
+        .flatMapTo(HashSet()) { retrieveFlattenedSubLocationIds(it) }
     }
 
   suspend fun retrieveFlattenedSubLocationIds(locationIds: List<String>): HashSet<String> {
@@ -1380,9 +1386,28 @@ constructor(
     return questionnaireResponses.maxByOrNull { it.meta.lastUpdated }
   }
 
+  /**
+   * Retrieves a sublist representing a specific page from the given list.
+   *
+   * @param data The complete list of data.
+   * @param page The page number to retrieve (1-based index).
+   * @param pageSize The number of items per page.
+   * @return A list containing the items on the specified page, or an empty list if the page is out
+   *   of range.
+   */
+  fun <T> getPage(data: List<T>, page: Int, pageSize: Int): List<T> {
+    val fromIndex = page * pageSize
+    val toIndex = minOf(fromIndex + pageSize, data.size)
+    return if (fromIndex in data.indices) {
+      data.subList(fromIndex, toIndex)
+    } else {
+      emptyList()
+    }
+  }
+
   companion object {
     const val RESOURCE_BATCH_SIZE = 50
-    const val SQL_WHERE_CLAUSE_LIMIT = 200 // Hard limit for WHERE CLAUSE items is 1000
+    const val SQL_WHERE_CLAUSE_LIMIT = 400 // Hard limit for WHERE CLAUSE items is 1000
     const val SNOMED_SYSTEM = "http://hl7.org/fhir/R4B/valueset-condition-clinical.html"
     const val PATIENT_CONDITION_RESOLVED_CODE = "resolved"
     const val PATIENT_CONDITION_RESOLVED_DISPLAY = "Resolved"

android/engine/src/main/java/org/smartregister/fhircore/engine/data/local/register/RegisterRepository.kt

@@ -73,21 +73,24 @@ constructor(
     registerId: String,
     fhirResourceConfig: FhirResourceConfig?,
     paramsMap: Map<String, String>?,
+    loadAll: Boolean,
   ): List<RepositoryResourceData> {
     val registerConfiguration = retrieveRegisterConfiguration(registerId, paramsMap)
     val requiredFhirResourceConfig = fhirResourceConfig ?: registerConfiguration.fhirResource
     val configComputedRuleValues = registerConfiguration.configRules.configRulesComputedValues()
 
+    val filterByRelatedEntityLocationMetaTag =
+      registerConfiguration.filterDataByRelatedEntityLocation
+    val pageSize = registerConfiguration.pageSize
     val registerDataMap =
       searchNestedResources(
         baseResourceIds = null,
         fhirResourceConfig = requiredFhirResourceConfig,
         configComputedRuleValues = configComputedRuleValues,
         activeResourceFilters = registerConfiguration.activeResourceFilters,
-        filterByRelatedEntityLocationMetaTag =
-          registerConfiguration.filterDataByRelatedEntityLocation,
+        filterByRelatedEntityLocationMetaTag = filterByRelatedEntityLocationMetaTag,
         currentPage = currentPage,
-        pageSize = registerConfiguration.pageSize,
+        pageSize = pageSize,
       )
 
     populateSecondaryResources(
@@ -96,7 +99,13 @@ constructor(
       resultsDataMap = registerDataMap,
     )
 
-    return registerDataMap.values.toList()
+    // Handle pagination via code for resources filtered by REL tag. The list size should be
+    // (currentPage + 1).times(pageSize).
+    return if (filterByRelatedEntityLocationMetaTag && !loadAll) {
+      getPage(registerDataMap.values.toList(), currentPage, pageSize)
+    } else {
+      registerDataMap.values as List
+    }
   }
 
   /** Count register data for the provided [registerId]. Use the configured base resource filters */
@@ -125,7 +134,7 @@ constructor(
       )
     }
 
-    val locationIds = retrieveRelatedEntitySyncLocationIds()
+    val locationIds = retrieveRelatedEntitySyncLocationIds().chunked(SQL_WHERE_CLAUSE_LIMIT)
     var total = 0L
     for (ids in locationIds) {
       val search =
@@ -196,7 +205,7 @@ constructor(
   private suspend fun populateSecondaryResources(
     secondaryResources: List<FhirResourceConfig>?,
     configComputedRuleValues: Map<String, Any>,
-    resultsDataMap: MutableMap<String, RepositoryResourceData>,
+    resultsDataMap: Map<String, RepositoryResourceData>,
   ) {
     if (!secondaryResources.isNullOrEmpty()) {
       val secondaryRepositoryResourceData = mutableListOf<RepositoryResourceData>()

android/engine/src/main/java/org/smartregister/fhircore/engine/domain/repository/Repository.kt

@@ -32,6 +32,7 @@ interface Repository {
     registerId: String,
     fhirResourceConfig: FhirResourceConfig? = null,
     paramsMap: Map<String, String>? = emptyMap(),
+    loadAll: Boolean = false,
   ): List<RepositoryResourceData>
 
   /**

android/quest/src/main/java/org/smartregister/fhircore/quest/data/register/RegisterPagingSource.kt

@@ -60,6 +60,7 @@ class RegisterPagingSource(
             registerId = registerPagingSourceState.registerId,
             fhirResourceConfig = fhirResourceConfig,
             paramsMap = actionParameters,
+            loadAll = registerPagingSourceState.loadAll,
           )
           .map {
             rulesExecutor.processResourceData(