Skip to content

Commit

Permalink
search-api: Add grep code indexing and search
Browse files Browse the repository at this point in the history
This patch introduces `POST /search-api/v1/search/grep`.

A new endpoint that can be used to search all the indexed grep codes by
prefix, query and a direct code filter.
  • Loading branch information
jnatten committed Dec 5, 2024
1 parent 54ac86a commit 1a32da4
Show file tree
Hide file tree
Showing 28 changed files with 848 additions and 310 deletions.
3 changes: 2 additions & 1 deletion project/searchapi.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ object searchapi extends Module {
"SearchParams",
"DraftSearchParams",
"SubjectAggregations",
"SubjectAggsInput"
"SubjectAggsInput",
"GrepSearchInput"
)
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ class ComponentRegistry(properties: SearchApiProperties)
with FeideApiClient
with RedisClient
with InternController
with GrepIndexService
with SearchApiClient
with GrepApiClient
with Props
with SwaggerDocControllerConfig {
with SwaggerDocControllerConfig
with GrepSearchService {
override val props: SearchApiProperties = properties
import props._

Expand All @@ -80,6 +82,8 @@ class ComponentRegistry(properties: SearchApiProperties)
lazy val learningPathIndexService = new LearningPathIndexService
lazy val draftIndexService = new DraftIndexService
lazy val multiDraftSearchService = new MultiDraftSearchService
lazy val grepIndexService = new GrepIndexService
lazy val grepSearchService = new GrepSearchService

lazy val searchController = new SearchController
lazy val healthController: TapirHealthController = new TapirHealthController
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,22 @@ class SearchApiProperties extends BaseProps with StrictLogging {
val draftIndexName = propOrElse("DRAFT_SEARCH_INDEX_NAME", "drafts")
val learningpathIndexName = propOrElse("LEARNINGPATH_SEARCH_INDEX_NAME", "learningpaths")
val conceptIndexName = propOrElse("DRAFT_CONCEPT_SEARCH_INDEX_NAME", "draftconcepts")
val grepIndexName = propOrElse("GREP_SEARCH_INDEX_NAME", "greps")

def SearchIndex(searchType: SearchType) = searchType match {
case SearchType.Articles => articleIndexName
case SearchType.Drafts => draftIndexName
case SearchType.LearningPaths => learningpathIndexName
case SearchType.Concepts => conceptIndexName
case SearchType.Grep => grepIndexName
}

def indexToSearchType(indexName: String): Try[SearchType] = indexName match {
case `articleIndexName` => Success(SearchType.Articles)
case `draftIndexName` => Success(SearchType.Drafts)
case `learningpathIndexName` => Success(SearchType.LearningPaths)
case `conceptIndexName` => Success(SearchType.Concepts)
case `grepIndexName` => Success(SearchType.Grep)
case _ => Failure(new IllegalArgumentException(s"Unknown index name: $indexName"))
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,17 @@ import no.ndla.network.model.RequestInfo
import no.ndla.network.tapir.NoNullJsonPrinter.jsonBody
import no.ndla.network.tapir.{AllErrors, TapirController}
import no.ndla.network.tapir.TapirUtil.errorOutputsFor
import no.ndla.search.model.domain.ReindexResult
import no.ndla.searchapi.Props
import no.ndla.searchapi.integration.{GrepApiClient, TaxonomyApiClient}
import no.ndla.searchapi.model.api.ErrorHandling
import no.ndla.searchapi.model.domain.{IndexingBundle, ReindexResult}
import no.ndla.searchapi.model.domain.IndexingBundle
import no.ndla.searchapi.model.search.SearchType
import no.ndla.searchapi.service.search.{
ArticleIndexService,
DraftConceptIndexService,
DraftIndexService,
GrepIndexService,
IndexService,
LearningPathIndexService
}
Expand All @@ -46,7 +48,7 @@ import sttp.tapir.server.ServerEndpoint

trait InternController {
this: IndexService & ArticleIndexService & LearningPathIndexService & DraftIndexService & DraftConceptIndexService &
TaxonomyApiClient & GrepApiClient & Props & ErrorHandling & MyNDLAApiClient & TapirController =>
TaxonomyApiClient & GrepApiClient & GrepIndexService & Props & ErrorHandling & MyNDLAApiClient & TapirController =>
val internController: InternController

class InternController extends TapirController with StrictLogging {
Expand Down Expand Up @@ -100,6 +102,7 @@ trait InternController {
reindexById,
reindexArticle,
reindexDraft,
reindexGrep,
reindexLearningpath,
reindexConcept
)
Expand Down Expand Up @@ -230,6 +233,21 @@ trait InternController {
resolveResultFutures(List(articleIndex))
}

def reindexGrep: ServerEndpoint[Any, Eff] = endpoint.post
.in("index" / "grep")
.in(query[Option[Int]]("numShards"))
.errorOut(stringInternalServerError)
.out(stringBody)
.serverLogicPure { numShards =>
val requestInfo = RequestInfo.fromThreadContext()
val grepIndex = Future {
requestInfo.setThreadContextRequestInfo()
("greps", grepIndexService.indexDocuments(numShards, None))
}

resolveResultFutures(List(grepIndex))
}

def reindexLearningpath: ServerEndpoint[Any, Eff] = endpoint.post
.in("index" / "learningpath")
.in(query[Option[Int]]("numShards"))
Expand All @@ -256,12 +274,14 @@ trait InternController {
draftIndexService.cleanupIndexes(): Unit
learningPathIndexService.cleanupIndexes(): Unit
draftConceptIndexService.cleanupIndexes(): Unit
grepIndexService.cleanupIndexes(): Unit

val articles = articleIndexService.reindexWithShards(numShards)
val drafts = draftIndexService.reindexWithShards(numShards)
val learningpaths = learningPathIndexService.reindexWithShards(numShards)
val concept = draftConceptIndexService.reindexWithShards(numShards)
List(articles, drafts, learningpaths, concept).sequence match {
val greps = grepIndexService.reindexWithShards(numShards)
List(articles, drafts, learningpaths, concept, greps).sequence match {
case Success(_) =>
s"Reindexing with $numShards shards completed in ${System.currentTimeMillis() - startTime}ms".asRight
case Failure(ex) =>
Expand All @@ -280,12 +300,14 @@ trait InternController {
draftIndexService.cleanupIndexes(): Unit
learningPathIndexService.cleanupIndexes(): Unit
draftConceptIndexService.cleanupIndexes(): Unit
grepIndexService.cleanupIndexes(): Unit

val articles = articleIndexService.updateReplicaNumber(numReplicas)
val drafts = draftIndexService.updateReplicaNumber(numReplicas)
val learningpaths = learningPathIndexService.updateReplicaNumber(numReplicas)
val concepts = draftConceptIndexService.updateReplicaNumber(numReplicas)
List(articles, drafts, learningpaths, concepts).sequence match {
val greps = grepIndexService.updateReplicaNumber(numReplicas)
List(articles, drafts, learningpaths, concepts, greps).sequence match {
case Success(_) =>
s"Updated replication setting for indexes to $numReplicas replicas. Populating may take some time.".asRight
case Failure(ex) =>
Expand Down Expand Up @@ -326,6 +348,7 @@ trait InternController {
articleIndexService.cleanupIndexes(): Unit
draftIndexService.cleanupIndexes(): Unit
draftConceptIndexService.cleanupIndexes(): Unit
grepIndexService.cleanupIndexes(): Unit

val publishedIndexingBundle = IndexingBundle(
grepBundle = Some(grepBundle),
Expand Down Expand Up @@ -356,6 +379,10 @@ trait InternController {
Future {
requestInfo.setThreadContextRequestInfo()
("concepts", draftConceptIndexService.indexDocuments(numShards, draftIndexingBundle))
},
Future {
requestInfo.setThreadContextRequestInfo()
("greps", grepIndexService.indexDocuments(numShards, Some(grepBundle)))
}
)
if (runInBackground) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,16 @@ import no.ndla.network.tapir.Parameters.feideHeader
import no.ndla.network.tapir.{AllErrors, DynamicHeaders, NonEmptyString, TapirController}
import no.ndla.network.tapir.TapirUtil.errorOutputsFor
import no.ndla.network.tapir.auth.Permission.DRAFT_API_WRITE
import no.ndla.searchapi.controller.parameters.{DraftSearchParams, SearchParams, SubjectAggsInput}
import no.ndla.searchapi.controller.parameters.{DraftSearchParams, GrepSearchInput, SearchParams, SubjectAggsInput}
import no.ndla.searchapi.Props
import no.ndla.searchapi.integration.SearchApiClient
import no.ndla.searchapi.model.api.grep.GrepSearchResults
import no.ndla.searchapi.model.api.{ErrorHandling, GroupSearchResult, MultiSearchResult, SubjectAggregations}
import no.ndla.searchapi.model.domain.{LearningResourceType, Sort}
import no.ndla.searchapi.model.search.SearchType
import no.ndla.searchapi.model.search.settings.{MultiDraftSearchSettings, SearchSettings}
import no.ndla.searchapi.service.search.{
GrepSearchService,
MultiDraftSearchService,
MultiSearchService,
SearchConverterService,
Expand All @@ -47,7 +49,7 @@ import sttp.tapir.server.ServerEndpoint

trait SearchController {
this: SearchApiClient & MultiSearchService & SearchConverterService & SearchService & MultiDraftSearchService &
FeideApiClient & Props & ErrorHandling & TapirController =>
FeideApiClient & Props & ErrorHandling & TapirController & GrepSearchService =>
val searchController: SearchController

class SearchController extends TapirController {
Expand Down Expand Up @@ -159,7 +161,8 @@ trait SearchController {
searchDraftLearningResources,
searchDraftLearningResourcesGet,
postSearchLearningResources,
subjectAggs
subjectAggs,
searchGrep
)

def subjectAggs: ServerEndpoint[Any, Eff] = endpoint.post
Expand Down Expand Up @@ -593,6 +596,16 @@ trait SearchController {
}
}

def searchGrep: ServerEndpoint[Any, Eff] = endpoint.post
.summary("Search for grep codes")
.description("Search for grep codes")
.in("grep")
.in(jsonBody[GrepSearchInput])
.out(jsonBody[GrepSearchResults])
.errorOut(errorOutputsFor(400, 401, 403))
.requirePermission(DRAFT_API_WRITE)
.serverLogicPure { _ => input => grepSearchService.searchGreps(input) }

/** This method fetches availability based on FEIDE access token in the request This does an actual api-call to the
* feide api and should be used sparingly.
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Part of NDLA search-api
* Copyright (C) 2024 NDLA
*
* See LICENSE
*
*/

package no.ndla.searchapi.controller.parameters

import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
import io.circe.{Decoder, Encoder}
import no.ndla.network.tapir.NonEmptyString
import no.ndla.searchapi.model.api.grep.GrepSort
import sttp.tapir.Schema.annotations.description

// format: off
@description("Input parameters to subject aggregations endpoint")
case class GrepSearchInput(
@description("A comma separated list of prefixes that should be returned in the search.")
prefixFilter: Option[List[String]],

@description("A comma separated list of codes that should be returned in the search.")
codes: Option[List[String]],

@description("A query to filter the query by.")
query: Option[NonEmptyString],

@description("The page number of the search hits to display.")
page: Option[Int],

@description(s"The number of search hits to display for each page.")
pageSize: Option[Int],

@description("The sort order of the search hits.")
sort: Option[GrepSort],

@description("The ISO 639-1 language code describing language used in query-params")
language: Option[String]
)

object GrepSearchInput {
implicit val encoder: Encoder[GrepSearchInput] = deriveEncoder
implicit val decoder: Decoder[GrepSearchInput] = deriveDecoder
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Part of NDLA search-api
* Copyright (C) 2024 NDLA
*
* See LICENSE
*
*/

package no.ndla.searchapi.model.api.grep

import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
import io.circe.{Decoder, Encoder}
import no.ndla.searchapi.model.api.Title
import sttp.tapir.Schema.annotations.description

@description("Information about a single grep search result entry")
case class GrepResult(
@description("The grep code") code: String,
@description("The greps title") title: Title
)

object GrepResult {
implicit val encoder: Encoder[GrepResult] = deriveEncoder
implicit val decoder: Decoder[GrepResult] = deriveDecoder
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Part of NDLA search-api
* Copyright (C) 2024 NDLA
*
* See LICENSE
*
*/

package no.ndla.searchapi.model.api.grep

import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
import io.circe.{Decoder, Encoder}
import sttp.tapir.Schema.annotations.description

@description("Information about search-results")
case class GrepSearchResults(
@description("The total number of resources matching this query") totalCount: Long,
@description("For which page results are shown from") page: Int,
@description("The number of results per page") pageSize: Int,
@description("The chosen search language") language: String,
@description("The search results") results: Seq[GrepResult]
)

object GrepSearchResults {
implicit val encoder: Encoder[GrepSearchResults] = deriveEncoder
implicit val decoder: Decoder[GrepSearchResults] = deriveDecoder
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Part of NDLA search-api
* Copyright (C) 2024 NDLA
*
* See LICENSE
*
*/

package no.ndla.searchapi.model.api.grep

import com.scalatsi.TypescriptType.{TSLiteralString, TSUnion}
import com.scalatsi.{TSNamedType, TSType}
import enumeratum.*
import sttp.tapir.Codec.PlainCodec
import sttp.tapir.Schema
import sttp.tapir.codec.enumeratum.*

sealed abstract class GrepSort(override val entryName: String) extends EnumEntry
object GrepSort extends Enum[GrepSort] with CirceEnum[GrepSort] {
val values: IndexedSeq[GrepSort] = findValues
val all: Seq[String] = values.map(_.entryName)

case object ByRelevanceDesc extends GrepSort("-relevance")
case object ByRelevanceAsc extends GrepSort("relevance")
case object ByTitleDesc extends GrepSort("-title")
case object ByTitleAsc extends GrepSort("title")
case object ByCodeDesc extends GrepSort("-code")
case object ByCodeAsc extends GrepSort("code")

implicit val schema: Schema[GrepSort] = schemaForEnumEntry[GrepSort]
implicit val codec: PlainCodec[GrepSort] = plainCodecEnumEntry[GrepSort]
implicit val enumTsType: TSNamedType[GrepSort] =
TSType.alias[GrepSort]("GrepSort", TSUnion(all.map(s => TSLiteralString(s))))

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/*
* Part of NDLA search-api
* Copyright (C) 2024 NDLA
*
* See LICENSE
*
*/

package no.ndla.searchapi.model.search

case class SearchPagination(
page: Int,
pageSize: Int,
startAt: Int
)
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ object SearchType extends Enum[SearchType] with CirceEnumWithErrors[SearchType]
case object Drafts extends SearchType("draft")
case object LearningPaths extends SearchType("learningpath")
case object Concepts extends SearchType("concept")
case object Grep extends SearchType("grep")

def all: List[String] = SearchType.values.map(_.toString).toList
override def values: IndexedSeq[SearchType] = findValues
Expand Down
Loading

0 comments on commit 1a32da4

Please sign in to comment.