fix: update limit for es response (#4303)

openedx · Apr 9, 2024 · a700f20 · a700f20
1 parent 3119a11
commit a700f20
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 15 deletions.
diff --git a/course_discovery/apps/api/tests/test_serializers.py b/course_discovery/apps/api/tests/test_serializers.py
@@ -5,7 +5,6 @@
 from urllib.parse import urlencode
 
 import ddt
-import pytest
 import responses
 from django.test import TestCase
 from django.utils.text import slugify
@@ -2623,9 +2622,7 @@ def get_expected_data(cls, person, request):
         }
 
 
-@pytest.mark.django_db
-@pytest.mark.usefixtures('elasticsearch_dsl_default_connection')
-class TestProgramSearchDocumentSerializer(TestCase):
+class TestProgramSearchDocumentSerializer(ElasticsearchTestMixin, TestCase):
     serializer_class = ProgramSearchDocumentSerializer
 
     def setUp(self):
@@ -2722,9 +2719,7 @@ def get_expected_data(cls, program, request):
         return expected
 
 
-@pytest.mark.django_db
-@pytest.mark.usefixtures('elasticsearch_dsl_default_connection')
-class TestLearnerPathwaySearchDocumentSerializer(TestCase):
+class TestLearnerPathwaySearchDocumentSerializer(ElasticsearchTestMixin, TestCase):
     serializer_class = LearnerPathwaySearchDocumentSerializer
 
     def setUp(self):
@@ -2782,9 +2777,7 @@ def get_expected_data(cls, learner_pathway, request):
         return expected
 
 
-@pytest.mark.django_db
-@pytest.mark.usefixtures('elasticsearch_dsl_default_connection')
-class TestTypeaheadCourseRunSearchSerializer:
+class TestTypeaheadCourseRunSearchSerializer(ElasticsearchTestMixin, TestCase):
     serializer_class = TypeaheadCourseRunSearchSerializer
 
     @classmethod
@@ -2809,9 +2802,7 @@ def serialize_course_run(self, course_run):
         return serializer
 
 
-@pytest.mark.django_db
-@pytest.mark.usefixtures('elasticsearch_dsl_default_connection')
-class TestTypeaheadProgramSearchSerializer:
+class TestTypeaheadProgramSearchSerializer(ElasticsearchTestMixin, TestCase):
     serializer_class = TypeaheadProgramSearchSerializer
 
     @classmethod

diff --git a/course_discovery/apps/core/utils.py b/course_discovery/apps/core/utils.py
@@ -71,6 +71,11 @@ def set_alias(cls, connection, alias, index):
 
         connection.indices.update_aliases(body)
 
+    @classmethod
+    def update_max_result_window(cls, connection, max_result_window, index):
+        if connection.indices.exists(index=index):
+            connection.indices.put_settings(body={"index": {"max_result_window": max_result_window}})
+
     @classmethod
     def create_index(cls, index, conn_name='default'):
         """

diff --git a/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py b/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py
@@ -117,11 +117,13 @@ def _update(self, models, options):
                     )
                     if record_count_is_sane:
                         ElasticsearchUtils.set_alias(conn, alias, new_index_name)
+                        ElasticsearchUtils.update_max_result_window(conn, settings.MAX_RESULT_WINDOW, new_index_name)
                         indexes_pending.pop(new_index_name, None)
                     else:
                         indexes_pending[new_index_name] = index_info_string
                 else:
                     ElasticsearchUtils.set_alias(conn, alias, new_index_name)
+                    ElasticsearchUtils.update_max_result_window(conn, settings.MAX_RESULT_WINDOW, new_index_name)
                     indexes_pending.pop(new_index_name, None)
 
         for index_alias_mapper in alias_mappings:

diff --git a/course_discovery/settings/base.py b/course_discovery/settings/base.py
@@ -461,11 +461,13 @@
 # (by default it uses the database driver's default setting)
 # https://docs.djangoproject.com/en/3.1/ref/models/querysets/#iterator
 # Thus set the 'chunk_size'
-ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 10000
+ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 15000
 
 # Defining default pagination for all requests to ElasticSearch,
 # whose parameters 'size' and 'from' are not explicitly set.
-ELASTICSEARCH_DSL_LOAD_PER_QUERY = 10000
+ELASTICSEARCH_DSL_LOAD_PER_QUERY = 15000
+
+MAX_RESULT_WINDOW = 15000
 
 ELASTICSEARCH_DSL = {
     'default': {'hosts': '127.0.0.1:9200'}

diff --git a/course_discovery/settings/test.py b/course_discovery/settings/test.py
@@ -157,3 +157,7 @@
             'error_msg': 'Course edit was unsuccessful. The course URL slug "[{url_slug}]" is an invalid format. Please ensure that the slug is in the format `boot-camps/<primary_subject>/<organization_name>-<course_title>`',
         }},
 }
+
+ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 10000
+
+ELASTICSEARCH_DSL_LOAD_PER_QUERY = 10000