From 7b2a58ed93e71a8eb43b89523e80c2ac3958ab6e Mon Sep 17 00:00:00 2001 From: Ali Akbar Date: Thu, 28 Mar 2024 20:58:44 +0500 Subject: [PATCH] fix: update limit for es response --- .../apps/api/tests/test_serializers.py | 17 ++++------------- course_discovery/apps/core/utils.py | 5 +++++ .../management/commands/update_index.py | 2 ++ course_discovery/settings/base.py | 6 ++++-- course_discovery/settings/test.py | 4 ++++ 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/course_discovery/apps/api/tests/test_serializers.py b/course_discovery/apps/api/tests/test_serializers.py index 3cbe348e87..c83bd84558 100644 --- a/course_discovery/apps/api/tests/test_serializers.py +++ b/course_discovery/apps/api/tests/test_serializers.py @@ -5,7 +5,6 @@ from urllib.parse import urlencode import ddt -import pytest import responses from django.test import TestCase from django.utils.text import slugify @@ -2623,9 +2622,7 @@ def get_expected_data(cls, person, request): } -@pytest.mark.django_db -@pytest.mark.usefixtures('elasticsearch_dsl_default_connection') -class TestProgramSearchDocumentSerializer(TestCase): +class TestProgramSearchDocumentSerializer(ElasticsearchTestMixin, TestCase): serializer_class = ProgramSearchDocumentSerializer def setUp(self): @@ -2722,9 +2719,7 @@ def get_expected_data(cls, program, request): return expected -@pytest.mark.django_db -@pytest.mark.usefixtures('elasticsearch_dsl_default_connection') -class TestLearnerPathwaySearchDocumentSerializer(TestCase): +class TestLearnerPathwaySearchDocumentSerializer(ElasticsearchTestMixin, TestCase): serializer_class = LearnerPathwaySearchDocumentSerializer def setUp(self): @@ -2782,9 +2777,7 @@ def get_expected_data(cls, learner_pathway, request): return expected -@pytest.mark.django_db -@pytest.mark.usefixtures('elasticsearch_dsl_default_connection') -class TestTypeaheadCourseRunSearchSerializer: +class TestTypeaheadCourseRunSearchSerializer(ElasticsearchTestMixin, TestCase): serializer_class = TypeaheadCourseRunSearchSerializer @classmethod @@ -2809,9 +2802,7 @@ def serialize_course_run(self, course_run): return serializer -@pytest.mark.django_db -@pytest.mark.usefixtures('elasticsearch_dsl_default_connection') -class TestTypeaheadProgramSearchSerializer: +class TestTypeaheadProgramSearchSerializer(ElasticsearchTestMixin, TestCase): serializer_class = TypeaheadProgramSearchSerializer @classmethod diff --git a/course_discovery/apps/core/utils.py b/course_discovery/apps/core/utils.py index 9560f0de84..94ffc5bb08 100644 --- a/course_discovery/apps/core/utils.py +++ b/course_discovery/apps/core/utils.py @@ -71,6 +71,11 @@ def set_alias(cls, connection, alias, index): connection.indices.update_aliases(body) + @classmethod + def update_max_result_window(cls, connection, max_result_window, index): + if connection.indices.exists(index=index): + connection.indices.put_settings(body={"index": {"max_result_window": max_result_window}}) + @classmethod def create_index(cls, index, conn_name='default'): """ diff --git a/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py b/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py index ebff29cd18..3a5e0f2dfe 100644 --- a/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py +++ b/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py @@ -117,11 +117,13 @@ def _update(self, models, options): ) if record_count_is_sane: ElasticsearchUtils.set_alias(conn, alias, new_index_name) + ElasticsearchUtils.update_max_result_window(conn, settings.MAX_RESULT_WINDOW, new_index_name) indexes_pending.pop(new_index_name, None) else: indexes_pending[new_index_name] = index_info_string else: ElasticsearchUtils.set_alias(conn, alias, new_index_name) + ElasticsearchUtils.update_max_result_window(conn, settings.MAX_RESULT_WINDOW, new_index_name) indexes_pending.pop(new_index_name, None) for index_alias_mapper in alias_mappings: diff --git a/course_discovery/settings/base.py b/course_discovery/settings/base.py index 493638dc6e..9ba33591ea 100644 --- a/course_discovery/settings/base.py +++ b/course_discovery/settings/base.py @@ -461,11 +461,13 @@ # (by default it uses the database driver's default setting) # https://docs.djangoproject.com/en/3.1/ref/models/querysets/#iterator # Thus set the 'chunk_size' -ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 10000 +ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 15000 # Defining default pagination for all requests to ElasticSearch, # whose parameters 'size' and 'from' are not explicitly set. -ELASTICSEARCH_DSL_LOAD_PER_QUERY = 10000 +ELASTICSEARCH_DSL_LOAD_PER_QUERY = 15000 + +MAX_RESULT_WINDOW = 15000 ELASTICSEARCH_DSL = { 'default': {'hosts': '127.0.0.1:9200'} diff --git a/course_discovery/settings/test.py b/course_discovery/settings/test.py index f999f3a95d..9994d1ae43 100644 --- a/course_discovery/settings/test.py +++ b/course_discovery/settings/test.py @@ -157,3 +157,7 @@ 'error_msg': 'Course edit was unsuccessful. The course URL slug "[{url_slug}]" is an invalid format. Please ensure that the slug is in the format `boot-camps//-`', }}, } + +ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 10000 + +ELASTICSEARCH_DSL_LOAD_PER_QUERY = 10000