Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: update limit for es response #4303

Merged
merged 1 commit into from
Apr 9, 2024
Merged

fix: update limit for es response #4303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 4 additions & 13 deletions course_discovery/apps/api/tests/test_serializers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
from urllib.parse import urlencode

import ddt
import pytest
import responses
from django.test import TestCase
from django.utils.text import slugify
Expand Down Expand Up @@ -2623,9 +2622,7 @@ def get_expected_data(cls, person, request):
}


@pytest.mark.django_db
@pytest.mark.usefixtures('elasticsearch_dsl_default_connection')
class TestProgramSearchDocumentSerializer(TestCase):
class TestProgramSearchDocumentSerializer(ElasticsearchTestMixin, TestCase):
serializer_class = ProgramSearchDocumentSerializer

def setUp(self):
Expand Down Expand Up @@ -2722,9 +2719,7 @@ def get_expected_data(cls, program, request):
return expected


@pytest.mark.django_db
@pytest.mark.usefixtures('elasticsearch_dsl_default_connection')
class TestLearnerPathwaySearchDocumentSerializer(TestCase):
class TestLearnerPathwaySearchDocumentSerializer(ElasticsearchTestMixin, TestCase):
serializer_class = LearnerPathwaySearchDocumentSerializer

def setUp(self):
Expand Down Expand Up @@ -2782,9 +2777,7 @@ def get_expected_data(cls, learner_pathway, request):
return expected


@pytest.mark.django_db
@pytest.mark.usefixtures('elasticsearch_dsl_default_connection')
class TestTypeaheadCourseRunSearchSerializer:
class TestTypeaheadCourseRunSearchSerializer(ElasticsearchTestMixin, TestCase):
serializer_class = TypeaheadCourseRunSearchSerializer

@classmethod
Expand All @@ -2809,9 +2802,7 @@ def serialize_course_run(self, course_run):
return serializer


@pytest.mark.django_db
@pytest.mark.usefixtures('elasticsearch_dsl_default_connection')
class TestTypeaheadProgramSearchSerializer:
class TestTypeaheadProgramSearchSerializer(ElasticsearchTestMixin, TestCase):
serializer_class = TypeaheadProgramSearchSerializer

@classmethod
Expand Down
5 changes: 5 additions & 0 deletions course_discovery/apps/core/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@ def set_alias(cls, connection, alias, index):

connection.indices.update_aliases(body)

@classmethod
def update_max_result_window(cls, connection, max_result_window, index):
if connection.indices.exists(index=index):
connection.indices.put_settings(body={"index": {"max_result_window": max_result_window}})

@classmethod
def create_index(cls, index, conn_name='default'):
"""
Expand Down
2 changes: 2 additions & 0 deletions course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,11 +117,13 @@ def _update(self, models, options):
)
if record_count_is_sane:
ElasticsearchUtils.set_alias(conn, alias, new_index_name)
ElasticsearchUtils.update_max_result_window(conn, settings.MAX_RESULT_WINDOW, new_index_name)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would be really nice if we could do this just once at the start of the loop.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be done only after set_alias.

indexes_pending.pop(new_index_name, None)
else:
indexes_pending[new_index_name] = index_info_string
else:
ElasticsearchUtils.set_alias(conn, alias, new_index_name)
ElasticsearchUtils.update_max_result_window(conn, settings.MAX_RESULT_WINDOW, new_index_name)
indexes_pending.pop(new_index_name, None)

for index_alias_mapper in alias_mappings:
Expand Down
6 changes: 4 additions & 2 deletions course_discovery/settings/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,11 +461,13 @@
# (by default it uses the database driver's default setting)
# https://docs.djangoproject.com/en/3.1/ref/models/querysets/#iterator
# Thus set the 'chunk_size'
ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 10000
ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 15000

# Defining default pagination for all requests to ElasticSearch,
# whose parameters 'size' and 'from' are not explicitly set.
ELASTICSEARCH_DSL_LOAD_PER_QUERY = 10000
ELASTICSEARCH_DSL_LOAD_PER_QUERY = 15000

MAX_RESULT_WINDOW = 15000

ELASTICSEARCH_DSL = {
'default': {'hosts': '127.0.0.1:9200'}
Expand Down
4 changes: 4 additions & 0 deletions course_discovery/settings/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,3 +157,7 @@
'error_msg': 'Course edit was unsuccessful. The course URL slug "[{url_slug}]" is an invalid format. Please ensure that the slug is in the format `boot-camps/<primary_subject>/<organization_name>-<course_title>`',
}},
}

ELASTICSEARCH_DSL_QUERYSET_PAGINATION = 10000

ELASTICSEARCH_DSL_LOAD_PER_QUERY = 10000
Loading