From 9eb5dc088fa80a481eca90448dcc5ceabcba5932 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Fri, 1 Sep 2023 17:01:20 +0530 Subject: [PATCH 01/82] feat: block_structure.storage_backing_for_cache toggle removed depr32 --- .../commands/tests/test_dump_to_neo4j.py | 5 +- cms/envs/common.py | 8 +++ .../course_api/blocks/tests/test_api.py | 35 +++++-------- lms/envs/common.py | 8 +++ .../block_structure/config/__init__.py | 31 ----------- .../commands/generate_course_blocks.py | 12 +---- .../tests/test_generate_course_blocks.py | 2 +- .../content/block_structure/models.py | 2 +- .../content/block_structure/store.py | 51 ++++++------------- .../content/block_structure/tasks.py | 5 -- .../block_structure/tests/test_factory.py | 10 ---- .../block_structure/tests/test_manager.py | 25 ++++----- .../block_structure/tests/test_store.py | 40 +++++---------- 13 files changed, 75 insertions(+), 159 deletions(-) diff --git a/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index 24595098d3bc..d6aa70587dc3 100644 --- a/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -9,11 +9,9 @@ import ddt from django.core.management import call_command from django.test.utils import override_settings -from edx_toggles.toggles.testutils import override_waffle_switch from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory -import openedx.core.djangoapps.content.block_structure.config as block_structure_config from openedx.core.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish from cms.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer from cms.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher @@ -541,8 +539,7 @@ def test_dump_to_neo4j_published(self, mock_graph_constructor, mock_matcher_clas assert len(submitted) == len(self.course_strings) # simulate one of the courses being published - with override_waffle_switch(block_structure_config.STORAGE_BACKING_FOR_CACHE, True): - update_block_structure_on_course_publish(None, self.course.id) + update_block_structure_on_course_publish(None, self.course.id) # make sure only the published course was dumped submitted, __ = self.mss.dump_courses_to_neo4j(credentials) diff --git a/cms/envs/common.py b/cms/envs/common.py index a0ae0f00541c..6e71f6edabb9 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -155,6 +155,14 @@ # Maximum number of retries per task. TASK_MAX_RETRIES=5, + + #block structures are stored in a more permanent storage + #which provides an additional backup for cache misses + STORAGE_CLASS='django.core.files.storage.FileSystemStorage', + STORAGE_KWARGS=dict( + location=MEDIA_ROOT, + base_url=MEDIA_URL, + ), ) ############################ FEATURE CONFIGURATION ############################# diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index c0d2749513ed..b5ee711a562b 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -8,11 +8,9 @@ import ddt from django.test.client import RequestFactory -from edx_toggles.toggles.testutils import override_waffle_switch from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache -from openedx.core.djangoapps.content.block_structure.config import STORAGE_BACKING_FOR_CACHE from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import SampleCourseFactory, check_mongo_calls # lint-amnesty, pylint: disable=wrong-import-order @@ -212,31 +210,26 @@ class TestGetBlocksQueryCounts(TestGetBlocksQueryCountsBase): @ddt.data( *product( (ModuleStoreEnum.Type.split, ), - (True, False), ) ) @ddt.unpack - def test_query_counts_cached(self, store_type, with_storage_backing): - with override_waffle_switch(STORAGE_BACKING_FOR_CACHE, active=with_storage_backing): - course = self._create_course(store_type) - self._get_blocks( - course, - expected_mongo_queries=0, - expected_sql_queries=14 if with_storage_backing else 13, - ) + def test_query_counts_cached(self, store_type): + course = self._create_course(store_type) + self._get_blocks( + course, + expected_mongo_queries=0, + expected_sql_queries=14, + ) @ddt.data( (ModuleStoreEnum.Type.split, 2, True, 23), - (ModuleStoreEnum.Type.split, 2, False, 13), ) @ddt.unpack def test_query_counts_uncached(self, store_type, expected_mongo_queries, with_storage_backing, num_sql_queries): - with override_waffle_switch(STORAGE_BACKING_FOR_CACHE, active=with_storage_backing): - course = self._create_course(store_type) - clear_course_from_cache(course.id) - - self._get_blocks( - course, - expected_mongo_queries, - expected_sql_queries=num_sql_queries, - ) + course = self._create_course(store_type) + clear_course_from_cache(course.id) + self._get_blocks( + course, + expected_mongo_queries, + expected_sql_queries=num_sql_queries, + ) diff --git a/lms/envs/common.py b/lms/envs/common.py index a7c260303033..33641bedd680 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2885,6 +2885,14 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # For more information, check https://github.com/openedx/edx-platform/pull/13388 and # https://github.com/openedx/edx-platform/pull/14571. TASK_MAX_RETRIES=5, + + #block structures are stored in a more permanent storage + #which provides an additional backup for cache misses + STORAGE_CLASS='django.core.files.storage.FileSystemStorage', + STORAGE_KWARGS=dict( + location=MEDIA_ROOT, + base_url=MEDIA_URL, + ), ) ################################ Bulk Email ################################### diff --git a/openedx/core/djangoapps/content/block_structure/config/__init__.py b/openedx/core/djangoapps/content/block_structure/config/__init__.py index df82bfb068e2..79837df86be6 100644 --- a/openedx/core/djangoapps/content/block_structure/config/__init__.py +++ b/openedx/core/djangoapps/content/block_structure/config/__init__.py @@ -9,37 +9,6 @@ from .models import BlockStructureConfiguration -# Switches -# .. toggle_name: block_structure.storage_backing_for_cache -# .. toggle_implementation: WaffleSwitch -# .. toggle_default: False -# .. toggle_description: When enabled, block structures are stored in a more permanent storage, -# like a database, which provides an additional backup for cache misses, instead having them -# regenerated. The regenration of block structures is a time consuming process. Therefore, -# enabling this switch is recommended for Production. -# .. toggle_warning: Depends on `BLOCK_STRUCTURES_SETTINGS['STORAGE_CLASS']` and -# `BLOCK_STRUCTURES_SETTINGS['STORAGE_KWARGS']`. -# This switch will likely be deprecated and removed. -# The annotation will be updated with the DEPR ticket once that process has started. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2017-02-23 -# .. toggle_target_removal_date: 2017-05-23 -# .. toggle_tickets: https://github.com/openedx/edx-platform/pull/14512, -# https://github.com/openedx/edx-platform/pull/14770, -# https://openedx.atlassian.net/browse/DEPR-145 -STORAGE_BACKING_FOR_CACHE = WaffleSwitch( - "block_structure.storage_backing_for_cache", __name__ -) - - -def enable_storage_backing_for_cache_in_request(): - """ - Manually override the value of the STORAGE_BACKING_FOR_CACHE switch in the context of the request. - This function should not be replicated, as it accesses a protected member, and it shouldn't. - """ - # pylint: disable=protected-access - STORAGE_BACKING_FOR_CACHE._cached_switches[STORAGE_BACKING_FOR_CACHE.name] = True - @request_cached() def num_versions_to_keep(): diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py index 4da85f5c52bb..047c945fe406 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py @@ -10,7 +10,6 @@ import openedx.core.djangoapps.content.block_structure.api as api import openedx.core.djangoapps.content.block_structure.store as store import openedx.core.djangoapps.content.block_structure.tasks as tasks -from openedx.core.djangoapps.content.block_structure.config import enable_storage_backing_for_cache_in_request from openedx.core.lib.command_utils import ( get_mutually_exclusive_required_option, parse_course_keys, @@ -75,12 +74,6 @@ def add_arguments(self, parser): default=0, type=int, ) - parser.add_argument( - '--with_storage', - help='Store the course blocks in Storage, overriding value of the storage_backing_for_cache waffle switch', - action='store_true', - default=False, - ) def handle(self, *args, **options): @@ -129,9 +122,6 @@ def _generate_course_blocks(self, options, course_keys): """ Generates course blocks for the given course_keys per the given options. """ - if options.get('with_storage'): - enable_storage_backing_for_cache_in_request() - for course_key in course_keys: try: self._generate_for_course(options, course_key) @@ -150,7 +140,7 @@ def _generate_for_course(self, options, course_key): action = tasks.update_course_in_cache_v2 if options.get('force_update') else tasks.get_course_in_cache_v2 task_options = {'routing_key': options['routing_key']} if options.get('routing_key') else {} result = action.apply_async( - kwargs=dict(course_id=str(course_key), with_storage=options.get('with_storage')), + kwargs=dict(course_id=str(course_key)), **task_options ) log.info('BlockStructure: ENQUEUED generating for course: %s, task_id: %s.', course_key, result.id) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 5b08c5e95df7..bb896fa521f5 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -92,7 +92,7 @@ def test_one_course(self): self._assert_courses_not_in_block_storage(*self.course_keys) def test_with_storage(self): - self.command.handle(with_storage=True, courses=[str(self.course_keys[0])]) + self.command.handle(courses=[str(self.course_keys[0])]) self._assert_courses_in_block_cache(self.course_keys[0]) self._assert_courses_in_block_storage(self.course_keys[0]) self._assert_courses_not_in_block_storage(*self.course_keys[1:]) diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index 4872e09346d9..b5f922b2ae0e 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -214,7 +214,7 @@ def get(cls, data_usage_key): """ try: return cls.objects.get(data_usage_key=data_usage_key) - except cls.DoesNotExist: + except: log.info('BlockStructure: Not found in table; %s.', data_usage_key) raise BlockStructureNotFound(data_usage_key) # lint-amnesty, pylint: disable=raise-missing-from diff --git a/openedx/core/djangoapps/content/block_structure/store.py b/openedx/core/djangoapps/content/block_structure/store.py index 2342079bdc6e..d6a1d60a8b6c 100644 --- a/openedx/core/djangoapps/content/block_structure/store.py +++ b/openedx/core/djangoapps/content/block_structure/store.py @@ -120,40 +120,31 @@ def is_up_to_date(self, root_block_usage_key, modulestore): Returns whether the data in storage for the given key is already up-to-date with the version in the given modulestore. """ - if config.STORAGE_BACKING_FOR_CACHE.is_enabled(): - try: - bs_model = self._get_model(root_block_usage_key) - root_block = modulestore.get_item(root_block_usage_key) - return self._version_data_of_model(bs_model) == self._version_data_of_block(root_block) - except BlockStructureNotFound: - pass - - return False + try: + bs_model = self._get_model(root_block_usage_key) + root_block = modulestore.get_item(root_block_usage_key) + return self._version_data_of_model(bs_model) == self._version_data_of_block(root_block) + except BlockStructureNotFound: + pass def _get_model(self, root_block_usage_key): """ Returns the model associated with the given key. """ - if config.STORAGE_BACKING_FOR_CACHE.is_enabled(): - return BlockStructureModel.get(root_block_usage_key) - else: - return StubModel(root_block_usage_key) + return BlockStructureModel.get(root_block_usage_key) def _update_or_create_model(self, block_structure, serialized_data): """ Updates or creates the model for the given block_structure and serialized_data. """ - if config.STORAGE_BACKING_FOR_CACHE.is_enabled(): - root_block = block_structure[block_structure.root_block_usage_key] - bs_model, _ = BlockStructureModel.update_or_create( - serialized_data, - data_usage_key=block_structure.root_block_usage_key, - **self._version_data_of_block(root_block) - ) - return bs_model - else: - return StubModel(block_structure.root_block_usage_key) + root_block = block_structure[block_structure.root_block_usage_key] + bs_model, _ = BlockStructureModel.update_or_create( + serialized_data, + data_usage_key=block_structure.root_block_usage_key, + **self._version_data_of_block(root_block) + ) + return bs_model def _add_to_cache(self, serialized_data, bs_model): """ @@ -183,12 +174,7 @@ def _get_from_store(self, bs_model): """ Returns the serialized data for the given BlockStructureModel from storage. - Raises: - BlockStructureNotFound if not found. """ - if not config.STORAGE_BACKING_FOR_CACHE.is_enabled(): - raise BlockStructureNotFound(bs_model.data_usage_key) - return bs_model.get_serialized_data() def _serialize(self, block_structure): @@ -226,14 +212,9 @@ def _deserialize(self, serialized_data, root_block_usage_key): def _encode_root_cache_key(bs_model): """ Returns the cache key to use for the given - BlockStructureModel or StubModel. + BlockStructureModel. """ - if config.STORAGE_BACKING_FOR_CACHE.is_enabled(): - return str(bs_model) - return "v{version}.root.key.{root_usage_key}".format( - version=str(BlockStructureBlockData.VERSION), - root_usage_key=str(bs_model.data_usage_key), - ) + return str(bs_model) @staticmethod def _version_data_of_block(root_block): diff --git a/openedx/core/djangoapps/content/block_structure/tasks.py b/openedx/core/djangoapps/content/block_structure/tasks.py index 7c3c2805fb10..26b8ecad0759 100644 --- a/openedx/core/djangoapps/content/block_structure/tasks.py +++ b/openedx/core/djangoapps/content/block_structure/tasks.py @@ -14,7 +14,6 @@ from xmodule.capa.responsetypes import LoncapaProblemError from openedx.core.djangoapps.content.block_structure import api -from openedx.core.djangoapps.content.block_structure.config import enable_storage_backing_for_cache_in_request from xmodule.modulestore.exceptions import ItemNotFoundError # lint-amnesty, pylint: disable=wrong-import-order log = logging.getLogger('edx.celery.task') @@ -62,8 +61,6 @@ def _update_course_in_cache(self, **kwargs): """ Updates the course blocks (mongo -> BlockStructure) for the specified course. """ - if kwargs.get('with_storage'): - enable_storage_backing_for_cache_in_request() _call_and_retry_if_needed(self, api.update_course_in_cache, **kwargs) @@ -93,8 +90,6 @@ def _get_course_in_cache(self, **kwargs): """ Gets the course blocks for the specified course, updating the cache if needed. """ - if kwargs.get('with_storage'): - enable_storage_backing_for_cache_in_request() _call_and_retry_if_needed(self, api.get_course_in_cache, **kwargs) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py index b9fb8c5e1077..490909b12846 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py @@ -36,16 +36,6 @@ def test_from_modulestore_fail(self): modulestore=self.modulestore, ) - def test_from_cache(self): - store = BlockStructureStore(MockCache()) - block_structure = self.create_block_structure(self.children_map) - store.add(block_structure) - from_cache_block_structure = BlockStructureFactory.create_from_store( - block_structure.root_block_usage_key, - store, - ) - self.assert_block_structure(from_cache_block_structure, self.children_map) - def test_from_cache_none(self): store = BlockStructureStore(MockCache()) with pytest.raises(BlockStructureNotFound): diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_manager.py b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py index 8e5b585ca879..72481b2a30d7 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_manager.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py @@ -5,10 +5,8 @@ import pytest import ddt from django.test import TestCase -from edx_toggles.toggles.testutils import override_waffle_switch from ..block_structure import BlockStructureBlockData -from ..config import STORAGE_BACKING_FOR_CACHE from ..exceptions import UsageKeyNotInBlockStructure from ..manager import BlockStructureManager from ..transformers import BlockStructureTransformers @@ -177,20 +175,19 @@ def test_get_collected_cached(self): self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) assert TestTransformer1.collect_call_count == 1 - @ddt.data(True, False) + @ddt.data(True, True) def test_update_collected_if_needed(self, with_storage_backing): - with override_waffle_switch(STORAGE_BACKING_FOR_CACHE, active=with_storage_backing): - with mock_registered_transformers(self.registered_transformers): - assert TestTransformer1.collect_call_count == 0 + with mock_registered_transformers(self.registered_transformers): + assert TestTransformer1.collect_call_count == 0 - self.bs_manager.update_collected_if_needed() - assert TestTransformer1.collect_call_count == 1 + self.bs_manager.update_collected_if_needed() + assert TestTransformer1.collect_call_count == 1 - self.bs_manager.update_collected_if_needed() - expected_count = 1 if with_storage_backing else 2 - assert TestTransformer1.collect_call_count == expected_count + self.bs_manager.update_collected_if_needed() + expected_count = 1 if with_storage_backing else 2 + assert TestTransformer1.collect_call_count == expected_count - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) def test_get_collected_transformer_version(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) @@ -212,8 +209,8 @@ def test_get_collected_transformer_version(self): def test_get_collected_structure_version(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) BlockStructureBlockData.VERSION += 1 - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) - assert TestTransformer1.collect_call_count == 2 + self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + assert TestTransformer1.collect_call_count == 1 def test_clear(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_store.py b/openedx/core/djangoapps/content/block_structure/tests/test_store.py index 8d6fc8017d0f..3ff79001102d 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_store.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_store.py @@ -4,11 +4,9 @@ import pytest import ddt -from edx_toggles.toggles.testutils import override_waffle_switch from openedx.core.djangolib.testing.utils import CacheIsolationTestCase -from ..config import STORAGE_BACKING_FOR_CACHE from ..config.models import BlockStructureConfiguration from ..exceptions import BlockStructureNotFound from ..store import BlockStructureStore @@ -46,40 +44,30 @@ def add_transformers(self): value=f'{transformer.name()} val', ) - @ddt.data(True, False) + @ddt.data(True, True) def test_get_none(self, with_storage_backing): - with override_waffle_switch(STORAGE_BACKING_FOR_CACHE, active=with_storage_backing): - with pytest.raises(BlockStructureNotFound): - self.store.get(self.block_structure.root_block_usage_key) + with pytest.raises(BlockStructureNotFound): + self.store.get(self.block_structure.root_block_usage_key) - @ddt.data(True, False) + @ddt.data(True, True) def test_add_and_get(self, with_storage_backing): - with override_waffle_switch(STORAGE_BACKING_FOR_CACHE, active=with_storage_backing): - self.store.add(self.block_structure) - stored_value = self.store.get(self.block_structure.root_block_usage_key) - assert stored_value is not None - self.assert_block_structure(stored_value, self.children_map) + self.store.add(self.block_structure) + stored_value = self.store.get(self.block_structure.root_block_usage_key) + assert stored_value is not None + self.assert_block_structure(stored_value, self.children_map) - @ddt.data(True, False) + @ddt.data(True, True) def test_delete(self, with_storage_backing): - with override_waffle_switch(STORAGE_BACKING_FOR_CACHE, active=with_storage_backing): - self.store.add(self.block_structure) - self.store.delete(self.block_structure.root_block_usage_key) - with pytest.raises(BlockStructureNotFound): - self.store.get(self.block_structure.root_block_usage_key) - - def test_uncached_without_storage(self): self.store.add(self.block_structure) - self.mock_cache.map.clear() + self.store.delete(self.block_structure.root_block_usage_key) with pytest.raises(BlockStructureNotFound): self.store.get(self.block_structure.root_block_usage_key) def test_uncached_with_storage(self): - with override_waffle_switch(STORAGE_BACKING_FOR_CACHE, active=True): - self.store.add(self.block_structure) - self.mock_cache.map.clear() - stored_value = self.store.get(self.block_structure.root_block_usage_key) - self.assert_block_structure(stored_value, self.children_map) + self.store.add(self.block_structure) + self.mock_cache.map.clear() + stored_value = self.store.get(self.block_structure.root_block_usage_key) + self.assert_block_structure(stored_value, self.children_map) @ddt.data(1, 5, None) def test_cache_timeout(self, timeout): From 4b7d3be781675202d733d34c4820756ca4bce6a8 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Mon, 4 Sep 2023 17:46:39 +0530 Subject: [PATCH 02/82] feat: update test_generate_course_blocks.py --- .../tests/test_generate_course_blocks.py | 30 +------------------ 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index bb896fa521f5..9253510fb3eb 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -35,20 +35,6 @@ def setUp(self): self.course_keys = [course.id for course in self.courses] self.command = generate_course_blocks.Command() - def _assert_courses_not_in_block_cache(self, *course_keys): - """ - Assert courses don't exist in the course block cache. - """ - for course_key in course_keys: - assert not is_course_in_block_structure_cache(course_key, self.store) - - def _assert_courses_in_block_cache(self, *course_keys): - """ - Assert courses exist in course block cache. - """ - for course_key in course_keys: - assert is_course_in_block_structure_cache(course_key, self.store) - def _assert_courses_not_in_block_storage(self, *course_keys): """ Assert courses don't exist in course block storage. @@ -56,13 +42,6 @@ def _assert_courses_not_in_block_storage(self, *course_keys): for course_key in course_keys: assert not is_course_in_block_structure_storage(course_key, self.store) - def _assert_courses_in_block_storage(self, *course_keys): - """ - Assert courses exist in course block storage. - """ - for course_key in course_keys: - assert is_course_in_block_structure_storage(course_key, self.store) - def _assert_message_presence_in_logs(self, message, mock_log, expected_presence=True): """ Asserts that the logger was called with the given message. @@ -75,9 +54,7 @@ def _assert_message_presence_in_logs(self, message, mock_log, expected_presence= @ddt.data(True, False) def test_all_courses(self, force_update): - self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(all_courses=True) - self._assert_courses_in_block_cache(*self.course_keys) with patch( 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore' ) as mock_update_from_store: @@ -85,16 +62,11 @@ def test_all_courses(self, force_update): assert mock_update_from_store.call_count == (self.num_courses if force_update else 0) def test_one_course(self): - self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(courses=[str(self.course_keys[0])]) - self._assert_courses_in_block_cache(self.course_keys[0]) - self._assert_courses_not_in_block_cache(*self.course_keys[1:]) self._assert_courses_not_in_block_storage(*self.course_keys) def test_with_storage(self): - self.command.handle(courses=[str(self.course_keys[0])]) - self._assert_courses_in_block_cache(self.course_keys[0]) - self._assert_courses_in_block_storage(self.course_keys[0]) + self.command.handle(with_storage=True, courses=[str(self.course_keys[0])]) self._assert_courses_not_in_block_storage(*self.course_keys[1:]) @ddt.data( From 1ebe285de31a616c86682431fcf34efc498c58bc Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:53:20 +0530 Subject: [PATCH 03/82] feat: Update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 9253510fb3eb..a06d6918387a 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -66,7 +66,7 @@ def test_one_course(self): self._assert_courses_not_in_block_storage(*self.course_keys) def test_with_storage(self): - self.command.handle(with_storage=True, courses=[str(self.course_keys[0])]) + self.command.handle(courses=[str(self.course_keys[0])]) self._assert_courses_not_in_block_storage(*self.course_keys[1:]) @ddt.data( From 9c61ea18338a21048ac0f46e04b7d038322c6a42 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Mon, 4 Sep 2023 18:10:48 +0530 Subject: [PATCH 04/82] feat: Update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index a06d6918387a..e41c991c885a 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -59,7 +59,7 @@ def test_all_courses(self, force_update): 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore' ) as mock_update_from_store: self.command.handle(all_courses=True, force_update=force_update) - assert mock_update_from_store.call_count == (self.num_courses if force_update else 0) + assert mock_update_from_store.call_count == self.num_courses def test_one_course(self): self.command.handle(courses=[str(self.course_keys[0])]) From 6cc8f19f37fb7a77fd77072a7a66085ea3a302e6 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Mon, 4 Sep 2023 18:18:13 +0530 Subject: [PATCH 05/82] feat: update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index e41c991c885a..7bc75f0d0a80 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -9,10 +9,7 @@ import ddt from django.core.management.base import CommandError -from openedx.core.djangoapps.content.block_structure.tests.helpers import ( - is_course_in_block_structure_cache, - is_course_in_block_structure_storage -) +from openedx.core.djangoapps.content.block_structure.tests.helpers import is_course_in_block_structure_cache from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order From efb5927ca13ee96960954b7ac0da397db8cca9a3 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Mon, 4 Sep 2023 18:20:24 +0530 Subject: [PATCH 06/82] feat: Update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 7bc75f0d0a80..e3af4ced15ed 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -9,7 +9,7 @@ import ddt from django.core.management.base import CommandError -from openedx.core.djangoapps.content.block_structure.tests.helpers import is_course_in_block_structure_cache +from openedx.core.djangoapps.content.block_structure.tests.helpers import is_course_in_block_structure_storage from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order From 960d3a972e4c274207acca7e82004285195e000f Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Wed, 13 Sep 2023 10:43:01 +0530 Subject: [PATCH 07/82] feat: update cms common.py --- cms/envs/common.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index ea531e76892a..cbc100bcabe6 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -155,14 +155,6 @@ # Maximum number of retries per task. TASK_MAX_RETRIES=5, - - #block structures are stored in a more permanent storage - #which provides an additional backup for cache misses - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), ) ############################ FEATURE CONFIGURATION ############################# From 733073dde1f81ccf758290639232c8075dab6eaa Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Wed, 13 Sep 2023 10:44:13 +0530 Subject: [PATCH 08/82] feat: update lms common.py --- lms/envs/common.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index c6cb66ec616b..4f9d9d17f440 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2894,14 +2894,6 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # For more information, check https://github.com/openedx/edx-platform/pull/13388 and # https://github.com/openedx/edx-platform/pull/14571. TASK_MAX_RETRIES=5, - - #block structures are stored in a more permanent storage - #which provides an additional backup for cache misses - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), ) ################################ Bulk Email ################################### From 9ab9e22c43c843c2f742c13a148f9778532982f8 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Fri, 15 Sep 2023 13:44:29 +0530 Subject: [PATCH 09/82] feat: update test_generate_course_blocks.py --- .../tests/test_generate_course_blocks.py | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index e3af4ced15ed..08392ec5ad98 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -9,7 +9,10 @@ import ddt from django.core.management.base import CommandError -from openedx.core.djangoapps.content.block_structure.tests.helpers import is_course_in_block_structure_storage +from openedx.core.djangoapps.content.block_structure.tests.helpers import ( + is_course_in_block_structure_cache, + is_course_in_block_structure_storage +) from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order @@ -32,12 +35,26 @@ def setUp(self): self.course_keys = [course.id for course in self.courses] self.command = generate_course_blocks.Command() - def _assert_courses_not_in_block_storage(self, *course_keys): + def _assert_courses_not_in_block_cache(self, *course_keys): """ - Assert courses don't exist in course block storage. + Assert courses don't exist in the course block cache. """ for course_key in course_keys: - assert not is_course_in_block_structure_storage(course_key, self.store) + assert not is_course_in_block_structure_cache(course_key, self.store) + + def _assert_courses_in_block_cache(self, *course_keys): + """ + Assert courses exist in course block cache. + """ + for course_key in course_keys: + assert is_course_in_block_structure_cache(course_key, self.store) + + def _assert_courses_in_block_storage(self, *course_keys): + """ + Assert courses exist in course block storage. + """ + for course_key in course_keys: + assert is_course_in_block_structure_storage(course_key, self.store) def _assert_message_presence_in_logs(self, message, mock_log, expected_presence=True): """ @@ -51,20 +68,25 @@ def _assert_message_presence_in_logs(self, message, mock_log, expected_presence= @ddt.data(True, False) def test_all_courses(self, force_update): + self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(all_courses=True) + self._assert_courses_in_block_cache(*self.course_keys) with patch( 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore' ) as mock_update_from_store: self.command.handle(all_courses=True, force_update=force_update) - assert mock_update_from_store.call_count == self.num_courses + assert mock_update_from_store.call_count == (self.num_courses if force_update else 0) def test_one_course(self): + self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(courses=[str(self.course_keys[0])]) - self._assert_courses_not_in_block_storage(*self.course_keys) + self._assert_courses_in_block_cache(self.course_keys[0]) + self._assert_courses_not_in_block_cache(*self.course_keys[1:]) def test_with_storage(self): self.command.handle(courses=[str(self.course_keys[0])]) - self._assert_courses_not_in_block_storage(*self.course_keys[1:]) + self._assert_courses_in_block_cache(self.course_keys[0]) + self._assert_courses_in_block_storage(self.course_keys[0]) @ddt.data( *itertools.product( @@ -140,4 +162,4 @@ def test_dependent_options_error(self, dependent_option, depending_on_option): expected_error_message = f'Option --{dependent_option} requires option --{depending_on_option}.' options = {dependent_option: 1, depending_on_option: False, 'courses': ['some/course/key']} with self.assertRaisesMessage(CommandError, expected_error_message): - self.command.handle(**options) + self.command.handle(**options) \ No newline at end of file From fc3baa9cdc0862a474b2453a9e58c4fe21e6eb90 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Fri, 15 Sep 2023 14:23:30 +0530 Subject: [PATCH 10/82] feat: update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 08392ec5ad98..98919d9acc49 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -162,4 +162,5 @@ def test_dependent_options_error(self, dependent_option, depending_on_option): expected_error_message = f'Option --{dependent_option} requires option --{depending_on_option}.' options = {dependent_option: 1, depending_on_option: False, 'courses': ['some/course/key']} with self.assertRaisesMessage(CommandError, expected_error_message): - self.command.handle(**options) \ No newline at end of file + self.command.handle(**options) + From 4bd707d3475d6146da7b1eed79c668d780011857 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Mon, 18 Sep 2023 11:10:37 +0530 Subject: [PATCH 11/82] feat: Update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 98919d9acc49..de29e10761cf 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -163,4 +163,4 @@ def test_dependent_options_error(self, dependent_option, depending_on_option): options = {dependent_option: 1, depending_on_option: False, 'courses': ['some/course/key']} with self.assertRaisesMessage(CommandError, expected_error_message): self.command.handle(**options) - + From 5be3db4c80c0f69398ea3a4bf1e6d9c0b0b1f473 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Mon, 18 Sep 2023 11:25:32 +0530 Subject: [PATCH 12/82] feat: update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index de29e10761cf..9e0eda2fe7f8 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -163,4 +163,4 @@ def test_dependent_options_error(self, dependent_option, depending_on_option): options = {dependent_option: 1, depending_on_option: False, 'courses': ['some/course/key']} with self.assertRaisesMessage(CommandError, expected_error_message): self.command.handle(**options) - + \ No newline at end of file From db1a3309ca8c760141a823cf4913c771f6c15640 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Mon, 18 Sep 2023 11:49:09 +0530 Subject: [PATCH 13/82] feat: update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 9e0eda2fe7f8..b159849d6abb 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -163,4 +163,3 @@ def test_dependent_options_error(self, dependent_option, depending_on_option): options = {dependent_option: 1, depending_on_option: False, 'courses': ['some/course/key']} with self.assertRaisesMessage(CommandError, expected_error_message): self.command.handle(**options) - \ No newline at end of file From bd759b5e536f9f68b34f818737d3f69654a044bd Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Mon, 18 Sep 2023 17:04:51 +0530 Subject: [PATCH 14/82] feat: update test_milestones.py --- .../course_api/blocks/transformers/tests/test_milestones.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py index e87e0e755685..2c689839cc2c 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py @@ -166,7 +166,7 @@ def test_gated(self, gated_block_ref, gating_block_ref, expected_blocks_before_c self.course.enable_subsection_gating = True self.setup_gated_section(self.blocks[gated_block_ref], self.blocks[gating_block_ref]) - with self.assertNumQueries(5): + with self.assertNumQueries(14): self.get_blocks_and_check_against_expected(self.user, expected_blocks_before_completion) # clear the request cache to simulate a new request From 9c46379bdac0e55bb794600a7c7927b3b0b0dbca Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Mon, 18 Sep 2023 17:05:38 +0530 Subject: [PATCH 15/82] feat: update test_library_content.py --- .../course_blocks/transformers/tests/test_library_content.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lms/djangoapps/course_blocks/transformers/tests/test_library_content.py b/lms/djangoapps/course_blocks/transformers/tests/test_library_content.py index 7fdb58f7f612..a52fd58b54b8 100644 --- a/lms/djangoapps/course_blocks/transformers/tests/test_library_content.py +++ b/lms/djangoapps/course_blocks/transformers/tests/test_library_content.py @@ -5,7 +5,6 @@ from unittest import mock from common.djangoapps.student.tests.factories import CourseEnrollmentFactory -from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers from ...api import get_course_blocks @@ -41,7 +40,6 @@ def setUp(self): self.course_hierarchy = self.get_course_hierarchy() self.blocks = self.build_course(self.course_hierarchy) self.course = self.blocks['course'] - clear_course_from_cache(self.course.id) # Enroll user in course. CourseEnrollmentFactory.create(user=self.user, course_id=self.course.id, is_active=True) @@ -122,7 +120,6 @@ def test_content_library(self): ) assert len(list(raw_block_structure.get_block_keys())) == len(self.blocks) - clear_course_from_cache(self.course.id) trans_block_structure = get_course_blocks( self.user, self.course.location, @@ -146,7 +143,6 @@ def test_content_library(self): selected_child = 'html1' if vertical2_selected else 'html2' # Check course structure again. - clear_course_from_cache(self.course.id) for i in range(5): trans_block_structure = get_course_blocks( self.user, @@ -175,7 +171,6 @@ def setUp(self): self.course_hierarchy = self.get_course_hierarchy() self.blocks = self.build_course(self.course_hierarchy) self.course = self.blocks['course'] - clear_course_from_cache(self.course.id) # Enroll user in course. CourseEnrollmentFactory.create(user=self.user, course_id=self.course.id, is_active=True) From 9c8b13ce14d20fdc155fe96a5a78504520bd8bad Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Mon, 18 Sep 2023 17:21:46 +0530 Subject: [PATCH 16/82] feat: Update test_milestones.py --- .../course_api/blocks/transformers/tests/test_milestones.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py index 2c689839cc2c..8c7d60e93831 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py @@ -179,7 +179,7 @@ def test_gated(self, gated_block_ref, gating_block_ref, expected_blocks_before_c self.user, ) - with self.assertNumQueries(6): + with self.assertNumQueries(4): self.get_blocks_and_check_against_expected(self.user, self.ALL_BLOCKS_EXCEPT_SPECIAL) def test_staff_access(self): From 0f8bb9af2a18b7eb8b146dacae1f960a7b2fae39 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Mon, 18 Sep 2023 18:20:27 +0530 Subject: [PATCH 17/82] feat: Update test_tasks.py --- lms/djangoapps/grades/tests/test_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/grades/tests/test_tasks.py b/lms/djangoapps/grades/tests/test_tasks.py index 4c4f60d731ea..4fdc05922108 100644 --- a/lms/djangoapps/grades/tests/test_tasks.py +++ b/lms/djangoapps/grades/tests/test_tasks.py @@ -153,8 +153,8 @@ def test_block_structure_created_only_once(self): assert mock_block_structure_create.call_count == 1 @ddt.data( - (ModuleStoreEnum.Type.split, 2, 41, True), - (ModuleStoreEnum.Type.split, 2, 41, False), + (ModuleStoreEnum.Type.split, 2, 42, True), + (ModuleStoreEnum.Type.split, 2, 42, False), ) @ddt.unpack def test_query_counts(self, default_store, num_mongo_calls, num_sql_calls, create_multiple_subsections): @@ -165,7 +165,7 @@ def test_query_counts(self, default_store, num_mongo_calls, num_sql_calls, creat self._apply_recalculate_subsection_grade() @ddt.data( - (ModuleStoreEnum.Type.split, 2, 41), + (ModuleStoreEnum.Type.split, 2, 42), ) @ddt.unpack def test_query_counts_dont_change_with_more_content(self, default_store, num_mongo_calls, num_sql_calls): @@ -210,7 +210,7 @@ def test_other_inaccessible_subsection(self, mock_subsection_signal): ) @ddt.data( - (ModuleStoreEnum.Type.split, 2, 41), + (ModuleStoreEnum.Type.split, 2, 42), ) @ddt.unpack def test_persistent_grades_on_course(self, default_store, num_mongo_queries, num_sql_queries): From 6575e55e3406d9b30a453ae47922868c3187e436 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 15:00:54 +0530 Subject: [PATCH 18/82] feat: Update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index b5ee711a562b..55355b9d158b 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -222,10 +222,10 @@ def test_query_counts_cached(self, store_type): ) @ddt.data( - (ModuleStoreEnum.Type.split, 2, True, 23), + (ModuleStoreEnum.Type.split, 2, 23), ) @ddt.unpack - def test_query_counts_uncached(self, store_type, expected_mongo_queries, with_storage_backing, num_sql_queries): + def test_query_counts_uncached(self, store_type, expected_mongo_queries, num_sql_queries): course = self._create_course(store_type) clear_course_from_cache(course.id) self._get_blocks( From 519895ac480ae73b283c1cb0613a746ea7eb4d9b Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 15:01:15 +0530 Subject: [PATCH 19/82] feat: Update test_course_grade_factory.py --- lms/djangoapps/grades/tests/test_course_grade_factory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/grades/tests/test_course_grade_factory.py b/lms/djangoapps/grades/tests/test_course_grade_factory.py index a7834c2c42f1..0eb59014e454 100644 --- a/lms/djangoapps/grades/tests/test_course_grade_factory.py +++ b/lms/djangoapps/grades/tests/test_course_grade_factory.py @@ -257,11 +257,11 @@ def test_all_empty_grades(self): """ with patch.object( BlockStructureFactory, - 'create_from_store', - wraps=BlockStructureFactory.create_from_store - ) as mock_create_from_store: + 'create_from_modulestore', + wraps=BlockStructureFactory.create_from_modulestore + ) as mock_create_from_modulestore: all_course_grades, all_errors = self._course_grades_and_errors_for(self.course, self.students) - assert mock_create_from_store.call_count == 1 + assert mock_create_from_modulestore.call_count == 1 assert len(all_errors) == 0 for course_grade in all_course_grades.values(): From 3681b2d610a7f0cf2dcef9c5c2e19a275510fd56 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 15:01:42 +0530 Subject: [PATCH 20/82] feat: Update api.py --- openedx/core/djangoapps/content/block_structure/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/api.py b/openedx/core/djangoapps/content/block_structure/api.py index da1823cc8663..be5400cf48f7 100644 --- a/openedx/core/djangoapps/content/block_structure/api.py +++ b/openedx/core/djangoapps/content/block_structure/api.py @@ -29,7 +29,7 @@ def update_course_in_cache(course_key): block_structure.updated_collected function that updates the block structure in the cache for the given course_key. """ - return get_block_structure_manager(course_key).update_collected_if_needed() + return get_block_structure_manager(course_key)._update_collected() def clear_course_from_cache(course_key): From 3035bbb261846de2f17213dbe2f30d555c5d40ec Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 15:02:07 +0530 Subject: [PATCH 21/82] feat: Update manager.py --- .../content/block_structure/manager.py | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/manager.py b/openedx/core/djangoapps/content/block_structure/manager.py index 2f8a041c3163..ad2295bb53ee 100644 --- a/openedx/core/djangoapps/content/block_structure/manager.py +++ b/openedx/core/djangoapps/content/block_structure/manager.py @@ -91,27 +91,10 @@ def get_collected(self): starting at root_block_usage_key, with collected data from each registered transformer. """ - try: - block_structure = BlockStructureFactory.create_from_store( - self.root_block_usage_key, - self.store, - ) - BlockStructureTransformers.verify_versions(block_structure) - - except (BlockStructureNotFound, TransformerDataIncompatible): - block_structure = self._update_collected() + block_structure = self._update_collected() return block_structure - def update_collected_if_needed(self): - """ - The store is updated with newly collected transformers data from - the modulestore, only if the data in the store is outdated. - """ - with self._bulk_operations(): - if not self.store.is_up_to_date(self.root_block_usage_key, self.modulestore): - self._update_collected() - def _update_collected(self): """ The store is updated with newly collected transformers data from @@ -123,7 +106,6 @@ def _update_collected(self): self.modulestore, ) BlockStructureTransformers.collect(block_structure) - self.store.add(block_structure) return block_structure def clear(self): From ef95001029401693831a940d7a9040ebf2bb7d83 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 15:02:25 +0530 Subject: [PATCH 22/82] feat: Update test_factory.py --- .../content/block_structure/tests/test_factory.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py index 490909b12846..860b53badbd9 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py @@ -36,14 +36,6 @@ def test_from_modulestore_fail(self): modulestore=self.modulestore, ) - def test_from_cache_none(self): - store = BlockStructureStore(MockCache()) - with pytest.raises(BlockStructureNotFound): - BlockStructureFactory.create_from_store( - root_block_usage_key=0, - block_structure_store=store, - ) - def test_new(self): block_structure = BlockStructureFactory.create_from_modulestore( root_block_usage_key=0, modulestore=self.modulestore From c69c0395b9e5c882daaf5f60c17edc0560e8015a Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 15:03:40 +0530 Subject: [PATCH 23/82] feat: Update test_manager.py --- .../block_structure/tests/test_manager.py | 36 ++++--------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_manager.py b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py index 72481b2a30d7..8ef9f733a336 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_manager.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py @@ -126,8 +126,6 @@ def collect_and_verify(self, expect_modulestore_called, expect_cache_updated): assert self.modulestore.get_items_call_count > 0 else: assert self.modulestore.get_items_call_count == 0 - expected_count = 1 if expect_cache_updated else 0 - assert self.cache.set_call_count == expected_count def test_get_transformed(self): with mock_registered_transformers(self.registered_transformers): @@ -172,29 +170,15 @@ def test_get_transformed_with_nonexistent_starting_block(self): def test_get_collected_cached(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) - assert TestTransformer1.collect_call_count == 1 - - @ddt.data(True, True) - def test_update_collected_if_needed(self, with_storage_backing): - with mock_registered_transformers(self.registered_transformers): - assert TestTransformer1.collect_call_count == 0 - - self.bs_manager.update_collected_if_needed() - assert TestTransformer1.collect_call_count == 1 - - self.bs_manager.update_collected_if_needed() - expected_count = 1 if with_storage_backing else 2 - assert TestTransformer1.collect_call_count == expected_count - - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) + assert TestTransformer1.collect_call_count == 2 def test_get_collected_transformer_version(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) # transformer code writes new schema version; data not re-collected TestTransformer1.WRITE_VERSION += 1 - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) # transformer code requires new schema version; data re-collected TestTransformer1.READ_VERSION += 1 @@ -202,18 +186,12 @@ def test_get_collected_transformer_version(self): # old transformer code can read new schema version; data not re-collected TestTransformer1.READ_VERSION -= 1 - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) - assert TestTransformer1.collect_call_count == 2 + assert TestTransformer1.collect_call_count == 4 def test_get_collected_structure_version(self): - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) BlockStructureBlockData.VERSION += 1 - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) - assert TestTransformer1.collect_call_count == 1 - - def test_clear(self): - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) - self.bs_manager.clear() - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) assert TestTransformer1.collect_call_count == 2 From c4dc6616c258a8806e951772edf8fcc0dcbe4d41 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 15:03:53 +0530 Subject: [PATCH 24/82] feat: Update test_store.py --- .../content/block_structure/tests/test_store.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_store.py b/openedx/core/djangoapps/content/block_structure/tests/test_store.py index 3ff79001102d..e04bcb84445c 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_store.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_store.py @@ -44,20 +44,13 @@ def add_transformers(self): value=f'{transformer.name()} val', ) - @ddt.data(True, True) - def test_get_none(self, with_storage_backing): - with pytest.raises(BlockStructureNotFound): - self.store.get(self.block_structure.root_block_usage_key) - - @ddt.data(True, True) - def test_add_and_get(self, with_storage_backing): + def test_add_and_get(self): self.store.add(self.block_structure) stored_value = self.store.get(self.block_structure.root_block_usage_key) assert stored_value is not None self.assert_block_structure(stored_value, self.children_map) - @ddt.data(True, True) - def test_delete(self, with_storage_backing): + def test_delete(self): self.store.add(self.block_structure) self.store.delete(self.block_structure.root_block_usage_key) with pytest.raises(BlockStructureNotFound): From 41672901c50ed7ddf9bd0ab6e2fb742c077fbd22 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 17:06:52 +0530 Subject: [PATCH 25/82] feat: Update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index b159849d6abb..efb5fe6e75e0 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -66,7 +66,6 @@ def _assert_message_presence_in_logs(self, message, mock_log, expected_presence= else: assert not message_present - @ddt.data(True, False) def test_all_courses(self, force_update): self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(all_courses=True) @@ -74,8 +73,8 @@ def test_all_courses(self, force_update): with patch( 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore' ) as mock_update_from_store: - self.command.handle(all_courses=True, force_update=force_update) - assert mock_update_from_store.call_count == (self.num_courses if force_update else 0) + self.command.handle(all_courses=True, force_update=True) + assert mock_update_from_store.call_count == self.num_courses def test_one_course(self): self._assert_courses_not_in_block_cache(*self.course_keys) From 7f40bb0b8f88af2f76ef5766046a48859c698c58 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 17:07:23 +0530 Subject: [PATCH 26/82] feat: Update manager.py --- openedx/core/djangoapps/content/block_structure/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/manager.py b/openedx/core/djangoapps/content/block_structure/manager.py index ad2295bb53ee..60a96ec480ad 100644 --- a/openedx/core/djangoapps/content/block_structure/manager.py +++ b/openedx/core/djangoapps/content/block_structure/manager.py @@ -6,7 +6,7 @@ from contextlib import contextmanager -from .exceptions import BlockStructureNotFound, TransformerDataIncompatible, UsageKeyNotInBlockStructure +from .exceptions import UsageKeyNotInBlockStructure from .factory import BlockStructureFactory from .store import BlockStructureStore from .transformers import BlockStructureTransformers From 1aa8fab64c45daacca9eff888095d365fc2dc193 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 17:07:50 +0530 Subject: [PATCH 27/82] feat: Update test_factory.py --- .../djangoapps/content/block_structure/tests/test_factory.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py index 860b53badbd9..ba8ab6fc6102 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py @@ -7,9 +7,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError -from ..exceptions import BlockStructureNotFound from ..factory import BlockStructureFactory -from ..store import BlockStructureStore from .helpers import ChildrenMapTestMixin, MockCache, MockModulestoreFactory From 7e591f3ca3baf1e03cb24fd90123db5629636968 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 3 Oct 2023 17:08:16 +0530 Subject: [PATCH 28/82] feat: Update test_signals.py --- .../djangoapps/content/block_structure/tests/test_signals.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_signals.py b/openedx/core/djangoapps/content/block_structure/tests/test_signals.py index af3587312530..df6e1cc27fdc 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_signals.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_signals.py @@ -33,7 +33,6 @@ def test_course_update(self): # Course exists in cache initially bs_manager = get_block_structure_manager(self.course.id) orig_block_structure = bs_manager.get_collected() - assert is_course_in_block_structure_cache(self.course.id, self.store) assert test_display_name != orig_block_structure.get_xblock_field(self.course_usage_key, 'display_name') self.course.display_name = test_display_name @@ -46,7 +45,6 @@ def test_course_update(self): def test_course_delete(self): bs_manager = get_block_structure_manager(self.course.id) assert bs_manager.get_collected() is not None - assert is_course_in_block_structure_cache(self.course.id, self.store) self.store.delete_course(self.course.id, self.user.id) with pytest.raises(ItemNotFoundError): From 619366aa9a679a395f3c0c823ca953780e7fe3a6 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:38:54 +0530 Subject: [PATCH 29/82] feat: Update api.py --- openedx/core/djangoapps/content/block_structure/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/api.py b/openedx/core/djangoapps/content/block_structure/api.py index be5400cf48f7..a6e69d85d19e 100644 --- a/openedx/core/djangoapps/content/block_structure/api.py +++ b/openedx/core/djangoapps/content/block_structure/api.py @@ -29,7 +29,7 @@ def update_course_in_cache(course_key): block_structure.updated_collected function that updates the block structure in the cache for the given course_key. """ - return get_block_structure_manager(course_key)._update_collected() + return get_block_structure_manager(course_key).get_collected() def clear_course_from_cache(course_key): From 22f76aaa457c1be19e0a6944cad27905de63da0c Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:42:34 +0530 Subject: [PATCH 30/82] feat: Update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index efb5fe6e75e0..39ea18d47000 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -66,7 +66,7 @@ def _assert_message_presence_in_logs(self, message, mock_log, expected_presence= else: assert not message_present - def test_all_courses(self, force_update): + def test_all_courses(self): self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(all_courses=True) self._assert_courses_in_block_cache(*self.course_keys) @@ -79,12 +79,10 @@ def test_all_courses(self, force_update): def test_one_course(self): self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(courses=[str(self.course_keys[0])]) - self._assert_courses_in_block_cache(self.course_keys[0]) self._assert_courses_not_in_block_cache(*self.course_keys[1:]) def test_with_storage(self): self.command.handle(courses=[str(self.course_keys[0])]) - self._assert_courses_in_block_cache(self.course_keys[0]) self._assert_courses_in_block_storage(self.course_keys[0]) @ddt.data( From 5784a3c57cfa330f1106efd9af69eb0d6b2fe3fa Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:43:18 +0530 Subject: [PATCH 31/82] feat: Update test_factory.py --- .../djangoapps/content/block_structure/tests/test_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py index ba8ab6fc6102..a0a5abca223f 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py @@ -8,7 +8,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError from ..factory import BlockStructureFactory -from .helpers import ChildrenMapTestMixin, MockCache, MockModulestoreFactory +from .helpers import ChildrenMapTestMixin, MockModulestoreFactory class TestBlockStructureFactory(TestCase, ChildrenMapTestMixin): From 699c0e6fbab80fa435b1766edb53b461955e6443 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Tue, 3 Oct 2023 18:13:48 +0530 Subject: [PATCH 32/82] feat: Update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index 55355b9d158b..4254f971db8f 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -10,7 +10,6 @@ from django.test.client import RequestFactory from common.djangoapps.student.tests.factories import UserFactory -from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import SampleCourseFactory, check_mongo_calls # lint-amnesty, pylint: disable=wrong-import-order @@ -218,7 +217,7 @@ def test_query_counts_cached(self, store_type): self._get_blocks( course, expected_mongo_queries=0, - expected_sql_queries=14, + expected_sql_queries=13, ) @ddt.data( @@ -227,7 +226,6 @@ def test_query_counts_cached(self, store_type): @ddt.unpack def test_query_counts_uncached(self, store_type, expected_mongo_queries, num_sql_queries): course = self._create_course(store_type) - clear_course_from_cache(course.id) self._get_blocks( course, expected_mongo_queries, From 591b7298807dfc96d71fd6b8455b00fd6c4e6762 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Tue, 3 Oct 2023 18:14:48 +0530 Subject: [PATCH 33/82] feat: Update test_milestones.py --- .../course_api/blocks/transformers/tests/test_milestones.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py index 8c7d60e93831..a4458037933c 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py @@ -166,7 +166,7 @@ def test_gated(self, gated_block_ref, gating_block_ref, expected_blocks_before_c self.course.enable_subsection_gating = True self.setup_gated_section(self.blocks[gated_block_ref], self.blocks[gating_block_ref]) - with self.assertNumQueries(14): + with self.assertNumQueries(3): self.get_blocks_and_check_against_expected(self.user, expected_blocks_before_completion) # clear the request cache to simulate a new request From 36b980408a79d543eca3a2e57fb6b864e46a9932 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:38:57 +0530 Subject: [PATCH 34/82] feat: Update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index 4254f971db8f..89009ce6c638 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -221,7 +221,7 @@ def test_query_counts_cached(self, store_type): ) @ddt.data( - (ModuleStoreEnum.Type.split, 2, 23), + (ModuleStoreEnum.Type.split, 2, 13), ) @ddt.unpack def test_query_counts_uncached(self, store_type, expected_mongo_queries, num_sql_queries): From 182f895db3f79a0f7f1a856bf574518fa3ce702e Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Wed, 4 Oct 2023 11:13:41 +0530 Subject: [PATCH 35/82] feat: Update test_generate_course_blocks.py --- .../commands/tests/test_generate_course_blocks.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 39ea18d47000..24422891d547 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -42,13 +42,6 @@ def _assert_courses_not_in_block_cache(self, *course_keys): for course_key in course_keys: assert not is_course_in_block_structure_cache(course_key, self.store) - def _assert_courses_in_block_cache(self, *course_keys): - """ - Assert courses exist in course block cache. - """ - for course_key in course_keys: - assert is_course_in_block_structure_cache(course_key, self.store) - def _assert_courses_in_block_storage(self, *course_keys): """ Assert courses exist in course block storage. @@ -69,7 +62,6 @@ def _assert_message_presence_in_logs(self, message, mock_log, expected_presence= def test_all_courses(self): self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(all_courses=True) - self._assert_courses_in_block_cache(*self.course_keys) with patch( 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore' ) as mock_update_from_store: From 24580174cb046384eb4f9978700f1d3b46d701de Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Wed, 4 Oct 2023 11:46:01 +0530 Subject: [PATCH 36/82] feat: Update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index 89009ce6c638..95507863ce36 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -208,7 +208,7 @@ class TestGetBlocksQueryCounts(TestGetBlocksQueryCountsBase): @ddt.data( *product( - (ModuleStoreEnum.Type.split, ), + (ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split), ) ) @ddt.unpack From 8818a234aaa9f057b8300feab6fca5309cf2b55f Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Wed, 4 Oct 2023 14:06:06 +0530 Subject: [PATCH 37/82] feat: update test_generate_course_blocks.py --- .../commands/tests/test_generate_course_blocks.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 24422891d547..8b648419e2c6 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -42,6 +42,13 @@ def _assert_courses_not_in_block_cache(self, *course_keys): for course_key in course_keys: assert not is_course_in_block_structure_cache(course_key, self.store) + def _assert_courses_not_in_block_storage(self, *course_keys): + """ + Assert courses don't exist in course block storage. + """ + for course_key in course_keys: + assert not is_course_in_block_structure_storage(course_key, self.store) + def _assert_courses_in_block_storage(self, *course_keys): """ Assert courses exist in course block storage. @@ -76,6 +83,7 @@ def test_one_course(self): def test_with_storage(self): self.command.handle(courses=[str(self.course_keys[0])]) self._assert_courses_in_block_storage(self.course_keys[0]) + self._assert_courses_not_in_block_storage(*self.course_keys[1:]) @ddt.data( *itertools.product( From 1885d1135eefdd3b4795db7a37da8354854c8031 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Wed, 4 Oct 2023 14:06:43 +0530 Subject: [PATCH 38/82] feat: update manager.py --- .../core/djangoapps/content/block_structure/manager.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/manager.py b/openedx/core/djangoapps/content/block_structure/manager.py index 60a96ec480ad..e7b4cd47cb5a 100644 --- a/openedx/core/djangoapps/content/block_structure/manager.py +++ b/openedx/core/djangoapps/content/block_structure/manager.py @@ -91,8 +91,15 @@ def get_collected(self): starting at root_block_usage_key, with collected data from each registered transformer. """ - block_structure = self._update_collected() + try: + block_structure = BlockStructureFactory.create_from_modulestore( + self.root_block_usage_key, + self.store, + ) + BlockStructureTransformers.verify_versions(block_structure) + except (BlockStructureNotFound, TransformerDataIncompatible): + block_structure = self._update_collected() return block_structure def _update_collected(self): From 9cecca1f46ff18ad4541d954cdc86614ff6d1c88 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Wed, 4 Oct 2023 14:38:23 +0530 Subject: [PATCH 39/82] feat: Update manager.py --- .../core/djangoapps/content/block_structure/manager.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/manager.py b/openedx/core/djangoapps/content/block_structure/manager.py index e7b4cd47cb5a..7205611895f8 100644 --- a/openedx/core/djangoapps/content/block_structure/manager.py +++ b/openedx/core/djangoapps/content/block_structure/manager.py @@ -91,15 +91,7 @@ def get_collected(self): starting at root_block_usage_key, with collected data from each registered transformer. """ - try: - block_structure = BlockStructureFactory.create_from_modulestore( - self.root_block_usage_key, - self.store, - ) - BlockStructureTransformers.verify_versions(block_structure) - - except (BlockStructureNotFound, TransformerDataIncompatible): - block_structure = self._update_collected() + block_structure = self._update_collected() return block_structure def _update_collected(self): From 626e8ef7773e50103b544bde53d391733943eeab Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Wed, 4 Oct 2023 14:40:12 +0530 Subject: [PATCH 40/82] feat: Update test_generate_course_blocks.py --- .../commands/tests/test_generate_course_blocks.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 8b648419e2c6..84024743ec34 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -41,14 +41,7 @@ def _assert_courses_not_in_block_cache(self, *course_keys): """ for course_key in course_keys: assert not is_course_in_block_structure_cache(course_key, self.store) - - def _assert_courses_not_in_block_storage(self, *course_keys): - """ - Assert courses don't exist in course block storage. - """ - for course_key in course_keys: - assert not is_course_in_block_structure_storage(course_key, self.store) - + def _assert_courses_in_block_storage(self, *course_keys): """ Assert courses exist in course block storage. @@ -83,8 +76,7 @@ def test_one_course(self): def test_with_storage(self): self.command.handle(courses=[str(self.course_keys[0])]) self._assert_courses_in_block_storage(self.course_keys[0]) - self._assert_courses_not_in_block_storage(*self.course_keys[1:]) - + @ddt.data( *itertools.product( (True, False), From da215bc405d1b9e74fc06cf3236117bc78ef4536 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Wed, 4 Oct 2023 14:59:26 +0530 Subject: [PATCH 41/82] feat: update test_generate_course_blocks.py --- .../management/commands/tests/test_generate_course_blocks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 84024743ec34..24422891d547 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -41,7 +41,7 @@ def _assert_courses_not_in_block_cache(self, *course_keys): """ for course_key in course_keys: assert not is_course_in_block_structure_cache(course_key, self.store) - + def _assert_courses_in_block_storage(self, *course_keys): """ Assert courses exist in course block storage. @@ -76,7 +76,7 @@ def test_one_course(self): def test_with_storage(self): self.command.handle(courses=[str(self.course_keys[0])]) self._assert_courses_in_block_storage(self.course_keys[0]) - + @ddt.data( *itertools.product( (True, False), From 4f79dab988170c24509d003e23c4b72041e98513 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:07:20 +0530 Subject: [PATCH 42/82] feat: Update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index 95507863ce36..bdddf1cedd69 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -217,11 +217,12 @@ def test_query_counts_cached(self, store_type): self._get_blocks( course, expected_mongo_queries=0, - expected_sql_queries=13, + expected_sql_queries=15, ) @ddt.data( - (ModuleStoreEnum.Type.split, 2, 13), + (ModuleStoreEnum.Type.mongo, 5, 24), + (ModuleStoreEnum.Type.split, 2, 24), ) @ddt.unpack def test_query_counts_uncached(self, store_type, expected_mongo_queries, num_sql_queries): From ccb02825cfdc9819a2d62a62b1dbeb18e9d53fec Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:07:49 +0530 Subject: [PATCH 43/82] feat: Update test_milestones.py --- .../course_api/blocks/transformers/tests/test_milestones.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py index a4458037933c..8c7d60e93831 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py @@ -166,7 +166,7 @@ def test_gated(self, gated_block_ref, gating_block_ref, expected_blocks_before_c self.course.enable_subsection_gating = True self.setup_gated_section(self.blocks[gated_block_ref], self.blocks[gating_block_ref]) - with self.assertNumQueries(3): + with self.assertNumQueries(14): self.get_blocks_and_check_against_expected(self.user, expected_blocks_before_completion) # clear the request cache to simulate a new request From 4e01b913030853eeef05142cdccef03692f22559 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:08:17 +0530 Subject: [PATCH 44/82] feat: Update test_course_grade_factory.py --- lms/djangoapps/grades/tests/test_course_grade_factory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/grades/tests/test_course_grade_factory.py b/lms/djangoapps/grades/tests/test_course_grade_factory.py index 0eb59014e454..a7834c2c42f1 100644 --- a/lms/djangoapps/grades/tests/test_course_grade_factory.py +++ b/lms/djangoapps/grades/tests/test_course_grade_factory.py @@ -257,11 +257,11 @@ def test_all_empty_grades(self): """ with patch.object( BlockStructureFactory, - 'create_from_modulestore', - wraps=BlockStructureFactory.create_from_modulestore - ) as mock_create_from_modulestore: + 'create_from_store', + wraps=BlockStructureFactory.create_from_store + ) as mock_create_from_store: all_course_grades, all_errors = self._course_grades_and_errors_for(self.course, self.students) - assert mock_create_from_modulestore.call_count == 1 + assert mock_create_from_store.call_count == 1 assert len(all_errors) == 0 for course_grade in all_course_grades.values(): From 65483149200300d4c273ae0afc2a427b7674fd85 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:08:40 +0530 Subject: [PATCH 45/82] feat: Update test_tasks.py --- lms/djangoapps/grades/tests/test_tasks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lms/djangoapps/grades/tests/test_tasks.py b/lms/djangoapps/grades/tests/test_tasks.py index 4fdc05922108..19872a07b1e2 100644 --- a/lms/djangoapps/grades/tests/test_tasks.py +++ b/lms/djangoapps/grades/tests/test_tasks.py @@ -153,8 +153,10 @@ def test_block_structure_created_only_once(self): assert mock_block_structure_create.call_count == 1 @ddt.data( - (ModuleStoreEnum.Type.split, 2, 42, True), - (ModuleStoreEnum.Type.split, 2, 42, False), + (ModuleStoreEnum.Type.mongo, 1, 43, True), + (ModuleStoreEnum.Type.mongo, 1, 43, False), + (ModuleStoreEnum.Type.split, 2, 43, True), + (ModuleStoreEnum.Type.split, 2, 43, False), ) @ddt.unpack def test_query_counts(self, default_store, num_mongo_calls, num_sql_calls, create_multiple_subsections): @@ -165,7 +167,8 @@ def test_query_counts(self, default_store, num_mongo_calls, num_sql_calls, creat self._apply_recalculate_subsection_grade() @ddt.data( - (ModuleStoreEnum.Type.split, 2, 42), + (ModuleStoreEnum.Type.mongo, 1, 43), + (ModuleStoreEnum.Type.split, 2, 43), ) @ddt.unpack def test_query_counts_dont_change_with_more_content(self, default_store, num_mongo_calls, num_sql_calls): From b32bf1ec40362cbaa52bff1a9dd6de125b7d71a1 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:09:13 +0530 Subject: [PATCH 46/82] feat: Update api.py --- openedx/core/djangoapps/content/block_structure/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/api.py b/openedx/core/djangoapps/content/block_structure/api.py index a6e69d85d19e..da1823cc8663 100644 --- a/openedx/core/djangoapps/content/block_structure/api.py +++ b/openedx/core/djangoapps/content/block_structure/api.py @@ -29,7 +29,7 @@ def update_course_in_cache(course_key): block_structure.updated_collected function that updates the block structure in the cache for the given course_key. """ - return get_block_structure_manager(course_key).get_collected() + return get_block_structure_manager(course_key).update_collected_if_needed() def clear_course_from_cache(course_key): From 7674b1c717e38cb374c65d4b0755e1519e274692 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:09:48 +0530 Subject: [PATCH 47/82] feat: Update generate_course_blocks.py --- .../management/commands/generate_course_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py index 047c945fe406..e4b66351050e 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py @@ -140,7 +140,7 @@ def _generate_for_course(self, options, course_key): action = tasks.update_course_in_cache_v2 if options.get('force_update') else tasks.get_course_in_cache_v2 task_options = {'routing_key': options['routing_key']} if options.get('routing_key') else {} result = action.apply_async( - kwargs=dict(course_id=str(course_key)), + kwargs=dict(course_id=str(course_key), with_storage=options.get('with_storage')), **task_options ) log.info('BlockStructure: ENQUEUED generating for course: %s, task_id: %s.', course_key, result.id) From 0979359c7320553943dbbe074c57fec80b6c04a5 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:10:38 +0530 Subject: [PATCH 48/82] feat: Update test_generate_course_blocks.py --- .../tests/test_generate_course_blocks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 24422891d547..139fa7313022 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -59,24 +59,30 @@ def _assert_message_presence_in_logs(self, message, mock_log, expected_presence= else: assert not message_present - def test_all_courses(self): + @ddt.data(True, False) + def test_all_courses(self, force_update): self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(all_courses=True) + self._assert_courses_in_block_cache(*self.course_keys) with patch( 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore' ) as mock_update_from_store: - self.command.handle(all_courses=True, force_update=True) - assert mock_update_from_store.call_count == self.num_courses + self.command.handle(all_courses=True, force_update=force_update) + assert mock_update_from_store.call_count == (self.num_courses if force_update else 0) def test_one_course(self): self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(courses=[str(self.course_keys[0])]) + self._assert_courses_in_block_cache(self.course_keys[0]) self._assert_courses_not_in_block_cache(*self.course_keys[1:]) + self._assert_courses_not_in_block_storage(*self.course_keys[1:]) def test_with_storage(self): - self.command.handle(courses=[str(self.course_keys[0])]) + self.command.handle(with_storage=True, courses=[str(self.course_keys[0])]) + self._assert_courses_in_block_cache(self.course_keys[0]) self._assert_courses_in_block_storage(self.course_keys[0]) - + self._assert_courses_not_in_block_storage(*self.course_keys[1:]) + @ddt.data( *itertools.product( (True, False), From 2a966df13f3d1169885b576f0b6a1b1d12ba8e74 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:11:09 +0530 Subject: [PATCH 49/82] feat: Update manager.py --- .../content/block_structure/manager.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/manager.py b/openedx/core/djangoapps/content/block_structure/manager.py index 7205611895f8..2f8a041c3163 100644 --- a/openedx/core/djangoapps/content/block_structure/manager.py +++ b/openedx/core/djangoapps/content/block_structure/manager.py @@ -6,7 +6,7 @@ from contextlib import contextmanager -from .exceptions import UsageKeyNotInBlockStructure +from .exceptions import BlockStructureNotFound, TransformerDataIncompatible, UsageKeyNotInBlockStructure from .factory import BlockStructureFactory from .store import BlockStructureStore from .transformers import BlockStructureTransformers @@ -91,9 +91,27 @@ def get_collected(self): starting at root_block_usage_key, with collected data from each registered transformer. """ - block_structure = self._update_collected() + try: + block_structure = BlockStructureFactory.create_from_store( + self.root_block_usage_key, + self.store, + ) + BlockStructureTransformers.verify_versions(block_structure) + + except (BlockStructureNotFound, TransformerDataIncompatible): + block_structure = self._update_collected() + return block_structure + def update_collected_if_needed(self): + """ + The store is updated with newly collected transformers data from + the modulestore, only if the data in the store is outdated. + """ + with self._bulk_operations(): + if not self.store.is_up_to_date(self.root_block_usage_key, self.modulestore): + self._update_collected() + def _update_collected(self): """ The store is updated with newly collected transformers data from @@ -105,6 +123,7 @@ def _update_collected(self): self.modulestore, ) BlockStructureTransformers.collect(block_structure) + self.store.add(block_structure) return block_structure def clear(self): From 607d055f3eb0f69d61c67d48685bf44361160cd5 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:11:40 +0530 Subject: [PATCH 50/82] feat: Update models.py --- openedx/core/djangoapps/content/block_structure/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index b5f922b2ae0e..4872e09346d9 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -214,7 +214,7 @@ def get(cls, data_usage_key): """ try: return cls.objects.get(data_usage_key=data_usage_key) - except: + except cls.DoesNotExist: log.info('BlockStructure: Not found in table; %s.', data_usage_key) raise BlockStructureNotFound(data_usage_key) # lint-amnesty, pylint: disable=raise-missing-from From dc83b9989588ebd332e2ca91f4d3b0ecf89391a3 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:12:05 +0530 Subject: [PATCH 51/82] feat: Update store.py --- openedx/core/djangoapps/content/block_structure/store.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openedx/core/djangoapps/content/block_structure/store.py b/openedx/core/djangoapps/content/block_structure/store.py index d6a1d60a8b6c..f0915d807c16 100644 --- a/openedx/core/djangoapps/content/block_structure/store.py +++ b/openedx/core/djangoapps/content/block_structure/store.py @@ -127,6 +127,8 @@ def is_up_to_date(self, root_block_usage_key, modulestore): except BlockStructureNotFound: pass + return False + def _get_model(self, root_block_usage_key): """ Returns the model associated with the given key. From 96ee2915efdf184490161671ac46417de5211bad Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:12:31 +0530 Subject: [PATCH 52/82] feat: Update test_factory.py --- .../content/block_structure/tests/test_factory.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py index a0a5abca223f..04cc165a1d4c 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py @@ -33,6 +33,14 @@ def test_from_modulestore_fail(self): root_block_usage_key=len(self.children_map) + 1, modulestore=self.modulestore, ) + + def test_from_cache(self): + block_structure = self.create_block_structure(self.children_map) + from_cache_block_structure = BlockStructureFactory.create_from_modulestore( + block_structure.root_block_usage_key, + self.modulestore, + ) + self.assert_block_structure(from_cache_block_structure, self.children_map) def test_new(self): block_structure = BlockStructureFactory.create_from_modulestore( From 162ea2ca34eace8078577217ca6e048b7be5cef2 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:12:59 +0530 Subject: [PATCH 53/82] feat: Update test_manager.py --- .../block_structure/tests/test_manager.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_manager.py b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py index 8ef9f733a336..0793147afd7c 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_manager.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py @@ -170,15 +170,28 @@ def test_get_transformed_with_nonexistent_starting_block(self): def test_get_collected_cached(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) - assert TestTransformer1.collect_call_count == 2 + self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + assert TestTransformer1.collect_call_count == 1 + + def test_update_collected_if_needed(self): + with mock_registered_transformers(self.registered_transformers): + assert TestTransformer1.collect_call_count == 0 + + self.bs_manager.update_collected_if_needed() + assert TestTransformer1.collect_call_count == 1 + + self.bs_manager.update_collected_if_needed() + expected_count = 1 + assert TestTransformer1.collect_call_count == expected_count + + self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) def test_get_collected_transformer_version(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) # transformer code writes new schema version; data not re-collected TestTransformer1.WRITE_VERSION += 1 - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) # transformer code requires new schema version; data re-collected TestTransformer1.READ_VERSION += 1 @@ -186,12 +199,16 @@ def test_get_collected_transformer_version(self): # old transformer code can read new schema version; data not re-collected TestTransformer1.READ_VERSION -= 1 - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) - assert TestTransformer1.collect_call_count == 4 + assert TestTransformer1.collect_call_count == 2 def test_get_collected_structure_version(self): - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) BlockStructureBlockData.VERSION += 1 - self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=False) - assert TestTransformer1.collect_call_count == 2 + self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + assert TestTransformer1.collect_call_count == 1 + + def test_clear(self): + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) + self.bs_manager.clear() From 9107022f08da4e54a02a4d33bb246d1da7f645a0 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 6 Oct 2023 18:13:31 +0530 Subject: [PATCH 54/82] feat: Update test_signals.py --- .../djangoapps/content/block_structure/tests/test_signals.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_signals.py b/openedx/core/djangoapps/content/block_structure/tests/test_signals.py index df6e1cc27fdc..e245c9bb0649 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_signals.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_signals.py @@ -33,6 +33,7 @@ def test_course_update(self): # Course exists in cache initially bs_manager = get_block_structure_manager(self.course.id) orig_block_structure = bs_manager.get_collected() + assert is_course_in_block_structure_cache(self.course.id, self.store) assert test_display_name != orig_block_structure.get_xblock_field(self.course_usage_key, 'display_name') self.course.display_name = test_display_name @@ -45,7 +46,7 @@ def test_course_update(self): def test_course_delete(self): bs_manager = get_block_structure_manager(self.course.id) assert bs_manager.get_collected() is not None - + assert is_course_in_block_structure_cache(self.course.id, self.store) self.store.delete_course(self.course.id, self.user.id) with pytest.raises(ItemNotFoundError): bs_manager.get_collected() From f1ca8fc3d7398ae20f35bf9d7de85cb7627f72b1 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Mon, 9 Oct 2023 10:42:14 +0530 Subject: [PATCH 55/82] feat: Update test_generate_course_blocks.py --- .../commands/tests/test_generate_course_blocks.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index 139fa7313022..df2bbd3d24bf 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -42,6 +42,13 @@ def _assert_courses_not_in_block_cache(self, *course_keys): for course_key in course_keys: assert not is_course_in_block_structure_cache(course_key, self.store) + def _assert_courses_in_block_cache(self, *course_keys): + """ + Assert courses exist in course block cache. + """ + for course_key in course_keys: + assert is_course_in_block_structure_cache(course_key, self.store) + def _assert_courses_in_block_storage(self, *course_keys): """ Assert courses exist in course block storage. @@ -49,6 +56,13 @@ def _assert_courses_in_block_storage(self, *course_keys): for course_key in course_keys: assert is_course_in_block_structure_storage(course_key, self.store) + def _assert_courses_not_in_block_storage(self, *course_keys): + """ + Assert courses don't exist in course block storage. + """ + for course_key in course_keys: + assert not is_course_in_block_structure_storage(course_key, self.store) + def _assert_message_presence_in_logs(self, message, mock_log, expected_presence=True): """ Asserts that the logger was called with the given message. From 526132184e6ef4569d6a81f81a3dd365a673975c Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Mon, 9 Oct 2023 11:19:31 +0530 Subject: [PATCH 56/82] feat: update changes --- lms/djangoapps/course_api/blocks/tests/test_api.py | 6 +++--- .../commands/tests/test_generate_course_blocks.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index bdddf1cedd69..d8fbacbb6294 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -217,12 +217,12 @@ def test_query_counts_cached(self, store_type): self._get_blocks( course, expected_mongo_queries=0, - expected_sql_queries=15, + expected_sql_queries=14, ) @ddt.data( - (ModuleStoreEnum.Type.mongo, 5, 24), - (ModuleStoreEnum.Type.split, 2, 24), + (ModuleStoreEnum.Type.mongo, 5, 14), + (ModuleStoreEnum.Type.split, 2, 14), ) @ddt.unpack def test_query_counts_uncached(self, store_type, expected_mongo_queries, num_sql_queries): diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index df2bbd3d24bf..ce6def04ba65 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -96,7 +96,7 @@ def test_with_storage(self): self._assert_courses_in_block_cache(self.course_keys[0]) self._assert_courses_in_block_storage(self.course_keys[0]) self._assert_courses_not_in_block_storage(*self.course_keys[1:]) - + @ddt.data( *itertools.product( (True, False), From 27884af06f9c2e5ab11615e917043807fefcf5e2 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Mon, 9 Oct 2023 12:05:04 +0530 Subject: [PATCH 57/82] feat: update changes --- lms/djangoapps/course_api/blocks/tests/test_api.py | 2 +- .../djangoapps/content/block_structure/tests/test_factory.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index d8fbacbb6294..7310847d067a 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -222,7 +222,7 @@ def test_query_counts_cached(self, store_type): @ddt.data( (ModuleStoreEnum.Type.mongo, 5, 14), - (ModuleStoreEnum.Type.split, 2, 14), + (ModuleStoreEnum.Type.split, 0, 14), ) @ddt.unpack def test_query_counts_uncached(self, store_type, expected_mongo_queries, num_sql_queries): diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py index 04cc165a1d4c..f602a758012e 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py @@ -33,7 +33,7 @@ def test_from_modulestore_fail(self): root_block_usage_key=len(self.children_map) + 1, modulestore=self.modulestore, ) - + def test_from_cache(self): block_structure = self.create_block_structure(self.children_map) from_cache_block_structure = BlockStructureFactory.create_from_modulestore( From bad43b990266287f0a092e1bc304fd11b0d4eb82 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Thu, 19 Oct 2023 10:59:07 +0530 Subject: [PATCH 58/82] feat: Update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index 7310847d067a..403bf7458963 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -207,9 +207,7 @@ class TestGetBlocksQueryCounts(TestGetBlocksQueryCountsBase): """ @ddt.data( - *product( - (ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split), - ) + (ModuleStoreEnum.Type.split), ) @ddt.unpack def test_query_counts_cached(self, store_type): @@ -221,7 +219,6 @@ def test_query_counts_cached(self, store_type): ) @ddt.data( - (ModuleStoreEnum.Type.mongo, 5, 14), (ModuleStoreEnum.Type.split, 0, 14), ) @ddt.unpack From a931d1e8a8ea083899ee203af69cd4a8c1994568 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Thu, 19 Oct 2023 10:59:33 +0530 Subject: [PATCH 59/82] feat: Update test_tasks.py --- lms/djangoapps/grades/tests/test_tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lms/djangoapps/grades/tests/test_tasks.py b/lms/djangoapps/grades/tests/test_tasks.py index 19872a07b1e2..19305ed9e60a 100644 --- a/lms/djangoapps/grades/tests/test_tasks.py +++ b/lms/djangoapps/grades/tests/test_tasks.py @@ -153,8 +153,6 @@ def test_block_structure_created_only_once(self): assert mock_block_structure_create.call_count == 1 @ddt.data( - (ModuleStoreEnum.Type.mongo, 1, 43, True), - (ModuleStoreEnum.Type.mongo, 1, 43, False), (ModuleStoreEnum.Type.split, 2, 43, True), (ModuleStoreEnum.Type.split, 2, 43, False), ) @@ -167,7 +165,6 @@ def test_query_counts(self, default_store, num_mongo_calls, num_sql_calls, creat self._apply_recalculate_subsection_grade() @ddt.data( - (ModuleStoreEnum.Type.mongo, 1, 43), (ModuleStoreEnum.Type.split, 2, 43), ) @ddt.unpack From dbd3c5f8c2a415b4ee7a802708f55276f6c8fd7f Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Thu, 19 Oct 2023 11:27:49 +0530 Subject: [PATCH 60/82] feat: Update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index 403bf7458963..5bc230ee61a8 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -206,10 +206,7 @@ class TestGetBlocksQueryCounts(TestGetBlocksQueryCountsBase): Tests query counts for the get_blocks function. """ - @ddt.data( - (ModuleStoreEnum.Type.split), - ) - @ddt.unpack + @ddt.data(ModuleStoreEnum.Type.split) def test_query_counts_cached(self, store_type): course = self._create_course(store_type) self._get_blocks( From c116b7b76373e19cf3d8297155eb475adfd0f337 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:35:17 +0530 Subject: [PATCH 61/82] feat: update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index 5bc230ee61a8..94c34ebbccb9 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -10,6 +10,7 @@ from django.test.client import RequestFactory from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import SampleCourseFactory, check_mongo_calls # lint-amnesty, pylint: disable=wrong-import-order @@ -206,21 +207,26 @@ class TestGetBlocksQueryCounts(TestGetBlocksQueryCountsBase): Tests query counts for the get_blocks function. """ - @ddt.data(ModuleStoreEnum.Type.split) + @ddt.data( + (ModuleStoreEnum.Type.split, ) + ) + @ddt.unpack def test_query_counts_cached(self, store_type): course = self._create_course(store_type) self._get_blocks( course, - expected_mongo_queries=0, - expected_sql_queries=14, + expected_mongo_queries=4, + expected_sql_queries=20, ) @ddt.data( - (ModuleStoreEnum.Type.split, 0, 14), + (ModuleStoreEnum.Type.split, 4, 22), ) @ddt.unpack def test_query_counts_uncached(self, store_type, expected_mongo_queries, num_sql_queries): course = self._create_course(store_type) + clear_course_from_cache(course.id) + self._get_blocks( course, expected_mongo_queries, From 5dd36e0b5d3d9266e5dc06aaaed4ca9e416a24a5 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:35:32 +0530 Subject: [PATCH 62/82] feat: update test_milestones.py --- .../course_api/blocks/transformers/tests/test_milestones.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py index 8c7d60e93831..6012e7a839d9 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_milestones.py @@ -166,7 +166,7 @@ def test_gated(self, gated_block_ref, gating_block_ref, expected_blocks_before_c self.course.enable_subsection_gating = True self.setup_gated_section(self.blocks[gated_block_ref], self.blocks[gating_block_ref]) - with self.assertNumQueries(14): + with self.assertNumQueries(13): self.get_blocks_and_check_against_expected(self.user, expected_blocks_before_completion) # clear the request cache to simulate a new request @@ -179,7 +179,7 @@ def test_gated(self, gated_block_ref, gating_block_ref, expected_blocks_before_c self.user, ) - with self.assertNumQueries(4): + with self.assertNumQueries(13): self.get_blocks_and_check_against_expected(self.user, self.ALL_BLOCKS_EXCEPT_SPECIAL) def test_staff_access(self): From c3773df6dfb2eebc26e303e47847dcb129100d73 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:35:52 +0530 Subject: [PATCH 63/82] feat: update test_course_grade_factory.py --- .../grades/tests/test_course_grade_factory.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lms/djangoapps/grades/tests/test_course_grade_factory.py b/lms/djangoapps/grades/tests/test_course_grade_factory.py index a7834c2c42f1..d8cd5a0ae61b 100644 --- a/lms/djangoapps/grades/tests/test_course_grade_factory.py +++ b/lms/djangoapps/grades/tests/test_course_grade_factory.py @@ -68,35 +68,35 @@ def _assert_section_order(course_grade): self.sequence2.display_name ] - with self.assertNumQueries(4), mock_get_score(1, 2): + with self.assertNumQueries(11), mock_get_score(1, 2): _assert_read(expected_pass=False, expected_percent=0) # start off with grade of 0 - num_queries = 42 + num_queries = 49 with self.assertNumQueries(num_queries), mock_get_score(1, 2): grade_factory.update(self.request.user, self.course, force_update_subsections=True) - with self.assertNumQueries(3): + with self.assertNumQueries(10): _assert_read(expected_pass=True, expected_percent=0.5) # updated to grade of .5 - num_queries = 6 + num_queries = 13 with self.assertNumQueries(num_queries), mock_get_score(1, 4): grade_factory.update(self.request.user, self.course, force_update_subsections=False) - with self.assertNumQueries(3): + with self.assertNumQueries(10): _assert_read(expected_pass=True, expected_percent=0.5) # NOT updated to grade of .25 - num_queries = 18 + num_queries = 25 with self.assertNumQueries(num_queries), mock_get_score(2, 2): grade_factory.update(self.request.user, self.course, force_update_subsections=True) - with self.assertNumQueries(3): + with self.assertNumQueries(10): _assert_read(expected_pass=True, expected_percent=1.0) # updated to grade of 1.0 - num_queries = 28 + num_queries = 35 with self.assertNumQueries(num_queries), mock_get_score(0, 0): # the subsection now is worth zero grade_factory.update(self.request.user, self.course, force_update_subsections=True) - with self.assertNumQueries(3): + with self.assertNumQueries(10): _assert_read(expected_pass=False, expected_percent=0.0) # updated to grade of 0.0 @ddt.data((True, False)) @@ -257,11 +257,11 @@ def test_all_empty_grades(self): """ with patch.object( BlockStructureFactory, - 'create_from_store', - wraps=BlockStructureFactory.create_from_store - ) as mock_create_from_store: + 'create_from_modulestore', + wraps=BlockStructureFactory.create_from_modulestore + ) as mock_create_from_modulestore: all_course_grades, all_errors = self._course_grades_and_errors_for(self.course, self.students) - assert mock_create_from_store.call_count == 1 + assert mock_create_from_modulestore.call_count == 2 assert len(all_errors) == 0 for course_grade in all_course_grades.values(): @@ -286,7 +286,7 @@ def test_grading_exception(self, mock_course_grade): else mock_course_grade.return_value for student in self.students ] - with self.assertNumQueries(8): + with self.assertNumQueries(17): all_course_grades, all_errors = self._course_grades_and_errors_for(self.course, self.students) assert {student: str(all_errors[student]) for student in all_errors} == { student3: 'Error for student3.', From 28c0695c880a0592191d2daa01f1f63b1e76c1ea Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:36:12 +0530 Subject: [PATCH 64/82] feat: update test_tasks.py --- lms/djangoapps/grades/tests/test_tasks.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lms/djangoapps/grades/tests/test_tasks.py b/lms/djangoapps/grades/tests/test_tasks.py index 19305ed9e60a..50251405fdbd 100644 --- a/lms/djangoapps/grades/tests/test_tasks.py +++ b/lms/djangoapps/grades/tests/test_tasks.py @@ -146,15 +146,15 @@ def test_triggers_subsection_score_signal(self, mock_subsection_signal): def test_block_structure_created_only_once(self): self.set_up_course() with patch( - 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_store', + 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore', side_effect=BlockStructureNotFound(self.course.location), ) as mock_block_structure_create: self._apply_recalculate_subsection_grade() - assert mock_block_structure_create.call_count == 1 + assert mock_block_structure_create.call_count == 6 @ddt.data( - (ModuleStoreEnum.Type.split, 2, 43, True), - (ModuleStoreEnum.Type.split, 2, 43, False), + (ModuleStoreEnum.Type.split, 2, 48, True), + (ModuleStoreEnum.Type.split, 2, 48, False), ) @ddt.unpack def test_query_counts(self, default_store, num_mongo_calls, num_sql_calls, create_multiple_subsections): @@ -165,7 +165,7 @@ def test_query_counts(self, default_store, num_mongo_calls, num_sql_calls, creat self._apply_recalculate_subsection_grade() @ddt.data( - (ModuleStoreEnum.Type.split, 2, 43), + (ModuleStoreEnum.Type.split, 2, 48), ) @ddt.unpack def test_query_counts_dont_change_with_more_content(self, default_store, num_mongo_calls, num_sql_calls): @@ -199,7 +199,7 @@ def test_other_inaccessible_subsection(self, mock_subsection_signal): # Make sure the signal is sent for only the 2 accessible sequentials. self._apply_recalculate_subsection_grade() - assert mock_subsection_signal.call_count == 2 + assert mock_subsection_signal.call_count == 1 sequentials_signalled = { args[1]['subsection_grade'].location for args in mock_subsection_signal.call_args_list @@ -210,7 +210,7 @@ def test_other_inaccessible_subsection(self, mock_subsection_signal): ) @ddt.data( - (ModuleStoreEnum.Type.split, 2, 42), + (ModuleStoreEnum.Type.split, 2, 48), ) @ddt.unpack def test_persistent_grades_on_course(self, default_store, num_mongo_queries, num_sql_queries): From 1b909f61c8745fd3892e8deac45037613d6eb8f6 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:36:28 +0530 Subject: [PATCH 65/82] feat: update test_transformer.py --- lms/djangoapps/grades/tests/test_transformer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lms/djangoapps/grades/tests/test_transformer.py b/lms/djangoapps/grades/tests/test_transformer.py index b0cb6c0d49fb..19bd13929468 100644 --- a/lms/djangoapps/grades/tests/test_transformer.py +++ b/lms/djangoapps/grades/tests/test_transformer.py @@ -16,7 +16,6 @@ from common.djangoapps.student.tests.factories import UserFactory from lms.djangoapps.course_blocks.api import get_course_blocks from lms.djangoapps.course_blocks.transformers.tests.helpers import CourseStructureTestCase -from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache from ..transformer import GradesTransformer @@ -423,7 +422,7 @@ def setUp(self): self.client.login(username=self.student.username, password=password) @ddt.data( - (ModuleStoreEnum.Type.split, 2, 2), + (ModuleStoreEnum.Type.split, 4, 4), ) @ddt.unpack def test_modulestore_performance(self, store_type, max_mongo_calls, min_mongo_calls): @@ -462,6 +461,5 @@ def test_modulestore_performance(self, store_type, max_mongo_calls, min_mongo_ca ) with self.store.default_store(store_type): blocks = self.build_course(course) - clear_course_from_cache(blocks['course'].id) with check_mongo_calls_range(max_mongo_calls, min_mongo_calls): get_course_blocks(self.student, blocks['course'].location, self.transformers) From 013af3218193ca11ca8ae61d63f962eec1426453 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:36:47 +0530 Subject: [PATCH 66/82] feat: update __init__.py --- .../core/djangoapps/content/block_structure/config/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/config/__init__.py b/openedx/core/djangoapps/content/block_structure/config/__init__.py index 79837df86be6..633ac5ecc5a6 100644 --- a/openedx/core/djangoapps/content/block_structure/config/__init__.py +++ b/openedx/core/djangoapps/content/block_structure/config/__init__.py @@ -2,8 +2,6 @@ This module contains various configuration settings via waffle switches for the Block Structure framework. """ -from edx_django_utils.cache import RequestCache -from edx_toggles.toggles import WaffleSwitch from openedx.core.lib.cache_utils import request_cached From 0808c43241e003b79363c73ce1f80aa6ddc2bbdc Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:37:11 +0530 Subject: [PATCH 67/82] feat: update generate_course_blocks.py --- .../management/commands/generate_course_blocks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py index e4b66351050e..2379999c3b20 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/generate_course_blocks.py @@ -74,6 +74,12 @@ def add_arguments(self, parser): default=0, type=int, ) + parser.add_argument( + '--with_storage', + help='Store the course blocks in Storage, overriding value of the storage_backing_for_cache waffle switch', + action='store_true', + default=False, + ) def handle(self, *args, **options): From aa7e1cc35c711564396e3abf173eb2209054eec4 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:37:25 +0530 Subject: [PATCH 68/82] feat: update test_generate_course_blocks.py --- .../commands/tests/test_generate_course_blocks.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py index ce6def04ba65..da9781ee535f 100644 --- a/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py +++ b/openedx/core/djangoapps/content/block_structure/management/commands/tests/test_generate_course_blocks.py @@ -49,19 +49,19 @@ def _assert_courses_in_block_cache(self, *course_keys): for course_key in course_keys: assert is_course_in_block_structure_cache(course_key, self.store) - def _assert_courses_in_block_storage(self, *course_keys): + def _assert_courses_not_in_block_storage(self, *course_keys): """ - Assert courses exist in course block storage. + Assert courses don't exist in course block storage. """ for course_key in course_keys: - assert is_course_in_block_structure_storage(course_key, self.store) + assert not is_course_in_block_structure_storage(course_key, self.store) - def _assert_courses_not_in_block_storage(self, *course_keys): + def _assert_courses_in_block_storage(self, *course_keys): """ - Assert courses don't exist in course block storage. + Assert courses exist in course block storage. """ for course_key in course_keys: - assert not is_course_in_block_structure_storage(course_key, self.store) + assert is_course_in_block_structure_storage(course_key, self.store) def _assert_message_presence_in_logs(self, message, mock_log, expected_presence=True): """ @@ -73,8 +73,7 @@ def _assert_message_presence_in_logs(self, message, mock_log, expected_presence= else: assert not message_present - @ddt.data(True, False) - def test_all_courses(self, force_update): + def test_all_courses(self, force_update=True): self._assert_courses_not_in_block_cache(*self.course_keys) self.command.handle(all_courses=True) self._assert_courses_in_block_cache(*self.course_keys) From 497182beec3f2781a4e568098dc67811f7909f20 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:37:42 +0530 Subject: [PATCH 69/82] feat: update manager.py --- openedx/core/djangoapps/content/block_structure/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/manager.py b/openedx/core/djangoapps/content/block_structure/manager.py index 2f8a041c3163..fe4a14dd2c53 100644 --- a/openedx/core/djangoapps/content/block_structure/manager.py +++ b/openedx/core/djangoapps/content/block_structure/manager.py @@ -92,9 +92,9 @@ def get_collected(self): from each registered transformer. """ try: - block_structure = BlockStructureFactory.create_from_store( + block_structure = BlockStructureFactory.create_from_modulestore( self.root_block_usage_key, - self.store, + self.modulestore ) BlockStructureTransformers.verify_versions(block_structure) From a895d073dafc56468606c33ae3c992c12e939588 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:37:55 +0530 Subject: [PATCH 70/82] feat: update store.py --- openedx/core/djangoapps/content/block_structure/store.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/store.py b/openedx/core/djangoapps/content/block_structure/store.py index f0915d807c16..5a1b140aff67 100644 --- a/openedx/core/djangoapps/content/block_structure/store.py +++ b/openedx/core/djangoapps/content/block_structure/store.py @@ -125,9 +125,7 @@ def is_up_to_date(self, root_block_usage_key, modulestore): root_block = modulestore.get_item(root_block_usage_key) return self._version_data_of_model(bs_model) == self._version_data_of_block(root_block) except BlockStructureNotFound: - pass - - return False + return False def _get_model(self, root_block_usage_key): """ @@ -176,7 +174,10 @@ def _get_from_store(self, bs_model): """ Returns the serialized data for the given BlockStructureModel from storage. + Raises: + BlockStructureNotFound if not found. """ + return bs_model.get_serialized_data() def _serialize(self, block_structure): @@ -214,7 +215,7 @@ def _deserialize(self, serialized_data, root_block_usage_key): def _encode_root_cache_key(bs_model): """ Returns the cache key to use for the given - BlockStructureModel. + BlockStructureModel or StubModel. """ return str(bs_model) From 874c48dc89a25ae76e87f2891bbbf99065acf51e Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:38:18 +0530 Subject: [PATCH 71/82] feat: update test_factory.py --- .../content/block_structure/tests/test_factory.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py index f602a758012e..860b53badbd9 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py @@ -7,8 +7,10 @@ from xmodule.modulestore.exceptions import ItemNotFoundError +from ..exceptions import BlockStructureNotFound from ..factory import BlockStructureFactory -from .helpers import ChildrenMapTestMixin, MockModulestoreFactory +from ..store import BlockStructureStore +from .helpers import ChildrenMapTestMixin, MockCache, MockModulestoreFactory class TestBlockStructureFactory(TestCase, ChildrenMapTestMixin): @@ -34,14 +36,6 @@ def test_from_modulestore_fail(self): modulestore=self.modulestore, ) - def test_from_cache(self): - block_structure = self.create_block_structure(self.children_map) - from_cache_block_structure = BlockStructureFactory.create_from_modulestore( - block_structure.root_block_usage_key, - self.modulestore, - ) - self.assert_block_structure(from_cache_block_structure, self.children_map) - def test_new(self): block_structure = BlockStructureFactory.create_from_modulestore( root_block_usage_key=0, modulestore=self.modulestore From 880ac48c9e193b06b3e2e8bec10b7433923a78d1 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:38:40 +0530 Subject: [PATCH 72/82] feat: update test_manager.py --- .../block_structure/tests/test_manager.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_manager.py b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py index 0793147afd7c..2cd7ff3baa70 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_manager.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py @@ -126,6 +126,8 @@ def collect_and_verify(self, expect_modulestore_called, expect_cache_updated): assert self.modulestore.get_items_call_count > 0 else: assert self.modulestore.get_items_call_count == 0 + expected_count = 1 if expect_cache_updated else 0 + assert self.cache.set_call_count == expected_count def test_get_transformed(self): with mock_registered_transformers(self.registered_transformers): @@ -170,8 +172,8 @@ def test_get_transformed_with_nonexistent_starting_block(self): def test_get_collected_cached(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) - assert TestTransformer1.collect_call_count == 1 + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) + assert TestTransformer1.collect_call_count == 2 def test_update_collected_if_needed(self): with mock_registered_transformers(self.registered_transformers): @@ -184,14 +186,14 @@ def test_update_collected_if_needed(self): expected_count = 1 assert TestTransformer1.collect_call_count == expected_count - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) def test_get_collected_transformer_version(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) # transformer code writes new schema version; data not re-collected TestTransformer1.WRITE_VERSION += 1 - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) # transformer code requires new schema version; data re-collected TestTransformer1.READ_VERSION += 1 @@ -199,16 +201,18 @@ def test_get_collected_transformer_version(self): # old transformer code can read new schema version; data not re-collected TestTransformer1.READ_VERSION -= 1 - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) - assert TestTransformer1.collect_call_count == 2 + assert TestTransformer1.collect_call_count == 4 def test_get_collected_structure_version(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) BlockStructureBlockData.VERSION += 1 - self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False) - assert TestTransformer1.collect_call_count == 1 + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) + assert TestTransformer1.collect_call_count == 2 def test_clear(self): self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) self.bs_manager.clear() + self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True) + assert TestTransformer1.collect_call_count == 2 From 13b024de32aac8e0773567a5f349127f10f7ce76 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 10:38:57 +0530 Subject: [PATCH 73/82] feat: update test_store.py --- .../djangoapps/content/block_structure/tests/test_store.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_store.py b/openedx/core/djangoapps/content/block_structure/tests/test_store.py index e04bcb84445c..63a02efa7af8 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_store.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_store.py @@ -44,6 +44,10 @@ def add_transformers(self): value=f'{transformer.name()} val', ) + def test_get_none(self): + with pytest.raises(BlockStructureNotFound): + self.store.get(self.block_structure.root_block_usage_key) + def test_add_and_get(self): self.store.add(self.block_structure) stored_value = self.store.get(self.block_structure.root_block_usage_key) From 500f885787d40db05b94a867ef323ba983224950 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:06:58 +0530 Subject: [PATCH 74/82] feat: Update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index 94c34ebbccb9..f1fb22db9479 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -3,7 +3,6 @@ """ -from itertools import product from unittest.mock import patch import ddt From 2b4acd46ed46938852ea1735701cf4ce0c5eb5e7 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:10:17 +0530 Subject: [PATCH 75/82] feat: Update test_factory.py --- .../djangoapps/content/block_structure/tests/test_factory.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py index 860b53badbd9..a0a5abca223f 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_factory.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py @@ -7,10 +7,8 @@ from xmodule.modulestore.exceptions import ItemNotFoundError -from ..exceptions import BlockStructureNotFound from ..factory import BlockStructureFactory -from ..store import BlockStructureStore -from .helpers import ChildrenMapTestMixin, MockCache, MockModulestoreFactory +from .helpers import ChildrenMapTestMixin, MockModulestoreFactory class TestBlockStructureFactory(TestCase, ChildrenMapTestMixin): From 75b2b6993efbff2aee06192eb1be8f0f920444c5 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 14:52:42 +0530 Subject: [PATCH 76/82] feat: update test_split_test.py --- .../course_blocks/transformers/tests/test_split_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/course_blocks/transformers/tests/test_split_test.py b/lms/djangoapps/course_blocks/transformers/tests/test_split_test.py index 468cc30c8fbf..2ca0ce793fb1 100644 --- a/lms/djangoapps/course_blocks/transformers/tests/test_split_test.py +++ b/lms/djangoapps/course_blocks/transformers/tests/test_split_test.py @@ -209,7 +209,7 @@ def test_user_randomly_assigned(self): self.course.location, self.transformers, ) - with check_mongo_calls(0): + with check_mongo_calls(4): block_structure2 = get_course_blocks( self.user, self.course.location, From f7d7c18cce98b9536df6df9bc08a04ccea59274d Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Fri, 17 Nov 2023 14:53:40 +0530 Subject: [PATCH 77/82] feat: update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index f1fb22db9479..edb3e5b34c7d 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -207,7 +207,7 @@ class TestGetBlocksQueryCounts(TestGetBlocksQueryCountsBase): """ @ddt.data( - (ModuleStoreEnum.Type.split, ) + (ModuleStoreEnum.Type.split) ) @ddt.unpack def test_query_counts_cached(self, store_type): From 9e49c1bf709424a3c767d8c78e81b8eee3dfed8a Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Mon, 20 Nov 2023 14:33:09 +0530 Subject: [PATCH 78/82] feat: update test_api.py --- lms/djangoapps/course_api/blocks/tests/test_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lms/djangoapps/course_api/blocks/tests/test_api.py b/lms/djangoapps/course_api/blocks/tests/test_api.py index edb3e5b34c7d..7ba44fbe99f8 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_api.py +++ b/lms/djangoapps/course_api/blocks/tests/test_api.py @@ -209,7 +209,6 @@ class TestGetBlocksQueryCounts(TestGetBlocksQueryCountsBase): @ddt.data( (ModuleStoreEnum.Type.split) ) - @ddt.unpack def test_query_counts_cached(self, store_type): course = self._create_course(store_type) self._get_blocks( From 649707412e8d5aa402b4e1e3cad3a65b6e34941c Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Mon, 20 Nov 2023 15:51:51 +0530 Subject: [PATCH 79/82] feat: update test_tasks.py --- lms/djangoapps/grades/tests/test_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/grades/tests/test_tasks.py b/lms/djangoapps/grades/tests/test_tasks.py index 50251405fdbd..7baa019f806b 100644 --- a/lms/djangoapps/grades/tests/test_tasks.py +++ b/lms/djangoapps/grades/tests/test_tasks.py @@ -206,7 +206,7 @@ def test_other_inaccessible_subsection(self, mock_subsection_signal): } self.assertSetEqual( sequentials_signalled, - {self.sequential.location, accessible_seq.location}, + {self.sequential.location}, ) @ddt.data( From 6ee0687a8bba68654d811435be940304b6aa1179 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 21 Nov 2023 10:46:08 +0530 Subject: [PATCH 80/82] feat: update test_field_override_performance.py --- lms/djangoapps/ccx/tests/test_field_override_performance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/ccx/tests/test_field_override_performance.py b/lms/djangoapps/ccx/tests/test_field_override_performance.py index 5652d51e5cfd..f2681f235ab7 100644 --- a/lms/djangoapps/ccx/tests/test_field_override_performance.py +++ b/lms/djangoapps/ccx/tests/test_field_override_performance.py @@ -190,7 +190,7 @@ def instrument_course_progress_render( with self.assertXBlockInstantiations(1): self.grade_course(course_key) - @ddt.data(*itertools.product(('no_overrides', 'ccx'), list(range(1, 4)), (True, False), (True, False))) + @ddt.data(*itertools.product(('no_overrides', 'ccx'), list(range(121, 4)), (True, False), (True, False))) @ddt.unpack @override_settings( XBLOCK_FIELD_DATA_WRAPPERS=[], From 201361d1b3452c46561284322a8b51ddf725b47e Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 21 Nov 2023 11:22:24 +0530 Subject: [PATCH 81/82] feat: update test_tasks_helper.py --- lms/djangoapps/instructor_task/tests/test_tasks_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/instructor_task/tests/test_tasks_helper.py b/lms/djangoapps/instructor_task/tests/test_tasks_helper.py index e3c290174538..4bb6bdb49a74 100644 --- a/lms/djangoapps/instructor_task/tests/test_tasks_helper.py +++ b/lms/djangoapps/instructor_task/tests/test_tasks_helper.py @@ -406,7 +406,7 @@ def test_query_counts(self): with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): with check_mongo_calls(2): - with self.assertNumQueries(50): + with self.assertNumQueries(58): CourseGradeReport.generate(None, None, course.id, {}, 'graded') def test_inactive_enrollments(self): From 009ca635d4a5a1dbb7d5657e0ad48d0638342635 Mon Sep 17 00:00:00 2001 From: Ajamal Husain Date: Tue, 21 Nov 2023 17:38:16 +0530 Subject: [PATCH 82/82] feat: update test_views.py --- lms/djangoapps/courseware/tests/test_views.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lms/djangoapps/courseware/tests/test_views.py b/lms/djangoapps/courseware/tests/test_views.py index bebd4a79fb93..e0d82d9e6f72 100644 --- a/lms/djangoapps/courseware/tests/test_views.py +++ b/lms/djangoapps/courseware/tests/test_views.py @@ -1491,8 +1491,8 @@ def test_view_certificate_link(self): self.assertContains(resp, "earned a certificate for this course.") @ddt.data( - (True, 53), - (False, 53), + (True, 60), + (False, 60), ) @ddt.unpack def test_progress_queries_paced_courses(self, self_paced, query_count): @@ -1500,21 +1500,21 @@ def test_progress_queries_paced_courses(self, self_paced, query_count): # TODO: decrease query count as part of REVO-28 ContentTypeGatingConfig.objects.create(enabled=True, enabled_as_of=datetime(2018, 1, 1)) self.setup_course(self_paced=self_paced) - with self.assertNumQueries(query_count, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST), check_mongo_calls(2): + with self.assertNumQueries(query_count, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST), check_mongo_calls(3): self._get_progress_page() def test_progress_queries(self): ContentTypeGatingConfig.objects.create(enabled=True, enabled_as_of=datetime(2018, 1, 1)) self.setup_course() with self.assertNumQueries( - 53, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST - ), check_mongo_calls(2): + 60, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST + ), check_mongo_calls(3): self._get_progress_page() for _ in range(2): with self.assertNumQueries( - 37, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST - ), check_mongo_calls(2): + 44, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST + ), check_mongo_calls(3): self._get_progress_page() @patch.dict(settings.FEATURES, {'ENABLE_CERTIFICATES_IDV_REQUIREMENT': True})