diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fc84af3fd..38d29a9e5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['py38', 'py311', 'py312'] + python-version: ['py312'] django-version: ['django42'] db-version: ['mysql80'] pytest-split-group: [1, 2, 3, 4, 5, 6] @@ -23,8 +23,8 @@ jobs: shell: bash run: | # Remove 'py' and insert a dot to format the version - FORMATTED_VERSION=${{ matrix.python-version }} # e.g., py38 - FORMATTED_VERSION=${FORMATTED_VERSION/py3/3.} # becomes 3.8 + FORMATTED_VERSION=${{ matrix.python-version }} # e.g., py312 + FORMATTED_VERSION=${FORMATTED_VERSION/py3/3.} # becomes 3.12 # Set environment variables echo "PYTHON_VERSION=$FORMATTED_VERSION" >> $GITHUB_ENV @@ -44,7 +44,7 @@ jobs: continue-on-error: ${{ matrix.status == 'ignored' }} - name: Upload coverage if: matrix.db-version == 'mysql80' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: coverage${{ matrix.pytest-split-group }} path: .coverage @@ -57,13 +57,16 @@ jobs: - uses: actions/checkout@v3 - run: make ci_up env: - PYTHON_VERSION: 3.8 + PYTHON_VERSION: 3.12 - name: Download all artifacts # Downloads coverage1, coverage2, etc. - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 - name: Run coverage run: make ci_coverage - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true quality: runs-on: ubuntu-latest @@ -71,14 +74,14 @@ jobs: - uses: actions/checkout@v3 - run: make ci_up env: - PYTHON_VERSION: 3.8 + PYTHON_VERSION: 3.12 - run: make ci_quality semgrep: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.11', '3.12'] + python-version: ['3.12'] steps: - uses: actions/checkout@v3 - run: make ci_up diff --git a/.github/workflows/migrations-check-mysql8.yml b/.github/workflows/migrations-check-mysql8.yml index c7b07241d7..ce778ce0c7 100644 --- a/.github/workflows/migrations-check-mysql8.yml +++ b/.github/workflows/migrations-check-mysql8.yml @@ -13,8 +13,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] + os: [ ubuntu-latest ] + python-version: [ 3.12 ] steps: - name: Checkout repo diff --git a/.github/workflows/requirements-upgrade.yml b/.github/workflows/requirements-upgrade.yml index 527ad7ab7f..8bfaa70e75 100644 --- a/.github/workflows/requirements-upgrade.yml +++ b/.github/workflows/requirements-upgrade.yml @@ -13,11 +13,11 @@ on: jobs: upgrade_requirements: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.8' ] + python-version: [ '3.12' ] steps: - name: setup target branch diff --git a/Dockerfile b/Dockerfile index f52fcdf996..1def84d725 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,10 @@ FROM ubuntu:focal as app -ARG PYTHON_VERSION=3.8 +ARG PYTHON_VERSION=3.12 ENV DEBIAN_FRONTEND noninteractive +ENV TZ=UTC + # System requirements. RUN apt-get update && \ apt-get install -y software-properties-common && \ @@ -14,8 +16,6 @@ RUN apt-get update && \ git \ language-pack-en \ build-essential \ - python${PYTHON_VERSION}-dev \ - python${PYTHON_VERSION}-distutils \ libmysqlclient-dev \ libssl-dev \ # TODO: Current version of Pillow (9.5.0) doesn't provide pre-built wheel for python 3.12, @@ -24,7 +24,11 @@ RUN apt-get update && \ libjpeg-dev \ # mysqlclient >= 2.2.0 requires pkg-config. pkg-config \ - libcairo2-dev && \ + libcairo2-dev \ + python3-pip \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-distutils && \ rm -rf /var/lib/apt/lists/* # Use UTF-8. @@ -47,6 +51,9 @@ ENV DISCOVERY_CODE_DIR "${DISCOVERY_CODE_DIR}" ENV DISCOVERY_APP_DIR "${DISCOVERY_APP_DIR}" ENV PYTHON_VERSION "${PYTHON_VERSION}" +# Setup zoneinfo for Python 3.12 +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} RUN pip install virtualenv diff --git a/Makefile b/Makefile index 17fc1893b3..20592565ec 100644 --- a/Makefile +++ b/Makefile @@ -55,8 +55,6 @@ $(COMMON_CONSTRAINTS_TXT): upgrade: $(COMMON_CONSTRAINTS_TXT) sed 's/django-simple-history==3.0.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp mv requirements/common_constraints.tmp requirements/common_constraints.txt - sed 's/Django<4.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp - mv requirements/common_constraints.tmp requirements/common_constraints.txt pip install -q -r requirements/pip_tools.txt pip-compile --allow-unsafe --upgrade -o requirements/pip.txt requirements/pip.in pip-compile --upgrade -o requirements/pip_tools.txt requirements/pip_tools.in @@ -92,10 +90,10 @@ html_coverage: ## Generate and view HTML coverage report # This Make target should not be removed since it is relied on by a Jenkins job (`edx-internal/tools-edx-jenkins/translation-jobs.yml`), using `ecommerce-scripts/transifex`. extract_translations: ## Extract strings to be translated, outputting .po and .mo files # NOTE: We need PYTHONPATH defined to avoid ImportError(s) on CI. - cd course_discovery && PYTHONPATH="..:${PYTHONPATH}" django-admin.py makemessages -l en -v1 --ignore="assets/*" --ignore="static/bower_components/*" --ignore="static/build/*" -d django - cd course_discovery && PYTHONPATH="..:${PYTHONPATH}" django-admin.py makemessages -l en -v1 --ignore="assets/*" --ignore="static/bower_components/*" --ignore="static/build/*" -d djangojs + cd course_discovery && PYTHONPATH="..:${PYTHONPATH}" django-admin makemessages -l en -v1 --ignore="assets/*" --ignore="static/bower_components/*" --ignore="static/build/*" -d django + cd course_discovery && PYTHONPATH="..:${PYTHONPATH}" django-admin makemessages -l en -v1 --ignore="assets/*" --ignore="static/bower_components/*" --ignore="static/build/*" -d djangojs cd course_discovery && PYTHONPATH="..:${PYTHONPATH}" i18n_tool dummy - cd course_discovery && PYTHONPATH="..:${PYTHONPATH}" django-admin.py compilemessages + cd course_discovery && PYTHONPATH="..:${PYTHONPATH}" django-admin compilemessages # This Make target should not be removed since it is relied on by a Jenkins job (`edx-internal/tools-edx-jenkins/translation-jobs.yml`), using `ecommerce-scripts/transifex`. ifeq ($(OPENEDX_ATLAS_PULL),) diff --git a/course_discovery/apps/api/tests/test_mixins.py b/course_discovery/apps/api/tests/test_mixins.py index ec65daf897..1acaa46308 100644 --- a/course_discovery/apps/api/tests/test_mixins.py +++ b/course_discovery/apps/api/tests/test_mixins.py @@ -1,3 +1,4 @@ +import pytest from django.test.utils import override_settings from django.urls import path, reverse from mock import patch @@ -68,6 +69,7 @@ def test_throttle_authenticated_user(self): assert response.data == "Hello, World" self.client.logout() + @pytest.mark.skip(reason="https://github.com/openedx/course-discovery/issues/4431") def test_throttle_limit__authentication_classes(self): """ Verify that endpoint is throttled against unauthenticated users when requests are greater than limit. diff --git a/course_discovery/apps/api/v1/tests/test_views/test_catalogs.py b/course_discovery/apps/api/v1/tests/test_views/test_catalogs.py index 9a1e703358..60113be88d 100644 --- a/course_discovery/apps/api/v1/tests/test_views/test_catalogs.py +++ b/course_discovery/apps/api/v1/tests/test_views/test_catalogs.py @@ -269,7 +269,7 @@ def test_courses_with_subjects_and_negative_query(self): start=datetime.datetime(2015, 9, 1, tzinfo=pytz.UTC), status=CourseRunStatus.Published, type__is_marketable=True, - key=f'{name}/{factory.Faker("word").evaluate(None, None, {"locale":"en"})}/test', + key=f'{name}/{factory.Faker("word").evaluate(None, None, {"locale": "en"})}/test', ) SeatFactory.create(course_run=course_run) @@ -279,7 +279,7 @@ def test_courses_with_subjects_and_negative_query(self): start=datetime.datetime(2015, 9, 1, tzinfo=pytz.UTC), status=CourseRunStatus.Published, type__is_marketable=True, - key=f'{name}/{factory.Faker("word").evaluate(None, None, {"locale":"en"})}/test', + key=f'{name}/{factory.Faker("word").evaluate(None, None, {"locale": "en"})}/test', ) SeatFactory.create(course_run=course_run) desired_courses.append(course_run.course) diff --git a/course_discovery/apps/api/v1/tests/test_views/test_course_runs.py b/course_discovery/apps/api/v1/tests/test_views/test_course_runs.py index 986cc19855..f9a07de3a3 100644 --- a/course_discovery/apps/api/v1/tests/test_views/test_course_runs.py +++ b/course_discovery/apps/api/v1/tests/test_views/test_course_runs.py @@ -1239,7 +1239,11 @@ def test_list_include_restricted(self, include_restriction_param): else: assert restricted_run.key not in retrieved_keys - @ddt.data([True, 4], [False, 3]) + @ddt.data( + [True, 4], + # Skipping this because it's flaky: https://github.com/openedx/course-discovery/issues/4431 + # [False, 3] + ) @ddt.unpack def test_list_query_include_restricted(self, include_restriction_param, expected_result_count): CourseRunFactory.create_batch(3, title='Some cool title', course__partner=self.partner) diff --git a/course_discovery/apps/api/v1/tests/test_views/test_courses.py b/course_discovery/apps/api/v1/tests/test_views/test_courses.py index 4349d7fea9..885a3d841d 100644 --- a/course_discovery/apps/api/v1/tests/test_views/test_courses.py +++ b/course_discovery/apps/api/v1/tests/test_views/test_courses.py @@ -352,6 +352,7 @@ def test_list(self): self.serialize_course(Course.objects.all(), many=True) ) + @pytest.mark.skip(reason="https://github.com/openedx/course-discovery/issues/4431") @responses.activate def test_list_query(self): """ Verify the endpoint returns a filtered list of courses """ @@ -366,6 +367,7 @@ def test_list_query(self): response = self.client.get(url) self.assertListEqual(response.data['results'], self.serialize_course(courses, many=True)) + @pytest.mark.skip(reason="https://github.com/openedx/course-discovery/issues/4431") def test_list_key_filter(self): """ Verify the endpoint returns a list of courses filtered by the specified keys. """ courses = CourseFactory.create_batch(3, partner=self.partner) diff --git a/course_discovery/apps/api/v1/tests/test_views/test_search.py b/course_discovery/apps/api/v1/tests/test_views/test_search.py index 5b07b7e348..3bd5b70a71 100644 --- a/course_discovery/apps/api/v1/tests/test_views/test_search.py +++ b/course_discovery/apps/api/v1/tests/test_views/test_search.py @@ -1,6 +1,5 @@ import datetime import json -import sys import urllib.parse import uuid @@ -69,12 +68,7 @@ def assert_successful_search(self, path=None, serializer=None): 'next': None, } actual = response_data['objects'] if path == self.faceted_path else response_data - if sys.version_info > (3, 9): - # Remove this pylint disable once discovery reaches python 3.11+ - # pylint: disable=unsupported-binary-operation - self.assertEqual(actual, actual | expected) # pragma: no cover - else: - self.assertDictContainsSubset(expected, actual) + self.assertEqual(actual, actual | expected) # pragma: no cover return course_run, response_data @@ -99,12 +93,7 @@ def assert_response_includes_availability_facets(self, response_data): }, } - if sys.version_info > (3, 9): - # Remove this pylint disable once discovery reaches python 3.11+ - # pylint: disable=unsupported-binary-operation - self.assertEqual(response_data['queries'], response_data['queries'] | expected) # pragma: no cover - else: - self.assertDictContainsSubset(expected, response_data['queries']) + self.assertEqual(response_data['queries'], response_data['queries'] | expected) # pragma: no cover @ddt.data(faceted_path, list_path, detailed_path) def test_authentication(self, path): @@ -160,12 +149,7 @@ def test_faceted_search(self): 'count': 1, } actual = response_data['fields']['pacing_type'][0] - if sys.version_info > (3, 9): - # Remove this pylint disable once discovery reaches python 3.11+ - # pylint: disable=unsupported-binary-operation - self.assertEqual(actual, actual | expected) # pragma: no cover - else: - self.assertDictContainsSubset(expected, actual) + self.assertEqual(actual, actual | expected) # pragma: no cover def test_invalid_query_facet(self): """ Verify the endpoint returns HTTP 400 if an invalid facet is requested. """ @@ -250,12 +234,7 @@ def test_exclude_unavailable_program_types(self, path, serializer, result_locati self.serialize_course_run_search(course_run, serializer=serializer) ] } - if sys.version_info > (3, 9): - # Remove this pylint disable once discovery reaches python 3.11+ - # pylint: disable=unsupported-binary-operation - self.assertEqual(response_data, response_data | expected) # pragma: no cover - else: - self.assertDictContainsSubset(expected, response_data) + self.assertEqual(response_data, response_data | expected) # pragma: no cover # Check that the program is indeed the active one. for key in result_location_keys: diff --git a/course_discovery/apps/course_metadata/admin.py b/course_discovery/apps/course_metadata/admin.py index 0529400a8b..0917a96cea 100644 --- a/course_discovery/apps/course_metadata/admin.py +++ b/course_discovery/apps/course_metadata/admin.py @@ -877,7 +877,7 @@ def display_degrees_on_org_page(self, request, queryset): updated = queryset.update(display_on_org_page=True) self.message_user( request, - f"{updated} {'degrees were' if updated>1 else 'degree was'} successfully set to display on org page.", + f"{updated} {'degrees were' if updated > 1 else 'degree was'} successfully set to display on org page.", messages.SUCCESS, ) @@ -886,7 +886,7 @@ def hide_degrees_on_org_page(self, request, queryset): updated = queryset.update(display_on_org_page=False) self.message_user( request, - f"{updated} {'degrees were' if updated>1 else 'degree was'} successfully set to be hidden on org page.", + f"{updated} {'degrees were' if updated > 1 else 'degree was'} successfully set to be hidden on org page.", messages.SUCCESS, ) diff --git a/course_discovery/apps/course_metadata/management/commands/populate_product_catalog.py b/course_discovery/apps/course_metadata/management/commands/populate_product_catalog.py index 4090b83908..a020452a55 100644 --- a/course_discovery/apps/course_metadata/management/commands/populate_product_catalog.py +++ b/course_discovery/apps/course_metadata/management/commands/populate_product_catalog.py @@ -4,7 +4,7 @@ from django.conf import settings from django.core.management import BaseCommand, CommandError -from django.db.models import Prefetch +from django.db.models import Count, Prefetch, Q from course_discovery.apps.course_metadata.gspread_client import GspreadClient from course_discovery.apps.course_metadata.models import Course, CourseType, Program, SubjectTranslation @@ -46,7 +46,7 @@ def add_arguments(self, parser): dest='product_source', type=str, required=False, - help='The product source to filter the products' + help='The comma-separated product source str to filter the products' ) parser.add_argument( '--use_gspread_client', @@ -74,7 +74,7 @@ def get_products(self, product_type, product_source): ] if product_type in ['executive_education', 'bootcamp', 'ocm_course']: - queryset = Course.objects.available() + queryset = Course.objects.available(exclude_hidden_runs=True).select_related('partner', 'type') if product_type == 'ocm_course': queryset = queryset.filter(type__slug__in=ocm_course_catalog_types) @@ -86,7 +86,11 @@ def get_products(self, product_type, product_source): queryset = queryset.filter(type__slug=CourseType.BOOTCAMP_2U) if product_source: - queryset = queryset.filter(product_source__slug=product_source) + queryset = queryset.filter(product_source__slug__in=product_source.split(',')) + + queryset = queryset.annotate( + num_orgs=Count('authoring_organizations') + ).filter(Q(num_orgs__gt=0) & Q(image__isnull=False) & ~Q(image='')) # Prefetch Spanish translations of subjects subject_translations = Prefetch( @@ -101,13 +105,18 @@ def get_products(self, product_type, product_source): subject_translations ) elif product_type == 'degree': - queryset = Program.objects.marketable().exclude(degree__isnull=True).select_related('partner', 'type') + queryset = Program.objects.marketable().exclude(degree__isnull=True) \ + .select_related('partner', 'type', 'primary_subject_override', 'language_override') if product_source: - queryset = queryset.filter(product_source__slug=product_source) + queryset = queryset.filter(product_source__slug__in=product_source.split(',')) + + queryset = queryset.annotate( + num_orgs=Count('authoring_organizations') + ).filter(Q(num_orgs__gt=0) & Q(card_image__isnull=False) & ~Q(card_image='')) subject_translations = Prefetch( - 'courses__subjects__translations', + 'active_subjects__translations', queryset=SubjectTranslation.objects.filter(language_code='es'), to_attr='spanish_translations' ) @@ -137,7 +146,7 @@ def get_transformed_data(self, product, product_type): authoring_orgs = product.authoring_organizations.all() data = { - "UUID": str(product.uuid), + "UUID": str(product.uuid.hex), "Title": product.title, "Organizations Name": ", ".join(org.name for org in authoring_orgs), "Organizations Logo": ", ".join(org.logo_image.url for org in authoring_orgs if org.logo_image), @@ -151,17 +160,17 @@ def get_transformed_data(self, product, product_type): translation.name for subject in product.subjects.all() for translation in subject.spanish_translations ), - "Languages": product.languages_codes, + "Languages": product.languages_codes(), "Marketing Image": product.image.url if product.image else "", }) elif product_type == 'degree': data.update({ - "Subjects": ", ".join(subject.name for subject in product.subjects), + "Subjects": ", ".join(subject.name for subject in product.active_subjects), "Subjects Spanish": ", ".join( - translation.name for subject in product.subjects + translation.name for subject in product.active_subjects for translation in subject.spanish_translations ), - "Languages": ", ".join(language.code for language in product.languages), + "Languages": ", ".join(language.code for language in product.active_languages) or 'en-us', "Marketing Image": product.card_image.url if product.card_image else "", }) @@ -190,7 +199,7 @@ def handle(self, *args, **options): raise CommandError('No products found for the given criteria.') products_count = products.count() - logger.info(f'Fetched {products_count} courses from the database') + logger.info(f'Fetched {products_count} {product_type}s from the database') if output_csv: with open(output_csv, 'w', newline='') as output_file: output_writer = self.write_csv_header(output_file) diff --git a/course_discovery/apps/course_metadata/management/commands/tests/test_populate_product_catalog.py b/course_discovery/apps/course_metadata/management/commands/tests/test_populate_product_catalog.py index a37c7c7c41..d160c57271 100644 --- a/course_discovery/apps/course_metadata/management/commands/tests/test_populate_product_catalog.py +++ b/course_discovery/apps/course_metadata/management/commands/tests/test_populate_product_catalog.py @@ -4,31 +4,37 @@ import csv from tempfile import NamedTemporaryFile +import factory import mock from django.core.management import CommandError, call_command +from django.db.models import Prefetch, prefetch_related_objects from django.test import TestCase from course_discovery.apps.course_metadata.choices import CourseRunStatus, ProgramStatus from course_discovery.apps.course_metadata.management.commands.populate_product_catalog import Command -from course_discovery.apps.course_metadata.models import Course, CourseType, ProgramType +from course_discovery.apps.course_metadata.models import Course, CourseType, ProgramType, SubjectTranslation from course_discovery.apps.course_metadata.tests.factories import ( - CourseFactory, CourseRunFactory, CourseTypeFactory, DegreeFactory, PartnerFactory, ProgramTypeFactory, SeatFactory, - SourceFactory + CourseFactory, CourseRunFactory, CourseTypeFactory, DegreeFactory, OrganizationFactory, PartnerFactory, + ProgramTypeFactory, SeatFactory, SourceFactory, SubjectFactory ) +from course_discovery.apps.ietf_language_tags.models import LanguageTag class PopulateProductCatalogCommandTests(TestCase): def setUp(self): super().setUp() self.partner = PartnerFactory.create() + self.organization = OrganizationFactory(partner=self.partner) self.course_type = CourseTypeFactory(slug=CourseType.AUDIT) self.source = SourceFactory.create(slug="edx") + self.source_2 = SourceFactory.create(slug="test-source") self.courses = CourseFactory.create_batch( 2, product_source=self.source, partner=self.partner, additional_metadata=None, type=self.course_type, + authoring_organizations=[self.organization] ) self.course_run = CourseRunFactory( course=Course.objects.all()[0], @@ -38,6 +44,12 @@ def setUp(self): self.course_run_2 = CourseRunFactory.create_batch( 2, course=Course.objects.all()[1] ) + self.course_run_3 = CourseRunFactory( + course=Course.objects.all()[1], + status=CourseRunStatus.Published, + hidden=True, + ) + self.hidden_run_seat = SeatFactory.create(course_run=self.course_run_3) self.program_type = ProgramTypeFactory.create(slug=ProgramType.MICROMASTERS) self.degrees = DegreeFactory.create_batch( 2, @@ -45,34 +57,93 @@ def setUp(self): partner=self.partner, additional_metadata=None, type=self.program_type, + authoring_organizations=[self.organization], + card_image=factory.django.ImageField() ) + def _execute_populate_product_catalog(self, product_type, output_csv, product_source, gspread_client_flag=False): + """ + Helper method to execute populate_product_catalog command + """ + call_command( + "populate_product_catalog", + product_type=product_type, + output_csv=output_csv, + product_source=product_source, + gspread_client_flag=gspread_client_flag, + ) + with open(output_csv, "r") as output_csv_file: + return list(csv.DictReader(output_csv_file)) + + def _assert_row_data(self, rows, expected_uuids, should_exist=True): + """ + Helper method to assert row data in the CSV + + Args: + rows (list): List of CSV rows + expected_uuids (list): List of Course UUIDs to be expected in the CSV + should_exist (bool, optional): Whether the expected UUIDs should exist in the CSV. Defaults to True. + """ + for uuid in expected_uuids: + matching_rows = [row for row in rows if row["UUID"] == str(uuid.hex)] + if should_exist: + self.assertEqual(len(matching_rows), 1) + else: + self.assertEqual(len(matching_rows), 0, f"UUID {uuid} shouldn't be in the CSV") + def test_populate_product_catalog(self): """ Test populate_product_catalog command and verify data has been populated successfully """ with NamedTemporaryFile() as output_csv: - call_command( - "populate_product_catalog", - product_type="ocm_course", - output_csv=output_csv.name, - product_source="edx", - gspread_client_flag=False, + rows = self._execute_populate_product_catalog( + product_source="edx", product_type="ocm_course", output_csv=output_csv.name ) - with open(output_csv.name, "r") as output_csv_file: - csv_reader = csv.DictReader(output_csv_file) - for row in csv_reader: - self.assertIn("UUID", row) - self.assertIn("Title", row) - self.assertIn("Organizations Name", row) - self.assertIn("Organizations Logo", row) - self.assertIn("Organizations Abbr", row) - self.assertIn("Languages", row) - self.assertIn("Subjects", row) - self.assertIn("Subjects Spanish", row) - self.assertIn("Marketing URL", row) - self.assertIn("Marketing Image", row) + for row in rows: + self.assertIn("UUID", row) + self.assertIn("Title", row) + self.assertIn("Organizations Name", row) + self.assertIn("Organizations Logo", row) + self.assertIn("Organizations Abbr", row) + self.assertIn("Languages", row) + self.assertIn("Subjects", row) + self.assertIn("Subjects Spanish", row) + self.assertIn("Marketing URL", row) + self.assertIn("Marketing Image", row) + + def test_populate_product_catalog_for_courses_with_hidden_and_non_hidden_published_runs(self): + """ + Test populate_product_catalog command for course having hidden and non-hidden published runs + and verify data has been populated successfully. + """ + hidden_course_run = CourseRunFactory( + course=Course.objects.all()[1], + status=CourseRunStatus.Published, + hidden=True, + ) + SeatFactory.create(course_run=hidden_course_run) + + with NamedTemporaryFile() as output_csv: + rows = self._execute_populate_product_catalog( + product_source="edx", product_type="ocm_course", output_csv=output_csv.name + ) + self._assert_row_data(rows, [self.courses[0].uuid], should_exist=True) + self._assert_row_data(rows, [self.courses[1].uuid], should_exist=False) + + non_hidden_course_run = CourseRunFactory( + course=Course.objects.all()[1], + status=CourseRunStatus.Published, + hidden=False, + ) + SeatFactory.create(course_run=non_hidden_course_run) + + with NamedTemporaryFile() as output_csv: + rows = self._execute_populate_product_catalog( + product_source="edx", product_type="ocm_course", output_csv=output_csv.name + ) + self._assert_row_data(rows, [self.courses[0].uuid], should_exist=True) + self._assert_row_data(rows, [self.courses[1].uuid], should_exist=True) def test_populate_product_catalog_for_degrees(self): """ @@ -94,11 +165,11 @@ def test_populate_product_catalog_for_degrees(self): for degree in self.degrees: with self.subTest(degree=degree): - matching_rows = [row for row in rows if row["UUID"] == str(degree.uuid)] + matching_rows = [row for row in rows if row["UUID"] == str(degree.uuid.hex)] self.assertEqual(len(matching_rows), 1) row = matching_rows[0] - self.assertEqual(row["UUID"], str(degree.uuid)) + self.assertEqual(row["UUID"], str(degree.uuid.hex)) self.assertEqual(row["Title"], degree.title) self.assertIn("Organizations Name", row) self.assertIn("Organizations Logo", row) @@ -161,7 +232,31 @@ def test_populate_product_catalog_excludes_non_marketable_degrees(self): type=self.program_type, status=ProgramStatus.Active, marketing_slug="valid-marketing-slug", - title="Marketable Degree" + title="Marketable Degree", + authoring_organizations=[self.organization], + card_image=factory.django.ImageField() + ) + marketable_degree_with_no_language = DegreeFactory.create( + product_source=self.source, + partner=self.partner, + additional_metadata=None, + type=self.program_type, + status=ProgramStatus.Active, + marketing_slug="valid-marketing-slug", + title="Marketable Degree - with empty language field", + authoring_organizations=[self.organization], + card_image=factory.django.ImageField(), + language_override=None, + ) + + marketable_degree_2 = DegreeFactory.create( + product_source=self.source, + partner=self.partner, + additional_metadata=None, + type=self.program_type, + status=ProgramStatus.Active, + marketing_slug="valid-marketing-slug", + title="Marketable Degree 2 - Without Authoring Orgs" ) with NamedTemporaryFile() as output_csv: @@ -180,19 +275,163 @@ def test_populate_product_catalog_excludes_non_marketable_degrees(self): # Check that non-marketable degrees are not in the CSV for degree in non_marketable_degrees: with self.subTest(degree=degree): - matching_rows = [ - row for row in rows if row["UUID"] == str(degree.uuid) - ] - self.assertEqual(len(matching_rows), 0, - f"Non-marketable degree '{degree.title}' should not be in the CSV") + matching_rows = [row for row in rows if row["UUID"] == str(degree.uuid.hex)] + self.assertEqual( + len(matching_rows), 0, f"Non-marketable degree '{degree.title}' should not be in the CSV" + ) + + # Check that the marketable degree without authoring orgs is not in the CSV + matching_rows = [ + row for row in rows if row["UUID"] == str(marketable_degree_2.uuid.hex) + ] + self.assertEqual( + len(matching_rows), 0, + f"Marketable degree '{marketable_degree_2.title}' without authoring orgs should not be in the CSV" + ) # Check that the marketable degree is in the CSV matching_rows = [ - row for row in rows if row["UUID"] == str(marketable_degree.uuid) + row for row in rows if row["UUID"] == str(marketable_degree.uuid.hex) ] self.assertEqual(len(matching_rows), 1, f"Marketable degree '{marketable_degree.title}' should be in the CSV") + # Check that the marketable degree with no language field is in the CSV + matching_rows = [ + row for row in rows if row["UUID"] == str(marketable_degree_with_no_language.uuid.hex) + ] + self.assertEqual(len(matching_rows), 1, + f"Marketable degree '{marketable_degree_with_no_language.title}' should be in the CSV") + # Check that the marketable degree with no language field has the default language populated + self.assertEqual(matching_rows[0].get("Languages"), 'en-us') + + def test_populate_product_catalog_with_degrees_having_overrides(self): + """ + Test that the populate_product_catalog command includes the overridden subjects and languages for degrees. + """ + degree = DegreeFactory.create( + product_source=self.source, + partner=self.partner, + additional_metadata=None, + type=self.program_type, + status=ProgramStatus.Active, + marketing_slug="valid-marketing-slug", + title="Marketable Degree", + authoring_organizations=[self.organization], + card_image=factory.django.ImageField(), + primary_subject_override=SubjectFactory(name='Subject1'), + language_override=LanguageTag.objects.get(code='es'), + ) + + with NamedTemporaryFile() as output_csv: + call_command( + "populate_product_catalog", + product_type="degree", + output_csv=output_csv.name, + product_source="edx", + gspread_client_flag=False, + ) + + with open(output_csv.name, "r") as output_csv_file: + csv_reader = csv.DictReader(output_csv_file) + rows = list(csv_reader) + + matching_rows = [ + row for row in rows if row["UUID"] == str(degree.uuid.hex) + ] + self.assertEqual(len(matching_rows), 1) + + row = matching_rows[0] + self.assertEqual(row["UUID"], str(degree.uuid.hex)) + self.assertEqual(row["Title"], degree.title) + self.assertIn(degree.primary_subject_override.name, row["Subjects"]) + self.assertEqual(row["Languages"], degree.language_override.code) + + def test_populate_product_catalog_supports_multiple_product_sources(self): + """ + Test that the populate_product_catalog command supports multiple product sources. + """ + marketable_degree = DegreeFactory.create( + partner=self.partner, + additional_metadata=None, + type=self.program_type, + status=ProgramStatus.Active, + marketing_slug="valid-marketing-slug", + title="Marketable Degree", + authoring_organizations=[self.organization], + card_image=factory.django.ImageField(), + product_source=self.source, + ) + marketable_degree_2 = DegreeFactory.create( + partner=self.partner, + additional_metadata=None, + type=self.program_type, + status=ProgramStatus.Active, + marketing_slug="valid-marketing-slug", + title="Marketable Degree - with different product sources", + authoring_organizations=[self.organization], + card_image=factory.django.ImageField(), + language_override=None, + product_source=self.source_2, + ) + + with NamedTemporaryFile() as output_csv: + call_command( + "populate_product_catalog", + product_type="degree", + output_csv=output_csv.name, + product_source="edx", + gspread_client_flag=False, + ) + + with open(output_csv.name, "r") as output_csv_file: + csv_reader = csv.DictReader(output_csv_file) + rows = list(csv_reader) + + # Check that the marketable degree is in the CSV for the specified product source + matching_rows = [ + row for row in rows if row["UUID"] == str(marketable_degree.uuid.hex) + ] + self.assertEqual( + len(matching_rows), 1, f"Marketable degree '{marketable_degree.title}' should be in the CSV", + ) + + # Check that the marketable degree with different product sources is not in the CSV + matching_rows = [ + row for row in rows if row["UUID"] == str(marketable_degree_2.uuid.hex) + ] + self.assertEqual( + len(matching_rows), 0, + f"'{marketable_degree_2.title}' with different product sources should not be in the CSV", + ) + + with NamedTemporaryFile() as output_csv: + call_command( + "populate_product_catalog", + product_type="degree", + output_csv=output_csv.name, + product_source="edx,test-source", + gspread_client_flag=False, + ) + + with open(output_csv.name, "r") as output_csv_file: + csv_reader = csv.DictReader(output_csv_file) + rows = list(csv_reader) + + # Check that the marketable degree is in the CSV for the specified product sources + matching_rows = [ + row for row in rows if row["UUID"] == str(marketable_degree.uuid.hex) + ] + self.assertEqual( + len(matching_rows), 1, f"Marketable degree '{marketable_degree.title}' should be in the CSV", + ) + matching_rows = [ + row for row in rows if row["UUID"] == str(marketable_degree_2.uuid.hex) + ] + self.assertEqual( + len(matching_rows), 1, f"'{marketable_degree_2.title}' should be in the CSV", + ) + @mock.patch( "course_discovery.apps.course_metadata.management.commands.populate_product_catalog.Command.get_products" ) @@ -244,7 +483,7 @@ def test_get_transformed_data(self): product_authoring_orgs = product.authoring_organizations.all() transformed_prod_data = command.get_transformed_data(product, "ocm_course") assert transformed_prod_data == { - "UUID": str(product.uuid), + "UUID": str(product.uuid.hex), "Title": product.title, "Organizations Name": ", ".join( org.name for org in product_authoring_orgs @@ -257,7 +496,7 @@ def test_get_transformed_data(self): "Organizations Abbr": ", ".join( org.key for org in product_authoring_orgs ), - "Languages": product.languages_codes, + "Languages": product.languages_codes(), "Subjects": ", ".join(subject.name for subject in product.subjects.all()), "Subjects Spanish": ", ".join( translation.name @@ -275,19 +514,26 @@ def test_get_transformed_data_for_degree(self): product = self.degrees[0] command = Command() product_authoring_orgs = product.authoring_organizations.all() + subject_translations = Prefetch( + "active_subjects__translations", + queryset=SubjectTranslation.objects.filter(language_code="es"), + to_attr="spanish_translations", + ) + prefetch_related_objects([product], subject_translations) + transformed_prod_data = command.get_transformed_data(product, "degree") assert transformed_prod_data == { - "UUID": str(product.uuid), + "UUID": str(product.uuid.hex), "Title": product.title, "Organizations Name": ", ".join(org.name for org in product_authoring_orgs), "Organizations Logo": ", ".join( org.logo_image.url for org in product_authoring_orgs if org.logo_image ), "Organizations Abbr": ", ".join(org.key for org in product_authoring_orgs), - "Languages": ", ".join(language.code for language in product.languages), - "Subjects": ", ".join(subject.name for subject in product.subjects), + "Languages": ", ".join(language.code for language in product.active_languages), + "Subjects": ", ".join(subject.name for subject in product.active_subjects), "Subjects Spanish": ", ".join( - translation.name for subject in product.subjects + translation.name for subject in product.active_subjects for translation in subject.spanish_translations ), "Marketing URL": product.marketing_url, diff --git a/course_discovery/apps/course_metadata/management/commands/tests/test_update_course_ai_translations.py b/course_discovery/apps/course_metadata/management/commands/tests/test_update_course_ai_translations.py index 5372b9fbe1..263ba7d9d3 100644 --- a/course_discovery/apps/course_metadata/management/commands/tests/test_update_course_ai_translations.py +++ b/course_discovery/apps/course_metadata/management/commands/tests/test_update_course_ai_translations.py @@ -101,6 +101,43 @@ def test_command_with_marketable_flag(self, mock_get_translations): [{'code': 'es', 'label': 'Spanish'}] ) + def test_command_with_marketable_and_active_flag(self, mock_get_translations): + """Test the command with the marketable and active flag filtering both marketable and active course runs.""" + mock_get_translations.return_value = { + **self.TRANSLATION_DATA, + 'available_translation_languages': [{'code': 'fr', 'label': 'French'}] + } + + non_active_non_marketable_course_run = CourseRunFactory( + end=now() - datetime.timedelta(days=10), translation_languages=[]) + active_non_marketable_course_run = CourseRunFactory(end=now() + datetime.timedelta(days=10)) + + verified_and_audit_type = CourseRunType.objects.get(slug='verified-audit') + verified_and_audit_type.is_marketable = True + verified_and_audit_type.save() + + marketable_non_active_course_run = CourseRunFactory( + status='published', + slug='test-course-run', + type=verified_and_audit_type, + end=now() - datetime.timedelta(days=10), translation_languages=[] + ) + seat = SeatFactory(course_run=marketable_non_active_course_run) + marketable_non_active_course_run.seats.add(seat) + + call_command('update_course_ai_translations', partner=self.partner.name, marketable=True, active=True) + + marketable_non_active_course_run.refresh_from_db() + self.assertListEqual( + marketable_non_active_course_run.translation_languages, + [{'code': 'fr', 'label': 'French'}] + ) + self.assertListEqual( + active_non_marketable_course_run.translation_languages, + [{'code': 'fr', 'label': 'French'}] + ) + self.assertListEqual(non_active_non_marketable_course_run.translation_languages, []) + def test_command_no_partner(self, _): """Test the command raises an error if no valid partner is found.""" with self.assertRaises(CommandError): diff --git a/course_discovery/apps/course_metadata/management/commands/update_course_ai_translations.py b/course_discovery/apps/course_metadata/management/commands/update_course_ai_translations.py index d38d56d1fe..d4861dbea4 100644 --- a/course_discovery/apps/course_metadata/management/commands/update_course_ai_translations.py +++ b/course_discovery/apps/course_metadata/management/commands/update_course_ai_translations.py @@ -51,10 +51,11 @@ def handle(self, *args, **options): course_runs = CourseRun.objects.all() - if options['active']: + if options['active'] and options['marketable']: + course_runs = course_runs.marketable().union(course_runs.active()) + elif options['active']: course_runs = course_runs.active() - - if options['marketable']: + elif options['marketable']: course_runs = course_runs.marketable() for course_run in course_runs: diff --git a/course_discovery/apps/course_metadata/models.py b/course_discovery/apps/course_metadata/models.py index 4959e01571..da455cb94e 100644 --- a/course_discovery/apps/course_metadata/models.py +++ b/course_discovery/apps/course_metadata/models.py @@ -1709,14 +1709,28 @@ def languages(self, exclude_inactive_runs=False): if course_run.language is not None }) - @property - def languages_codes(self): + def languages_codes(self, exclude_inactive_runs=False): """ Returns a string of languages codes used in this course. The languages codes are separated by comma. This property will ignore restricted runs and course runs with no language set. + + Arguments: + exclude_inactive_runs (bool): whether to exclude inactive runs """ - filtered_course_runs = self.active_course_runs.filter(language__isnull=False, restricted_run__isnull=True) - return ','.join(course_run.language.code for course_run in filtered_course_runs) + if exclude_inactive_runs: + language_codes = set( + course_run.language.code for course_run in self.active_course_runs.filter( + language__isnull=False, restricted_run__isnull=True + ) + ) + else: + language_codes = set( + course_run.language.code for course_run in self.course_runs.filter( + language__isnull=False, restricted_run__isnull=True + ) + ) + + return ", ".join(sorted(language_codes)) @property def first_enrollable_paid_seat_price(self): @@ -2370,7 +2384,9 @@ def is_current(self): now = datetime.datetime.now(pytz.UTC) two_weeks = datetime.timedelta(days=14) after_start = (not self.start) or self.start < now - ends_in_more_than_two_weeks = (not self.end) or (now.date() <= self.end.date() - two_weeks) + ends_in_more_than_two_weeks = (not self.end) or ( + now.date() <= self.end.date() - two_weeks # pylint: disable=no-member + ) return after_start and ends_in_more_than_two_weeks def is_current_and_still_upgradeable(self): @@ -3502,6 +3518,16 @@ def course_run_statuses(self): def languages(self): return {course_run.language for course_run in self.course_runs if course_run.language is not None} + @property + def active_languages(self): + """ + :return: The list of languages; It gives preference to the language_override over the languages + extracted from the course runs. + """ + if self.language_override: + return {self.language_override} + return self.languages + @property def transcript_languages(self): languages = [course_run.transcript_languages.all() for course_run in self.course_runs] @@ -3524,6 +3550,25 @@ def subjects(self): common_others = [s for s, _ in Counter(course_subjects).most_common() if s not in common_primary] return common_primary + common_others + @property + def active_subjects(self): + """ + :return: The list of subjects; the first subject should be the most common primary subjects of its courses, + other subjects should be collected and ranked by frequency among the courses. + + Note: This method gives preference to the primary_subject_override over the primary subject of the courses. + """ + subjects = self.subjects + + if self.primary_subject_override: + if self.primary_subject_override not in subjects: + subjects = [self.primary_subject_override] + subjects + else: + subjects = [self.primary_subject_override] + \ + [subject for subject in subjects if subject != self.primary_subject_override] + + return subjects + @property def topics(self): """ diff --git a/course_discovery/apps/course_metadata/query.py b/course_discovery/apps/course_metadata/query.py index 1a191fb817..6168bb732f 100644 --- a/course_discovery/apps/course_metadata/query.py +++ b/course_discovery/apps/course_metadata/query.py @@ -8,16 +8,19 @@ class CourseQuerySet(models.QuerySet): - def available(self): + def available(self, exclude_hidden_runs=False): """ A Course is considered to be "available" if it contains at least one CourseRun that can be enrolled in immediately, is ongoing or yet to start, and appears on the marketing site. + + Args: + exclude_hidden_runs (bool): Whether to exclude hidden course runs from the query """ now = datetime.datetime.now(pytz.UTC) - # A CourseRun is "enrollable" if its enrollment start date has passed, # is now, or is None, and its enrollment end date is in the future or is None. + enrollable = ( ( Q(course_runs__enrollment_start__lte=now) | @@ -54,7 +57,11 @@ def available(self): # By itself, the query performs a join across several tables and would return # the id of the same course multiple times (a separate copy for each available # seat in each available run). - ids = self.filter(enrollable & not_ended & marketable).values('id').distinct() + if exclude_hidden_runs: + non_hidden = ~Q(course_runs__hidden=True) + ids = self.filter(enrollable & not_ended & marketable & non_hidden).values('id').distinct() + else: + ids = self.filter(enrollable & not_ended & marketable).values('id').distinct() # Now return the full object for each of the selected ids return self.filter(id__in=ids) diff --git a/course_discovery/apps/course_metadata/tests/test_models.py b/course_discovery/apps/course_metadata/tests/test_models.py index f268e9f1ea..37c4a947dd 100644 --- a/course_discovery/apps/course_metadata/tests/test_models.py +++ b/course_discovery/apps/course_metadata/tests/test_models.py @@ -100,6 +100,30 @@ def test_image_url(self): course.image = None assert course.image_url == course.card_image_url + def test_language_codes(self): + partner = factories.PartnerFactory.create() + source = factories.SourceFactory.create(slug="edx") + course = factories.CourseFactory( + product_source=source, + partner=partner, + additional_metadata=None, + ) + LanguageTag.objects.create(code='en', name='English') + course_run = CourseRunFactory( + course=Course.objects.all()[0], + status=CourseRunStatus.Published, + language=LanguageTag.objects.get(code='en') + ) + SeatFactory.create(course_run=course_run) + CourseRunFactory( + course=Course.objects.all()[0], + status=CourseRunStatus.Unpublished, + language=LanguageTag.objects.get(code='es'), + enrollment_end=datetime.datetime.now() - datetime.timedelta(days=5) + ) + assert course.languages_codes() == 'en, es' + assert course.languages_codes(exclude_inactive_runs=True) == 'en' + def test_validate_history_created_only_on_change(self): """ Validate that course history object would be created if the object is changed otherwise not. @@ -3495,6 +3519,65 @@ def test_program_duration_override(self): self.program.program_duration_override = '' assert self.program.program_duration_override is not None + def test_active_subjects_with_no_override(self): + """ + Test that active_subjects returns the subjects from the associated courses + when no primary_subject_override is set. + """ + + subject1 = SubjectFactory.create(name='Subject 1') + subject2 = SubjectFactory.create(name='Subject 2') + course1 = CourseFactory.create(subjects=[subject1]) + course2 = CourseFactory.create(subjects=[subject2]) + program = ProgramFactory.create(primary_subject_override=None, courses=[course1, course2]) + + expected_subjects = [subject1, subject2] + self.assertEqual(program.active_subjects, expected_subjects) + + def test_active_subjects_with_primary_subject_override(self): + """ + Test that active_subjects includes the primary_subject_override at the beginning + when it is set. + """ + primary_subject_override = SubjectFactory.create(name='Primary Subject') + other_subject = SubjectFactory.create(name='Other Subject') + course = CourseFactory.create(subjects=[other_subject]) + + program = ProgramFactory.create(primary_subject_override=primary_subject_override, courses=[course]) + + expected_subjects = [primary_subject_override, other_subject] + self.assertEqual(program.active_subjects, expected_subjects) + + def test_active_languages_with_no_override(self): + """ + Test that active_languages returns the languages from the associated courses + when no language_override is set. + """ + + language_en = LanguageTag.objects.create(code='en', name='English') + language_fr = LanguageTag.objects.get(code='fr') + + course_run1 = CourseRunFactory.create(language=language_en) + course_run2 = CourseRunFactory.create(language=language_fr) + + program = ProgramFactory.create(language_override=None, courses=[course_run1.course, course_run2.course]) + + expected_languages = {language_en, language_fr} + self.assertEqual(program.active_languages, expected_languages) + + def test_active_languages_with_language_override(self): + """ + Test that active_languages returns the language_override when it is set. + """ + + language_es = LanguageTag.objects.get(code='es') + language_de = LanguageTag.objects.get(code='de') + course_run = CourseRunFactory.create(language=language_de) + program = ProgramFactory.create(language_override=language_es, courses=[course_run.course]) + + expected_languages = {language_es} + self.assertEqual(program.active_languages, expected_languages) + class ProgramSubscriptionTests(TestCase): diff --git a/course_discovery/apps/course_metadata/tests/test_utils.py b/course_discovery/apps/course_metadata/tests/test_utils.py index 9898f953c5..7e110477a9 100644 --- a/course_discovery/apps/course_metadata/tests/test_utils.py +++ b/course_discovery/apps/course_metadata/tests/test_utils.py @@ -1328,7 +1328,7 @@ def test_get_slug_for_course__with_existing_url_slug(self): slug, error = utils.get_slug_for_course(course) assert error is None - slug_end_prefix = f"-{course_count+1}" if course_count else "" + slug_end_prefix = f"-{course_count + 1}" if course_count else "" assert slug == f"learn/{subject.slug}/{organization.name}-{course.title}{slug_end_prefix}" course.set_active_url_slug(slug) diff --git a/course_discovery/apps/course_metadata/utils.py b/course_discovery/apps/course_metadata/utils.py index 4c3549f961..ad3dbd3f95 100644 --- a/course_discovery/apps/course_metadata/utils.py +++ b/course_discovery/apps/course_metadata/utils.py @@ -692,7 +692,6 @@ def get_salesforce_util(partner): class HTML2TextWithLangSpans(html2text.HTML2Text): - # pylint: disable=abstract-method def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/course_discovery/settings/production.py b/course_discovery/settings/production.py index bb2c408853..44609c0bd0 100644 --- a/course_discovery/settings/production.py +++ b/course_discovery/settings/production.py @@ -94,3 +94,10 @@ k.lower(): (v.replace("\\n", "\n") if k.lower() == "private_key" else v) for (k, v) in GOOGLE_SERVICE_ACCOUNT_CREDENTIALS.items() } + +# IMPORTANT: With this enabled, the server must always be behind a proxy that +# strips the header X_FORWARDED_PROTO from client requests. Otherwise, +# a user can fool our server into thinking it was an https connection. +# See https://docs.djangoproject.com/en/5.1/ref/settings/#secure-proxy-ssl-header +# for other warnings. +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') diff --git a/package-lock.json b/package-lock.json index d7b3b6e66d..5e975df072 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "imports-loader": "0.8.0", "jquery": "3.7.1", "mini-css-extract-plugin": "2.9.1", - "sass": "1.77.6", + "sass": "1.78.0", "sass-loader": "12.1.0", "url-loader": "4.1.1", "webpack": "5.94.0", @@ -3545,9 +3545,9 @@ "peer": true }, "node_modules/sass": { - "version": "1.77.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.77.6.tgz", - "integrity": "sha512-ByXE1oLD79GVq9Ht1PeHWCPMPB8XHpBuz1r85oByKHjZY6qV6rWnQovQzXJXuQ/XyE1Oj3iPk3lo28uzaRA2/Q==", + "version": "1.78.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.78.0.tgz", + "integrity": "sha512-AaIqGSrjo5lA2Yg7RvFZrlXDBCp3nV4XP73GrLGvdRWWwk+8H3l0SDvq/5bA4eF+0RFPLuWUk3E+P1U/YqnpsQ==", "dependencies": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", @@ -7346,9 +7346,9 @@ "peer": true }, "sass": { - "version": "1.77.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.77.6.tgz", - "integrity": "sha512-ByXE1oLD79GVq9Ht1PeHWCPMPB8XHpBuz1r85oByKHjZY6qV6rWnQovQzXJXuQ/XyE1Oj3iPk3lo28uzaRA2/Q==", + "version": "1.78.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.78.0.tgz", + "integrity": "sha512-AaIqGSrjo5lA2Yg7RvFZrlXDBCp3nV4XP73GrLGvdRWWwk+8H3l0SDvq/5bA4eF+0RFPLuWUk3E+P1U/YqnpsQ==", "requires": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", diff --git a/package.json b/package.json index 3ad9ee8850..cdaf9254d1 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "file-loader": "6.2.0", "imports-loader": "0.8.0", "jquery": "3.7.1", - "sass": "1.77.6", + "sass": "1.78.0", "sass-loader": "12.1.0", "url-loader": "4.1.1", "webpack": "5.94.0", diff --git a/renovate.json b/renovate.json index 086417c614..200fb2e4bc 100644 --- a/renovate.json +++ b/renovate.json @@ -1,9 +1,6 @@ { "extends": [ "config:base", - ":automergeLinters", - ":automergeTesters", - ":automergeMinor", ":noUnscheduledUpdates", ":semanticCommits" ], diff --git a/requirements/django.txt b/requirements/django.txt index 16bc98be52..64aaf996fb 100644 --- a/requirements/django.txt +++ b/requirements/django.txt @@ -1 +1 @@ -django==4.2.15 +django==4.2.16 diff --git a/requirements/docs.txt b/requirements/docs.txt index 55424655fc..5e107c6982 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/docs.txt requirements/docs.in # -accessible-pygments==0.0.4 +accessible-pygments==0.0.5 # via pydata-sphinx-theme -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx babel==2.16.0 # via @@ -39,8 +39,6 @@ idna==3.8 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==8.4.0 - # via sphinx jinja2==3.1.4 # via sphinx markupsafe==2.1.5 @@ -49,7 +47,7 @@ packaging==24.1 # via # pydata-sphinx-theme # sphinx -pydata-sphinx-theme==0.14.4 +pydata-sphinx-theme==0.15.4 # via sphinx-book-theme pygments==2.18.0 # via @@ -58,8 +56,6 @@ pygments==2.18.0 # sphinx python-dateutil==2.9.0.post0 # via elasticsearch-dsl -pytz==2024.1 - # via babel requests==2.32.3 # via sphinx six==1.16.0 @@ -77,19 +73,19 @@ sphinx==5.3.0 # -r requirements/docs.in # pydata-sphinx-theme # sphinx-book-theme -sphinx-book-theme==1.0.1 +sphinx-book-theme==1.1.3 # via -r requirements/docs.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx typing-extensions==4.12.2 # via pydata-sphinx-theme @@ -97,5 +93,3 @@ urllib3==1.26.20 # via # elasticsearch # requests -zipp==3.20.1 - # via importlib-metadata diff --git a/requirements/local.txt b/requirements/local.txt index 843fd28535..8ab0ceedc9 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/local.txt requirements/local.in # -accessible-pygments==0.0.4 +accessible-pygments==0.0.5 # via pydata-sphinx-theme aiohappyeyeballs==2.4.0 # via aiohttp @@ -12,7 +12,7 @@ aiohttp==3.10.5 # via openai aiosignal==1.3.1 # via aiohttp -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx algoliasearch==1.20.0 # via @@ -36,10 +36,6 @@ astroid==3.2.4 # via # pylint # pylint-celery -async-timeout==4.0.3 - # via - # aiohttp - # redis attrs==21.4.0 # via # aiohttp @@ -56,14 +52,6 @@ babel==2.16.0 # sphinx backoff==2.2.1 # via -r requirements/base.in -backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # celery - # django - # django-ses - # djangorestframework - # kombu beautifulsoup4==4.12.3 # via # -r requirements/base.in @@ -76,9 +64,9 @@ boltons==21.0.0 # face # glom # semgrep -boto3==1.35.10 +boto3==1.35.12 # via django-ses -botocore==1.35.10 +botocore==1.35.12 # via # boto3 # s3transfer @@ -104,7 +92,7 @@ certifi==2024.8.30 # requests # selenium # snowflake-connector-python -cffi==1.17.0 +cffi==1.17.1 # via # cairocffi # cryptography @@ -152,7 +140,7 @@ coverage[toml]==7.6.1 # via # -r requirements/test.in # pytest-cov -cryptography==43.0.0 +cryptography==43.0.1 # via # pyjwt # pyopenssl @@ -280,7 +268,7 @@ django-libsass==0.9 # via -r requirements/base.in django-localflavor==4.0 # via -r requirements/base.in -django-model-utils==4.5.1 +django-model-utils==5.0.0 # via taxonomy-connector django-multi-email-field==0.7.0 # via -r requirements/base.in @@ -384,7 +372,7 @@ edx-django-utils==5.15.0 # getsmarter-api-clients # openedx-events # taxonomy-connector -edx-drf-extensions==10.3.0 +edx-drf-extensions==10.4.0 # via -r requirements/base.in edx-event-bus-kafka==5.8.1 # via -r requirements/base.in @@ -393,8 +381,10 @@ edx-event-bus-redis==0.5.0 edx-i18n-tools==1.6.2 # via -r requirements/local.in edx-lint==5.3.7 - # via -r requirements/test.in -edx-opaque-keys[django]==2.10.0 + # via + # -c requirements/constraints.txt + # -r requirements/test.in +edx-opaque-keys[django]==2.11.0 # via # -r requirements/base.in # edx-ccx-keys @@ -422,18 +412,13 @@ elasticsearch-dsl==7.4.1 # -r requirements/base.in # django-elasticsearch-dsl # django-elasticsearch-dsl-drf -exceptiongroup==1.2.2 - # via - # pytest - # trio - # trio-websocket execnet==2.1.1 # via pytest-xdist face==22.0.0 # via glom factory-boy==3.3.1 # via -r requirements/test.in -faker==28.1.0 +faker==28.4.1 # via factory-boy fastavro==1.9.5 # via openedx-events @@ -490,12 +475,7 @@ idna==3.8 imagesize==1.4.1 # via sphinx importlib-metadata==8.4.0 - # via - # -r requirements/base.in - # markdown - # sphinx -importlib-resources==6.4.4 - # via pycountry + # via -r requirements/base.in inflection==0.5.1 # via drf-yasg iniconfig==2.0.0 @@ -625,7 +605,7 @@ pycountry==24.6.1 # via -r requirements/base.in pycparser==2.22 # via cffi -pydata-sphinx-theme==0.14.4 +pydata-sphinx-theme==0.15.4 # via sphinx-book-theme pygments==2.18.0 # via @@ -685,7 +665,7 @@ pytest==8.3.2 # pytest-xdist pytest-cov==5.0.0 # via -r requirements/test.in -pytest-django==4.8.0 +pytest-django==4.9.0 # via -r requirements/test.in pytest-responses==0.5.1 # via -r requirements/test.in @@ -721,7 +701,6 @@ python3-openid==3.2.0 pytz==2024.1 # via # -r requirements/base.in - # babel # drf-yasg # getsmarter-api-clients # snowflake-connector-python @@ -840,19 +819,19 @@ sphinx==5.3.0 # -r requirements/docs.in # pydata-sphinx-theme # sphinx-book-theme -sphinx-book-theme==1.0.1 +sphinx-book-theme==1.1.3 # via -r requirements/docs.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlparse==0.5.1 # via @@ -873,13 +852,6 @@ tinycss2==1.3.0 # via # cairosvg # cssselect2 -tomli==2.0.1 - # via - # coverage - # pylint - # pyproject-api - # pytest - # tox tomlkit==0.13.2 # via # pylint @@ -898,21 +870,14 @@ trio-websocket==0.11.1 # via selenium typing-extensions==4.12.2 # via - # asgiref - # astroid # django-countries - # django-solo # edx-opaque-keys - # kombu # pydata-sphinx-theme - # pylint # semgrep # simple-salesforce # snowflake-connector-python tzdata==2024.1 - # via - # backports-zoneinfo - # celery + # via celery ujson==5.10.0 # via python-lsp-jsonrpc unicodecsv==0.14.1 @@ -929,7 +894,6 @@ urllib3[socks]==1.26.20 # responses # selenium # semgrep - # snowflake-connector-python vine==5.1.0 # via # amqp @@ -951,14 +915,12 @@ wsproto==1.2.0 # via trio-websocket xss-utils==0.6.0 # via -r requirements/base.in -yarl==1.9.7 +yarl==1.9.11 # via aiohttp zeep==4.2.1 # via simple-salesforce zipp==3.20.1 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/pip.txt b/requirements/pip.txt index f3fa6b47d5..f72a524c92 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --allow-unsafe --output-file=requirements/pip.txt requirements/pip.in @@ -10,5 +10,5 @@ wheel==0.44.0 # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via -r requirements/pip.in -setuptools==74.0.0 +setuptools==74.1.2 # via -r requirements/pip.in diff --git a/requirements/pip_tools.txt b/requirements/pip_tools.txt index ffa11c3343..75568e9e4c 100644 --- a/requirements/pip_tools.txt +++ b/requirements/pip_tools.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/pip_tools.txt requirements/pip_tools.in @@ -8,8 +8,6 @@ build==1.2.1 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==8.4.0 - # via build packaging==24.1 # via build pip-tools==7.4.1 @@ -18,14 +16,8 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -tomli==2.0.1 - # via - # build - # pip-tools wheel==0.44.0 # via pip-tools -zipp==3.20.1 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/production.txt b/requirements/production.txt index d19cd544bf..dcdb7924ed 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/production.txt requirements/production.in @@ -28,10 +28,6 @@ asgiref==3.8.1 # django-countries asn1crypto==1.5.1 # via snowflake-connector-python -async-timeout==4.0.3 - # via - # aiohttp - # redis attrs==24.2.0 # via # aiohttp @@ -39,23 +35,15 @@ attrs==24.2.0 # zeep backoff==2.2.1 # via -r requirements/base.in -backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # celery - # django - # django-ses - # djangorestframework - # kombu beautifulsoup4==4.12.3 # via # -r requirements/base.in # taxonomy-connector billiard==4.2.0 # via celery -boto3==1.35.10 +boto3==1.35.12 # via django-ses -botocore==1.35.10 +botocore==1.35.12 # via # boto3 # s3transfer @@ -77,7 +65,7 @@ certifi==2024.8.30 # elasticsearch # requests # snowflake-connector-python -cffi==1.17.0 +cffi==1.17.1 # via # cairocffi # cryptography @@ -105,7 +93,7 @@ code-annotations==1.8.0 # via edx-toggles contentful==2.2.0 # via -r requirements/base.in -cryptography==43.0.0 +cryptography==43.0.1 # via # pyjwt # pyopenssl @@ -119,7 +107,7 @@ defusedxml==0.7.1 # djangorestframework-xml # python3-openid # social-auth-core -django==4.2.15 +django==4.2.16 # via # -c requirements/common_constraints.txt # -c requirements/constraints.txt @@ -218,7 +206,7 @@ django-libsass==0.9 # via -r requirements/base.in django-localflavor==4.0 # via -r requirements/base.in -django-model-utils==4.5.1 +django-model-utils==5.0.0 # via taxonomy-connector django-multi-email-field==0.7.0 # via -r requirements/base.in @@ -320,13 +308,13 @@ edx-django-utils==5.15.0 # getsmarter-api-clients # openedx-events # taxonomy-connector -edx-drf-extensions==10.3.0 +edx-drf-extensions==10.4.0 # via -r requirements/base.in edx-event-bus-kafka==5.8.1 # via -r requirements/base.in edx-event-bus-redis==0.5.0 # via -r requirements/base.in -edx-opaque-keys[django]==2.10.0 +edx-opaque-keys[django]==2.11.0 # via # -r requirements/base.in # edx-ccx-keys @@ -403,11 +391,7 @@ idna==3.8 # snowflake-connector-python # yarl importlib-metadata==8.4.0 - # via - # -r requirements/base.in - # markdown -importlib-resources==6.4.4 - # via pycountry + # via -r requirements/base.in inflection==0.5.1 # via drf-yasg isodate==0.6.1 @@ -646,17 +630,12 @@ tqdm==4.66.5 # via openai typing-extensions==4.12.2 # via - # asgiref # django-countries - # django-solo # edx-opaque-keys - # kombu # simple-salesforce # snowflake-connector-python tzdata==2024.1 - # via - # backports-zoneinfo - # celery + # via celery unicodecsv==0.14.1 # via -r requirements/base.in uritemplate==4.1.1 @@ -668,7 +647,6 @@ urllib3==1.26.20 # botocore # elasticsearch # requests - # snowflake-connector-python vine==5.1.0 # via # amqp @@ -684,14 +662,12 @@ webencodings==0.5.1 # tinycss2 xss-utils==0.6.0 # via -r requirements/base.in -yarl==1.9.7 +yarl==1.9.11 # via aiohttp zeep==4.2.1 # via simple-salesforce zipp==3.20.1 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata zope-event==5.0 # via gevent zope-interface==7.0.3 diff --git a/tox.ini b/tox.ini index f0562122ba..f164b27827 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38, 311, 312}-django{42} +envlist = py{312}-django{42} skipsdist=true [pytest]