Merge branch 'master' into huniafatima/remove-dockerfile-setup

.annotation_safe_list.yml

@@ -142,8 +142,6 @@ workflow.AssessmentWorkflowStep:
 # Via edx-celeryutils
 celery_utils.ChordData:
   ".. no_pii:": "No PII"
-celery_utils.FailedTask:
-  ".. no_pii:": "No PII"
 
 # Via completion XBlock
 completion.BlockCompletion:

.eslintignore

@@ -20,9 +20,21 @@ test_root/staticfiles
 common/static/xmodule
 
 
-# Symlinks into xmodule/js
+# Various intra-repo symlinks that we've added over the years to duct-tape the JS build together.
+# Ignore them so that we're not double-counting these violations.
+cms/static/edx-ui-toolkit
 cms/static/xmodule_js
+lms/static/common
+lms/static/course_bookmarks
+lms/static/course_experience
+lms/static/course_search
+lms/static/discussion
+lms/static/edx-ui-toolkit
+lms/static/learner_profile
+lms/static/support
+lms/static/teams
 lms/static/xmodule_js
+xmodule/js/common_static
 
 
 # Mako templates that generate .js files

.github/CODEOWNERS

@@ -22,7 +22,7 @@ openedx/core/djangoapps/enrollments/                                @openedx/2U-
 openedx/core/djangoapps/heartbeat/
 openedx/core/djangoapps/oauth_dispatch
 openedx/core/djangoapps/user_api/                                   @openedx/2U-aperture
-openedx/core/djangoapps/user_authn/                                 @openedx/2U-vanguards                                 
+openedx/core/djangoapps/user_authn/                                 @openedx/2U-infinity
 openedx/core/djangoapps/verified_track_content/                     @openedx/2u-infinity
 openedx/features/course_experience/
 xmodule/

.github/renovate.json5

@@ -32,7 +32,11 @@
   // When adding an ignoreDep, please include a reason and a public link that we can use to follow up and ensure
   // that the ignoreDep is removed.
   // This can be done as a comment within the ignoreDeps list.
-  "ignoreDeps": [],
+  "ignoreDeps": [
+    // karma-spec-reporter>0.20.0 does not seem compatible with our super-old 2016 Karma version (0.13.22).
+    // Ticket link: None, as upgrading Karma does not strike as worth the benefit.
+    "karma-spec-reporter"
+  ],
   "timezone": "America/New_York",
   "prConcurrentLimit": 3,
   "enabledManagers": ["npm"]

.github/workflows/js-tests.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        node-version: [18, 20]
+        node-version: [20]
         python-version:
           - "3.11"
 
@@ -28,7 +28,7 @@ jobs:
           node-version: ${{ matrix.node-version }}
 
       - name: Setup npm
-        run: npm i -g npm@10.5.x
+        run: npm i -g npm@10.7.x
 
       - name: Install Firefox 123.0
         run: |
@@ -64,13 +64,13 @@ jobs:
           make base-requirements
 
       - uses: c-hive/gha-npm-cache@v1
+
+      - name: Install npm
+        run: npm ci  
+
       - name: Run JS Tests
-        env:
-          TEST_SUITE: js-unit
-          SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
         run: |
-          npm install -g jest
-          xvfb-run --auto-servernum ./scripts/all-tests.sh
+          npm run test
 
       - name: Save Job Artifacts
         uses: actions/upload-artifact@v4

.github/workflows/publish-ci-docker-image.yml

@@ -1,42 +0,0 @@
-name: Push CI Runner Docker Image
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 1 * * 3"
-
-jobs:
-  push:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # This has to happen after checkout in order for gh to work.
-      - name: "Cancel scheduled job on forks"
-        if: github.repository != 'openedx/edx-platform' && github.event_name == 'schedule'
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        run: |
-          gh run cancel "${{ github.run_id }}"
-          gh run watch "${{ github.run_id }}"
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.TOOLS_EDX_ECR_USER_AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-1
-
-      - name: Log in to ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Build, tag, and push image to Amazon ECR
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: actions-runner
-          IMAGE_TAG: latest
-        run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f scripts/ci-runner.Dockerfile .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG

.github/workflows/pylint-checks.yml

@@ -22,7 +22,7 @@ jobs:
           - module-name: openedx-2
             path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/"
           - module-name: common
-            path: "common pavelib"
+            path: "common"
           - module-name: cms
             path: "cms"
           - module-name: xmodule

.github/workflows/quality-checks.yml

@@ -60,16 +60,29 @@ jobs:
           PIP_SRC: ${{ runner.temp }}
         run: |
           make test-requirements
-
+      
+      - name: Install npm
+        env:
+          PIP_SRC: ${{ runner.temp }}
+        run: npm ci  
+
+      - name: Install python packages
+        env:
+          PIP_SRC: ${{ runner.temp }}
+        run: |
+            pip install -e .
+      
       - name: Run Quality Tests
         env:
-          TEST_SUITE: quality
-          SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
           PIP_SRC: ${{ runner.temp }}
           TARGET_BRANCH: ${{ github.base_ref }}
         run: |
-          ./scripts/all-tests.sh
-
+            make pycodestyle
+            npm run lint
+            make xsslint
+            make pii_check
+            make check_keywords
+          
       - name: Save Job Artifacts
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/static-assets-check.yml

@@ -15,8 +15,8 @@ jobs:
         os: [ubuntu-24.04]
         python-version:
           - "3.11"
-        node-version: [18, 20]
-        npm-version: [10.5.x]
+        node-version: [20]
+        npm-version: [10.7.x]
         mongo-version:
           - "7.0"
 

.github/workflows/unit-test-shards.json

@@ -239,7 +239,6 @@
       "cms/djangoapps/cms_user_tasks/",
       "cms/djangoapps/course_creators/",
       "cms/djangoapps/export_course_metadata/",
-      "cms/djangoapps/maintenance/",
       "cms/djangoapps/models/",
       "cms/djangoapps/pipeline_js/",
       "cms/djangoapps/xblock_config/",
@@ -256,15 +255,13 @@
   "common-with-lms": {
     "settings": "lms.envs.test",
     "paths": [
-      "common/djangoapps/",
-      "pavelib/"
+      "common/djangoapps/"
     ]
   },
   "common-with-cms": {
     "settings": "cms.envs.test",
     "paths": [
-      "common/djangoapps/",
-      "pavelib/"
+      "common/djangoapps/"
     ]
   },
   "xmodule-with-lms": {

.github/workflows/unit-tests.yml

@@ -71,29 +71,7 @@ jobs:
 
       - name: install system requirements
         run: |
-          sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx openssl
-
-      # This is needed until the ENABLE_BLAKE2B_HASHING can be removed and we
-      # can stop using MD4 by default.
-      - name: enable md4 hashing in libssl
-        run: |
-          cat <<EOF | sudo tee /etc/ssl/openssl.cnf
-          # Use this in order to automatically load providers.
-          openssl_conf = openssl_init
-
-          [openssl_init]
-          providers = provider_sect
-
-          [provider_sect]
-          default = default_sect
-          legacy = legacy_sect
-
-          [default_sect]
-          activate = 1
-
-          [legacy_sect]
-          activate = 1
-          EOF
+          sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx
 
       - name: Start MongoDB
         uses: supercharge/[email protected]
@@ -180,7 +158,7 @@ jobs:
         shell: bash
         run: |
           echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
-          echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ pavelib/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+          echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
 
       - name: get GHA unit test paths
         shell: bash

.nvmrc

@@ -1 +1 @@
-18
+20

.pii_annotations.yml

@@ -1,7 +1,7 @@
 source_path: ./
 report_path: pii_report
 safelist_path: .annotation_safe_list.yml
-coverage_target: 94.5
+coverage_target: 85.3
 # See OEP-30 for more information on these values and what they mean:
 # https://open-edx-proposals.readthedocs.io/en/latest/oep-0030-arch-pii-markup-and-auditing.html#docstring-annotations
 annotations:

Makefile

@@ -1,8 +1,7 @@
 # Do things in edx-platform
 .PHONY: base-requirements check-types clean \
   compile-requirements detect_changed_source_translations dev-requirements \
-  docker_auth docker_build docker_tag_build_push_lms docker_tag_build_push_lms_dev \
-  docker_tag_build_push_cms docker_tag_build_push_cms_dev docs extract_translations \
+  docs extract_translations \
   guides help lint-imports local-requirements migrate migrate-lms migrate-cms \
   pre-requirements pull pull_xblock_translations pull_translations push_translations \
   requirements shell swagger \
@@ -67,9 +66,6 @@ pull_translations: clean_translations  ## pull translations via atlas
 detect_changed_source_translations: ## check if translation files are up-to-date
 	i18n_tool changed
 
-pull: ## update the Docker image used by "make shell"
-	docker pull edxops/edxapp:latest
-
 pre-requirements: ## install Python requirements for running pip-tools
 	pip install -r requirements/pip.txt
 	pip install -r requirements/pip-tools.txt
@@ -94,17 +90,9 @@ test-requirements: pre-requirements
 
 requirements: dev-requirements ## install development environment requirements
 
-shell: ## launch a bash shell in a Docker container with all edx-platform dependencies installed
-	docker run -it -e "NO_PYTHON_UNINSTALL=1" -e "PIP_INDEX_URL=https://pypi.python.org/simple" -e TERM \
-	-v `pwd`:/edx/app/edxapp/edx-platform:cached \
-	-v edxapp_lms_assets:/edx/var/edxapp/staticfiles/ \
-	-v edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules \
-	edxops/edxapp:latest /edx/app/edxapp/devstack.sh open
-
 # Order is very important in this list: files must appear after everything they include!
 REQ_FILES = \
 	requirements/edx/coverage \
-	requirements/edx/paver \
 	requirements/edx-sandbox/base \
 	requirements/edx/base \
 	requirements/edx/doc \
@@ -164,27 +152,6 @@ upgrade-package: ## update just one package to the latest usable release
 check-types: ## run static type-checking tests
 	mypy
 
-docker_auth:
-	echo "$$DOCKERHUB_PASSWORD" | docker login -u "$$DOCKERHUB_USERNAME" --password-stdin
-
-docker_build: docker_auth
-	DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target development -t openedx/lms-dev
-	DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target production -t openedx/lms
-	DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target development -t openedx/cms-dev
-	DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target production -t openedx/cms
-
-docker_tag_build_push_lms: docker_auth
-	docker buildx build -t openedx/lms:latest -t openedx/lms:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target production --push .
-
-docker_tag_build_push_lms_dev: docker_auth
-	docker buildx build -t openedx/lms-dev:latest -t openedx/lms-dev:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target development --push .
-
-docker_tag_build_push_cms: docker_auth
-	docker buildx build -t openedx/cms:latest -t openedx/cms:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target production --push .
-
-docker_tag_build_push_cms_dev: docker_auth
-	docker buildx build -t openedx/cms-dev:latest -t openedx/cms-dev:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target development --push .
-
 lint-imports:
 	lint-imports
 
@@ -204,3 +171,37 @@ migrate: migrate-lms migrate-cms
 # Part of https://github.com/openedx/wg-developer-experience/issues/136
 ubuntu-requirements: ## Install ubuntu 22.04 system packages needed for `pip install` to work on ubuntu.
 	sudo apt install libmysqlclient-dev libxmlsec1-dev
+
+xsslint: ## check xss for quality issuest
+	python scripts/xsslint/xss_linter.py \
+	--rule-totals \
+	--config=scripts.xsslint_config \
+	--thresholds=scripts/xsslint_thresholds.json
+
+pycodestyle: ## check python files for quality issues
+	pycodestyle .
+
+## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved
+pii_check: ## check django models for pii annotations
+	DJANGO_SETTINGS_MODULE=cms.envs.test \
+	code_annotations django_find_annotations \
+		--config_file .pii_annotations.yml \
+		--app_name cms \
+		--coverage \
+		--lint
+
+	DJANGO_SETTINGS_MODULE=lms.envs.test \
+	code_annotations django_find_annotations \
+		--config_file .pii_annotations.yml \
+		--app_name lms \
+		--coverage \
+		--lint
+
+check_keywords: ## check django models for reserve keywords
+	DJANGO_SETTINGS_MODULE=cms.envs.test \
+	python manage.py cms check_reserved_keywords \
+	--override_file db_keyword_overrides.yml
+
+	DJANGO_SETTINGS_MODULE=lms.envs.test \
+	python manage.py lms check_reserved_keywords \
+	--override_file db_keyword_overrides.yml

cms/djangoapps/contentstore/helpers.py

@@ -7,6 +7,7 @@
 import urllib
 from lxml import etree
 from mimetypes import guess_type
+import re
 
 from attrs import frozen, Factory
 from django.conf import settings
@@ -447,6 +448,10 @@ def _import_xml_node_to_parent(
         temp_xblock = xblock_class.parse_xml(node_without_children, runtime, keys)
         child_nodes = list(node)
 
+    if issubclass(xblock_class, XmlMixin) and "x-is-pointer-node" in getattr(temp_xblock, "data", ""):
+        # Undo the "pointer node" hack if needed (e.g. for capa problems)
+        temp_xblock.data = re.sub(r'([^>]+) x-is-pointer-node="no"', r'\1', temp_xblock.data, count=1)
+
     # Restore the original id_generator
     runtime.id_generator = original_id_generator