diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d5dcd126d6..9ff351eeb9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,16 +70,9 @@ jobs: ##### Create release on GitHub - name: Create or update GitHub release - # I wish there was an `--update` option to the `gh release create` command, but - # there isn't. - # https://cli.github.com/manual/gh_release_create - run: | - make release-description | tee release_description.md - export GH_ARGS="${{ github.ref_name }} --notes-file=release_description.md" - echo "gh args: '$GH_ARGS'" - ${{ env.gh_bin }} release create $GH_ARGS || ${{ env.gh_bin }} release edit $GH_ARGS + run: scriv github-release env: - GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} - name: Upload release asset to GitHub run: | export FILENAME="tutor-$(uname -s)_$(uname -m)" diff --git a/CHANGELOG.md b/CHANGELOG.md index c06895e3c7..0fab387728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,15 @@ instructions, because git commits are used to generate release notes: + +## v15.2.0 (2023-01-19) + +- 💥[Bugfix] Fix "example.com" links in registration emails. This is a breaking change for platforms that have modified the "id" field of the LMS site object in the database. These platforms should set `SITE_ID=1` in the common settings via a plugin. (by @regisb) +- [Bugfix] Running `tutor k8s upgrade --from=maple` won't apply and won't wait for the MySQL deployment to be ready if `RUN_MYSQL: false` (When you host your MySQL somewhere else like RDS) (by @CodeWithEmad) +- [Bugfix] Fix HTML component editing in studio by cherry-picking [upstream fix](https://github.com/openedx/edx-platform/pull/31500). (by @regisb) +- [Improvement] Changes annotations from `typing` to use built-in generic types from `__future__.annotations` (by @Carlos-Muniz) +- [Improvement] Resolve `CORS_ORIGIN_WHITELIST` warnings that pollute the LMS and CMS logs. As far as we know they were not causing any issue, apart from being a nuisance. (by @regisb) + ## v15.1.0 (2022-12-13) diff --git a/Makefile b/Makefile index fdd9a1cb98..9dbdeb7b34 100644 --- a/Makefile +++ b/Makefile @@ -54,12 +54,11 @@ format: ## Format code automatically isort: ## Sort imports. This target is not mandatory because the output may be incompatible with black formatting. Provided for convenience purposes. isort --skip=templates ${SRC_DIRS} -bootstrap-dev: ## Install dev requirements - pip install . - pip install -r requirements/dev.txt +changelog-entry: ## Create a new changelog entry + scriv create -bootstrap-dev-plugins: bootstrap-dev ## Install dev requirements and all supported plugins - pip install -r requirements/plugins.txt +changelog: ## Collect changelog entries in the CHANGELOG.md file + scriv collect ###### Code coverage @@ -78,23 +77,17 @@ coverage-html: coverage-report ## Generate HTML report for the code coverage coverage-browse-report: coverage-html ## Open the HTML report in the browser sensible-browser htmlcov/index.html -###### Deployment +###### Continuous integration tasks bundle: ## Bundle the tutor package in a single "dist/tutor" executable pyinstaller tutor.spec +bootstrap-dev: ## Install dev requirements + pip install . + pip install -r requirements/dev.txt -changelog-entry: ## Create a new changelog entry - scriv create - -changelog: ## Collect changelog entries in the CHANGELOG.md file - scriv collect - -release-description: ## Write the current release description to a file - @sed "s/TUTOR_VERSION/v$(shell make version)/g" docs/_release_description.md - @git log -1 --pretty=format:%b - -###### Continuous integration tasks +bootstrap-dev-plugins: bootstrap-dev ## Install dev requirements and all supported plugins + pip install -r requirements/plugins.txt pull-base-images: # Manually pull base images docker image pull docker.io/ubuntu:20.04 diff --git a/changelog.d/20230109_100009_codewithemad_upgrade_mysql_issue.md b/changelog.d/20230109_100009_codewithemad_upgrade_mysql_issue.md deleted file mode 100644 index 279deb9c9b..0000000000 --- a/changelog.d/20230109_100009_codewithemad_upgrade_mysql_issue.md +++ /dev/null @@ -1 +0,0 @@ -[Bugfix] Running `tutor k8s upgrade --from=maple` won't apply and won't wait for the MySQL deployment to be ready if `RUN_MYSQL: false` (When you host your MySQL somewhere else like RDS) (by @CodeWithEmad) \ No newline at end of file diff --git a/changelog.d/20230110_165850_kdmc_global_node_modules.md b/changelog.d/20230110_165850_kdmc_global_node_modules.md new file mode 100644 index 0000000000..9be7cc1e47 --- /dev/null +++ b/changelog.d/20230110_165850_kdmc_global_node_modules.md @@ -0,0 +1,10 @@ + + +- [Improvement] Remove the need to run ``npm install`` when setting up an edx-platform development environment. (by @kdmccormick) diff --git a/changelog.d/scriv.ini b/changelog.d/scriv.ini index 9236140917..6333f3ac5c 100644 --- a/changelog.d/scriv.ini +++ b/changelog.d/scriv.ini @@ -3,5 +3,6 @@ version = literal: tutor/__about__.py: __version__ categories = format = md md_header_level = 2 -new_fragment_template = file: scriv/new_fragment.${config:format}.j2 -entry_title_template = file: scriv/entry_title.${config:format}.j2 +new_fragment_template = file: changelog.d/scriv/new_fragment.${config:format}.j2 +entry_title_template = file: changelog.d/scriv/entry_title.${config:format}.j2 +ghrel_template = file: changelog.d/scriv/github_release.${config:format}.j2 diff --git a/docs/_release_description.md b/changelog.d/scriv/github_release.md.j2 similarity index 73% rename from docs/_release_description.md rename to changelog.d/scriv/github_release.md.j2 index 0cf51b6864..9f07e98250 100644 --- a/docs/_release_description.md +++ b/changelog.d/scriv/github_release.md.j2 @@ -1,13 +1,14 @@ Install this version from pip with: - pip install "tutor[full]==TUTOR_VERSION" + pip install "tutor[full]=={{ version }}" Or download the compiled binaries: - sudo curl -L "https://github.com/overhangio/tutor/releases/download/TUTOR_VERSION/tutor-$(uname -s)_$(uname -m)" -o /usr/local/bin/tutor + sudo curl -L "https://github.com/overhangio/tutor/releases/download/{{ version }}/tutor-$(uname -s)_$(uname -m)" -o /usr/local/bin/tutor sudo chmod 0755 /usr/local/bin/tutor See the [installation docs](https://docs.tutor.overhang.io/install.html) for more installation options and instructions. ## Changes +{{ body }} diff --git a/docs/conf.py b/docs/conf.py index 068538f186..1f17b4af85 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,6 +38,12 @@ ("py:class", "tutor.hooks.filters.P"), ("py:class", "tutor.hooks.filters.T"), ("py:class", "tutor.hooks.actions.P"), + ("py:class", "P"), + ("py:class", "P.args"), + ("py:class", "P.kwargs"), + ("py:class", "T"), + ("py:class", "t.Any"), + ("py:class", "t.Optional"), ] # -- Sphinx-Click configuration diff --git a/requirements/base.txt b/requirements/base.txt index c389db18b2..9a0c9dfbe7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements/base.in # @@ -62,7 +62,9 @@ six==1.16.0 tomli==2.0.1 # via mypy typing-extensions==4.4.0 - # via mypy + # via + # -r requirements/base.in + # mypy urllib3==1.26.13 # via # kubernetes diff --git a/requirements/dev.txt b/requirements/dev.txt index 586daf327f..0cafd87478 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements/dev.in # @@ -172,7 +172,7 @@ rsa==4.9 # via # -r requirements/base.txt # google-auth -scriv==1.0.0 +scriv==1.1.0 # via -r requirements/dev.in secretstorage==3.3.3 # via keyring @@ -204,7 +204,11 @@ types-setuptools==65.6.0.2 typing-extensions==4.4.0 # via # -r requirements/base.txt + # astroid + # black # mypy + # pylint + # rich urllib3==1.26.13 # via # -r requirements/base.txt diff --git a/requirements/docs.txt b/requirements/docs.txt index 0b4da19d22..afd1b36684 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements/docs.in # @@ -42,6 +42,8 @@ idna==3.4 # requests imagesize==1.4.1 # via sphinx +importlib-metadata==6.0.0 + # via sphinx jinja2==3.1.2 # via # -r requirements/base.txt @@ -147,6 +149,8 @@ websocket-client==1.4.2 # via # -r requirements/base.txt # kubernetes +zipp==3.11.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/tests/commands/base.py b/tests/commands/base.py index e68dc2932d..868e00d75b 100644 --- a/tests/commands/base.py +++ b/tests/commands/base.py @@ -1,4 +1,4 @@ -import typing as t +from __future__ import annotations import click.testing @@ -12,13 +12,13 @@ class TestCommandMixin: """ @staticmethod - def invoke(args: t.List[str]) -> click.testing.Result: + def invoke(args: list[str]) -> click.testing.Result: with temporary_root() as root: return TestCommandMixin.invoke_in_root(root, args) @staticmethod def invoke_in_root( - root: str, args: t.List[str], catch_exceptions: bool = True + root: str, args: list[str], catch_exceptions: bool = True ) -> click.testing.Result: """ Use this method for commands that all need to run in the same root: diff --git a/tests/commands/test_compose.py b/tests/commands/test_compose.py index aeb91e0888..9eb505e967 100644 --- a/tests/commands/test_compose.py +++ b/tests/commands/test_compose.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t import unittest from io import StringIO @@ -62,9 +63,9 @@ def test_compose_local_tmp_generation(self, _mock_stdout: StringIO) -> None: # Mount volumes compose.mount_tmp_volumes(mount_args, LocalContext("")) - compose_file: t.Dict[str, t.Any] = hooks.Filters.COMPOSE_LOCAL_TMP.apply({}) - actual_services: t.Dict[str, t.Any] = compose_file["services"] - expected_services: t.Dict[str, t.Any] = { + compose_file: dict[str, t.Any] = hooks.Filters.COMPOSE_LOCAL_TMP.apply({}) + actual_services: dict[str, t.Any] = compose_file["services"] + expected_services: dict[str, t.Any] = { "cms": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, "cms-worker": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, "lms": { @@ -78,11 +79,9 @@ def test_compose_local_tmp_generation(self, _mock_stdout: StringIO) -> None: } self.assertEqual(actual_services, expected_services) - compose_jobs_file: t.Dict[ - str, t.Any - ] = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP.apply({}) - actual_jobs_services: t.Dict[str, t.Any] = compose_jobs_file["services"] - expected_jobs_services: t.Dict[str, t.Any] = { + compose_jobs_file = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP.apply({}) + actual_jobs_services = compose_jobs_file["services"] + expected_jobs_services: dict[str, t.Any] = { "cms-job": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, "lms-job": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, } diff --git a/tests/hooks/test_filters.py b/tests/hooks/test_filters.py index 796ff72191..4c23310d30 100644 --- a/tests/hooks/test_filters.py +++ b/tests/hooks/test_filters.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t import unittest @@ -23,14 +24,14 @@ def filter1(value: int) -> int: def test_add_items(self) -> None: @hooks.filters.add("tests:add-sheeps") - def filter1(sheeps: t.List[int]) -> t.List[int]: + def filter1(sheeps: list[int]) -> list[int]: return sheeps + [0] hooks.filters.add_item("tests:add-sheeps", 1) hooks.filters.add_item("tests:add-sheeps", 2) hooks.filters.add_items("tests:add-sheeps", [3, 4]) - sheeps: t.List[int] = hooks.filters.apply("tests:add-sheeps", []) + sheeps: list[int] = hooks.filters.apply("tests:add-sheeps", []) self.assertEqual([0, 1, 2, 3, 4], sheeps) def test_filter_callbacks(self) -> None: diff --git a/tests/test_plugins_v0.py b/tests/test_plugins_v0.py index 85dddb9c10..311cac6477 100644 --- a/tests/test_plugins_v0.py +++ b/tests/test_plugins_v0.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t from unittest.mock import patch @@ -197,9 +198,7 @@ def test_dict_plugin(self) -> None: {"name": "myplugin", "config": {"set": {"KEY": "value"}}, "version": "0.1"} ) plugins.load("myplugin") - overriden_items: t.List[ - t.Tuple[str, t.Any] - ] = hooks.Filters.CONFIG_OVERRIDES.apply([]) + overriden_items = hooks.Filters.CONFIG_OVERRIDES.apply([]) versions = list(plugins.iter_info()) self.assertEqual("myplugin", plugin.name) self.assertEqual([("myplugin", "0.1")], versions) diff --git a/tutor/__about__.py b/tutor/__about__.py index a8254bd4b3..25ce44862b 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "15.1.0" +__version__ = "15.2.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index 758ff1ab0b..98da421013 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -1,3 +1,4 @@ +from __future__ import annotations import sys import typing as t @@ -61,7 +62,7 @@ def ensure_plugins_enabled(cls, ctx: click.Context) -> None: hooks.Actions.PROJECT_ROOT_READY.do(ctx.params["root"]) cls.IS_ROOT_READY = True - def list_commands(self, ctx: click.Context) -> t.List[str]: + def list_commands(self, ctx: click.Context) -> list[str]: """ This is run in the following cases: - shell autocompletion: tutor diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index a1e24f7bd0..921ff3cef7 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import re import typing as t @@ -16,15 +17,15 @@ from tutor.tasks import BaseComposeTaskRunner from tutor.types import Config -COMPOSE_FILTER_TYPE: TypeAlias = "hooks.filters.Filter[t.Dict[str, t.Any], []]" +COMPOSE_FILTER_TYPE: TypeAlias = "hooks.filters.Filter[dict[str, t.Any], []]" class ComposeTaskRunner(BaseComposeTaskRunner): def __init__(self, root: str, config: Config): super().__init__(root, config) self.project_name = "" - self.docker_compose_files: t.List[str] = [] - self.docker_compose_job_files: t.List[str] = [] + self.docker_compose_files: list[str] = [] + self.docker_compose_job_files: list[str] = [] def docker_compose(self, *command: str) -> int: """ @@ -55,7 +56,7 @@ def update_docker_compose_tmp( Update the contents of the docker-compose.tmp.yml and docker-compose.jobs.tmp.yml files, which are generated at runtime. """ - compose_base: t.Dict[str, t.Any] = { + compose_base: dict[str, t.Any] = { "version": "{{ DOCKER_COMPOSE_VERSION }}", "services": {}, } @@ -134,11 +135,11 @@ def convert( value: str, param: t.Optional["click.Parameter"], ctx: t.Optional[click.Context], - ) -> t.List["MountType"]: + ) -> list["MountType"]: mounts = self.convert_explicit_form(value) or self.convert_implicit_form(value) return mounts - def convert_explicit_form(self, value: str) -> t.List["MountParam.MountType"]: + def convert_explicit_form(self, value: str) -> list["MountParam.MountType"]: """ Argument is of the form "containers:/host/path:/container/path". """ @@ -146,8 +147,8 @@ def convert_explicit_form(self, value: str) -> t.List["MountParam.MountType"]: if not match: return [] - mounts: t.List["MountParam.MountType"] = [] - services: t.List[str] = [ + mounts: list["MountParam.MountType"] = [] + services: list[str] = [ service.strip() for service in match["services"].split(",") ] host_path = os.path.abspath(os.path.expanduser(match["host_path"])) @@ -159,11 +160,11 @@ def convert_explicit_form(self, value: str) -> t.List["MountParam.MountType"]: mounts.append((service, host_path, container_path)) return mounts - def convert_implicit_form(self, value: str) -> t.List["MountParam.MountType"]: + def convert_implicit_form(self, value: str) -> list["MountParam.MountType"]: """ Argument is of the form "/host/path" """ - mounts: t.List["MountParam.MountType"] = [] + mounts: list["MountParam.MountType"] = [] host_path = os.path.abspath(os.path.expanduser(value)) for service, container_path in hooks.Filters.COMPOSE_MOUNTS.iterate( os.path.basename(host_path) @@ -175,7 +176,7 @@ def convert_implicit_form(self, value: str) -> t.List["MountParam.MountType"]: def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> t.List[CompletionItem]: + ) -> list[CompletionItem]: """ Mount argument completion works only for the single path (implicit) form. The reason is that colons break words in bash completion: @@ -197,7 +198,7 @@ def shell_complete( def mount_tmp_volumes( - all_mounts: t.Tuple[t.List[MountParam.MountType], ...], + all_mounts: tuple[list[MountParam.MountType], ...], context: BaseComposeContext, ) -> None: for mounts in all_mounts: @@ -230,8 +231,8 @@ def mount_tmp_volume( @compose_tmp_filter.add() def _add_mounts_to_docker_compose_tmp( - docker_compose: t.Dict[str, t.Any], - ) -> t.Dict[str, t.Any]: + docker_compose: dict[str, t.Any], + ) -> dict[str, t.Any]: services = docker_compose.setdefault("services", {}) services.setdefault(service, {"volumes": []}) services[service]["volumes"].append(f"{host_path}:{container_path}") @@ -251,8 +252,8 @@ def start( context: BaseComposeContext, skip_build: bool, detach: bool, - mounts: t.Tuple[t.List[MountParam.MountType]], - services: t.List[str], + mounts: tuple[list[MountParam.MountType]], + services: list[str], ) -> None: command = ["up", "--remove-orphans"] if not skip_build: @@ -269,7 +270,7 @@ def start( @click.command(help="Stop a running platform") @click.argument("services", metavar="service", nargs=-1) @click.pass_obj -def stop(context: BaseComposeContext, services: t.List[str]) -> None: +def stop(context: BaseComposeContext, services: list[str]) -> None: config = tutor_config.load(context.root) context.job_runner(config).docker_compose("stop", *services) @@ -281,7 +282,7 @@ def stop(context: BaseComposeContext, services: t.List[str]) -> None: @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.argument("services", metavar="service", nargs=-1) @click.pass_context -def reboot(context: click.Context, detach: bool, services: t.List[str]) -> None: +def reboot(context: click.Context, detach: bool, services: list[str]) -> None: context.invoke(stop, services=services) context.invoke(start, detach=detach, services=services) @@ -295,7 +296,7 @@ def reboot(context: click.Context, detach: bool, services: t.List[str]) -> None: ) @click.argument("services", metavar="service", nargs=-1) @click.pass_obj -def restart(context: BaseComposeContext, services: t.List[str]) -> None: +def restart(context: BaseComposeContext, services: list[str]) -> None: config = tutor_config.load(context.root) command = ["restart"] if "all" in services: @@ -315,9 +316,7 @@ def restart(context: BaseComposeContext, services: t.List[str]) -> None: @jobs.do_group @mount_option @click.pass_obj -def do( - context: BaseComposeContext, mounts: t.Tuple[t.List[MountParam.MountType]] -) -> None: +def do(context: BaseComposeContext, mounts: tuple[list[MountParam.MountType]]) -> None: """ Run a custom job in the right container(s). """ @@ -345,8 +344,8 @@ def _mount_tmp_volumes(_job_name: str, *_args: t.Any, **_kwargs: t.Any) -> None: @click.pass_context def run( context: click.Context, - mounts: t.Tuple[t.List[MountParam.MountType]], - args: t.List[str], + mounts: tuple[list[MountParam.MountType]], + args: list[str], ) -> None: extra_args = ["--rm"] if not utils.is_a_tty(): @@ -411,7 +410,7 @@ def copyfrom( ) @click.argument("args", nargs=-1, required=True) @click.pass_context -def execute(context: click.Context, args: t.List[str]) -> None: +def execute(context: click.Context, args: list[str]) -> None: context.invoke(dc_command, command="exec", args=args) @@ -454,9 +453,9 @@ def status(context: click.Context) -> None: @click.pass_obj def dc_command( context: BaseComposeContext, - mounts: t.Tuple[t.List[MountParam.MountType]], + mounts: tuple[list[MountParam.MountType]], command: str, - args: t.List[str], + args: list[str], ) -> None: mount_tmp_volumes(mounts, context) config = tutor_config.load(context.root) @@ -465,8 +464,8 @@ def dc_command( @hooks.Filters.COMPOSE_MOUNTS.add() def _mount_edx_platform( - volumes: t.List[t.Tuple[str, str]], name: str -) -> t.List[t.Tuple[str, str]]: + volumes: list[tuple[str, str]], name: str +) -> list[tuple[str, str]]: """ When mounting edx-platform with `--mount=/path/to/edx-platform`, bind-mount the host repo in the lms/cms containers. diff --git a/tutor/commands/config.py b/tutor/commands/config.py index f48bf85aad..3d65801539 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -1,3 +1,4 @@ +from __future__ import annotations import json import typing as t @@ -27,7 +28,7 @@ class ConfigKeyParamType(click.ParamType): def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> t.List[click.shell_completion.CompletionItem]: + ) -> list[click.shell_completion.CompletionItem]: return [ click.shell_completion.CompletionItem(key) for key, _value in self._shell_complete_config_items(ctx, incomplete) @@ -36,7 +37,7 @@ def shell_complete( @staticmethod def _shell_complete_config_items( ctx: click.Context, incomplete: str - ) -> t.List[t.Tuple[str, ConfigValue]]: + ) -> list[tuple[str, ConfigValue]]: # Here we want to auto-complete the name of the config key. For that we need to # figure out the list of enabled plugins, and for that we need the project root. # The project root would ordinarily be stored in ctx.obj.root, but during @@ -58,7 +59,7 @@ class ConfigKeyValParamType(ConfigKeyParamType): name = "configkeyval" - def convert(self, value: str, param: t.Any, ctx: t.Any) -> t.Tuple[str, t.Any]: + def convert(self, value: str, param: t.Any, ctx: t.Any) -> tuple[str, t.Any]: result = serialize.parse_key_value(value) if result is None: self.fail(f"'{value}' is not of the form 'key=value'.", param, ctx) @@ -66,7 +67,7 @@ def convert(self, value: str, param: t.Any, ctx: t.Any) -> t.Tuple[str, t.Any]: def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> t.List[click.shell_completion.CompletionItem]: + ) -> list[click.shell_completion.CompletionItem]: """ Nice and friendly = auto-completion. """ @@ -117,7 +118,7 @@ def save( context: Context, interactive: bool, set_vars: Config, - unset_vars: t.List[str], + unset_vars: list[str], env_only: bool, ) -> None: config = tutor_config.load_minimal(context.root) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 908b75bf93..0fcb58df3d 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -1,4 +1,4 @@ -import typing as t +from __future__ import annotations import click @@ -70,7 +70,7 @@ def launch( context: click.Context, non_interactive: bool, pullimages: bool, - mounts: t.Tuple[t.List[compose.MountParam.MountType]], + mounts: tuple[list[compose.MountParam.MountType]], ) -> None: compose.mount_tmp_volumes(mounts, context.obj) try: diff --git a/tutor/commands/images.py b/tutor/commands/images.py index e6e78958bd..abc9207095 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t import click @@ -21,9 +22,9 @@ @hooks.Filters.IMAGES_BUILD.add() def _add_core_images_to_build( - build_images: t.List[t.Tuple[str, t.Tuple[str, ...], str, t.Tuple[str, ...]]], + build_images: list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], config: Config, -) -> t.List[t.Tuple[str, t.Tuple[str, ...], str, t.Tuple[str, ...]]]: +) -> list[tuple[str, tuple[str, ...], str, tuple[str, ...]]]: """ Add base images to the list of Docker images to build on `tutor build all`. """ @@ -35,8 +36,8 @@ def _add_core_images_to_build( @hooks.Filters.IMAGES_PULL.add() def _add_images_to_pull( - remote_images: t.List[t.Tuple[str, str]], config: Config -) -> t.List[t.Tuple[str, str]]: + remote_images: list[tuple[str, str]], config: Config +) -> list[tuple[str, str]]: """ Add base and vendor images to the list of Docker images to pull on `tutor pull all`. """ @@ -50,8 +51,8 @@ def _add_images_to_pull( @hooks.Filters.IMAGES_PUSH.add() def _add_core_images_to_push( - remote_images: t.List[t.Tuple[str, str]], config: Config -) -> t.List[t.Tuple[str, str]]: + remote_images: list[tuple[str, str]], config: Config +) -> list[tuple[str, str]]: """ Add base images to the list of Docker images to push on `tutor push all`. """ @@ -100,12 +101,12 @@ def images_command() -> None: @click.pass_obj def build( context: Context, - image_names: t.List[str], + image_names: list[str], no_cache: bool, - build_args: t.List[str], - add_hosts: t.List[str], + build_args: list[str], + add_hosts: list[str], target: str, - docker_args: t.List[str], + docker_args: list[str], ) -> None: config = tutor_config.load(context.root) command_args = [] @@ -132,7 +133,7 @@ def build( @click.command(short_help="Pull images from the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def pull(context: Context, image_names: t.List[str]) -> None: +def pull(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: for tag in find_remote_image_tags(config, hooks.Filters.IMAGES_PULL, image): @@ -142,7 +143,7 @@ def pull(context: Context, image_names: t.List[str]) -> None: @click.command(short_help="Push images to the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def push(context: Context, image_names: t.List[str]) -> None: +def push(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: for tag in find_remote_image_tags(config, hooks.Filters.IMAGES_PUSH, image): @@ -152,7 +153,7 @@ def push(context: Context, image_names: t.List[str]) -> None: @click.command(short_help="Print tag associated to a Docker image") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def printtag(context: Context, image_names: t.List[str]) -> None: +def printtag(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: for _name, _path, tag, _args in find_images_to_build(config, image): @@ -161,7 +162,7 @@ def printtag(context: Context, image_names: t.List[str]) -> None: def find_images_to_build( config: Config, image: str -) -> t.Iterator[t.Tuple[str, t.Tuple[str, ...], str, t.Tuple[str, ...]]]: +) -> t.Iterator[tuple[str, tuple[str, ...], str, tuple[str, ...]]]: """ Iterate over all images to build. @@ -182,7 +183,7 @@ def find_images_to_build( def find_remote_image_tags( config: Config, - filtre: "hooks.filters.Filter[t.List[t.Tuple[str, str]], [Config]]", + filtre: "hooks.filters.Filter[list[tuple[str, str]], [Config]]", image: str, ) -> t.Iterator[str]: """ diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 984030740c..de80725679 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -1,6 +1,7 @@ """ Common jobs that must be added both to local, dev and k8s commands. """ +from __future__ import annotations import functools import typing as t @@ -49,7 +50,7 @@ def _add_core_init_tasks() -> None: @click.command("init", help="Initialise all applications") @click.option("-l", "--limit", help="Limit initialisation to this service or plugin") -def initialise(limit: t.Optional[str]) -> t.Iterator[t.Tuple[str, str]]: +def initialise(limit: t.Optional[str]) -> t.Iterator[tuple[str, str]]: fmt.echo_info("Initialising all services...") filter_context = hooks.Contexts.APP(limit).name if limit else None @@ -99,7 +100,7 @@ def createuser( password: str, name: str, email: str, -) -> t.Iterable[t.Tuple[str, str]]: +) -> t.Iterable[tuple[str, str]]: """ Create an Open edX user @@ -127,7 +128,7 @@ def create_user_template( @click.command(help="Import the demo course") -def importdemocourse() -> t.Iterable[t.Tuple[str, str]]: +def importdemocourse() -> t.Iterable[tuple[str, str]]: template = """ # Import demo course git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course @@ -150,7 +151,7 @@ def importdemocourse() -> t.Iterable[t.Tuple[str, str]]: ), ) @click.argument("theme_name") -def settheme(domains: t.List[str], theme_name: str) -> t.Iterable[t.Tuple[str, str]]: +def settheme(domains: list[str], theme_name: str) -> t.Iterable[tuple[str, str]]: """ Assign a theme to the LMS and the CMS. @@ -159,7 +160,7 @@ def settheme(domains: t.List[str], theme_name: str) -> t.Iterable[t.Tuple[str, s yield ("lms", set_theme_template(theme_name, domains)) -def set_theme_template(theme_name: str, domain_names: t.List[str]) -> str: +def set_theme_template(theme_name: str, domain_names: list[str]) -> str: """ For each domain, get or create a Site object and assign the selected theme. """ @@ -231,7 +232,7 @@ def _patch_do_commands_callbacks() -> None: def _patch_callback( - job_name: str, func: t.Callable[P, t.Iterable[t.Tuple[str, str]]] + job_name: str, func: t.Callable[P, t.Iterable[tuple[str, str]]] ) -> t.Callable[P, None]: """ Modify a subcommand callback function such that its results are processed by `do_callback`. @@ -247,7 +248,7 @@ def new_callback(*args: P.args, **kwargs: P.kwargs) -> None: return new_callback -def do_callback(service_commands: t.Iterable[t.Tuple[str, str]]) -> None: +def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None: """ This function must be added as a callback to all `do` subcommands. diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 409e16e8b0..140edc0d21 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t import click @@ -70,7 +71,7 @@ def local(context: click.Context) -> None: @click.pass_context def launch( context: click.Context, - mounts: t.Tuple[t.List[compose.MountParam.MountType]], + mounts: tuple[list[compose.MountParam.MountType]], non_interactive: bool, pullimages: bool, ) -> None: diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index ed671494fd..225be11d5b 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import typing as t import urllib.request @@ -22,13 +23,13 @@ def __init__(self, allow_all: bool = False): def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> t.List[click.shell_completion.CompletionItem]: + ) -> list[click.shell_completion.CompletionItem]: return [ click.shell_completion.CompletionItem(name) for name in self.get_names(incomplete) ] - def get_names(self, incomplete: str) -> t.List[str]: + def get_names(self, incomplete: str) -> list[str]: candidates = [] if self.allow_all: candidates.append("all") @@ -67,7 +68,7 @@ def list_command() -> None: @click.command(help="Enable a plugin") @click.argument("plugin_names", metavar="plugin", nargs=-1, type=PluginName()) @click.pass_obj -def enable(context: Context, plugin_names: t.List[str]) -> None: +def enable(context: Context, plugin_names: list[str]) -> None: config = tutor_config.load_minimal(context.root) for plugin in plugin_names: plugins.load(plugin) @@ -87,10 +88,10 @@ def enable(context: Context, plugin_names: t.List[str]) -> None: "plugin_names", metavar="plugin", nargs=-1, type=PluginName(allow_all=True) ) @click.pass_obj -def disable(context: Context, plugin_names: t.List[str]) -> None: +def disable(context: Context, plugin_names: list[str]) -> None: config = tutor_config.load_minimal(context.root) disable_all = "all" in plugin_names - disabled: t.List[str] = [] + disabled: list[str] = [] for plugin in tutor_config.get_enabled_plugins(config): if disable_all or plugin in plugin_names: fmt.echo_info(f"Disabling plugin {plugin}...") diff --git a/tutor/config.py b/tutor/config.py index b77a494811..5512b8441e 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -1,5 +1,5 @@ +from __future__ import annotations import os -import typing as t from tutor import env, exceptions, fmt, hooks, plugins, serialize, utils from tutor.types import Config, ConfigValue, cast_config, get_typed @@ -108,7 +108,7 @@ def get_base() -> Config: Entries in this configuration are unrendered. """ base = get_template("base.yml") - extra_config: t.List[t.Tuple[str, ConfigValue]] = [] + extra_config: list[tuple[str, ConfigValue]] = [] extra_config = hooks.Filters.CONFIG_UNIQUE.apply(extra_config) extra_config = hooks.Filters.CONFIG_OVERRIDES.apply(extra_config) for name, value in extra_config: @@ -269,7 +269,7 @@ def enable_plugins(config: Config) -> None: plugins.load_all(get_enabled_plugins(config)) -def get_enabled_plugins(config: Config) -> t.List[str]: +def get_enabled_plugins(config: Config) -> list[str]: """ Return the list of plugins that are enabled, as per the configuration. Note that this may differ from the list of loaded plugins. For instance when a plugin is diff --git a/tutor/env.py b/tutor/env.py index b3e2299507..908a38e055 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import re import shutil @@ -111,7 +112,7 @@ def iter_templates_in(self, *prefix: str) -> t.Iterable[str]: The elements of `prefix` must contain only "/", and not os.sep. """ full_prefix = "/".join(prefix) - env_templates: t.List[str] = self.environment.loader.list_templates() + env_templates: list[str] = self.environment.loader.list_templates() for template in env_templates: if template.startswith(full_prefix): # Exclude templates that match certain patterns diff --git a/tutor/hooks/actions.py b/tutor/hooks/actions.py index 1ef6db44c3..ccaf46709b 100644 --- a/tutor/hooks/actions.py +++ b/tutor/hooks/actions.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" @@ -53,11 +55,11 @@ class Action(t.Generic[P]): This strong typing makes it easier for plugin developers to quickly check whether they are adding and calling action callbacks correctly. """ - INDEX: t.Dict[str, "Action[t.Any]"] = {} + INDEX: dict[str, "Action[t.Any]"] = {} def __init__(self, name: str) -> None: self.name = name - self.callbacks: t.List[ActionCallback[P]] = [] + self.callbacks: list[ActionCallback[P]] = [] def __repr__(self) -> str: return f"{self.__class__.__name__}('{self.name}')" diff --git a/tutor/hooks/contexts.py b/tutor/hooks/contexts.py index 0ac98216e0..75a3182b71 100644 --- a/tutor/hooks/contexts.py +++ b/tutor/hooks/contexts.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" @@ -6,7 +8,7 @@ class Context: - CURRENT: t.List[str] = [] + CURRENT: list[str] = [] def __init__(self, name: str): self.name = name diff --git a/tutor/hooks/filters.py b/tutor/hooks/filters.py index 85fa29423b..4aa10f9fd2 100644 --- a/tutor/hooks/filters.py +++ b/tutor/hooks/filters.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" @@ -58,11 +60,11 @@ class Filter(t.Generic[T, P]): they are adding and calling filter callbacks correctly. """ - INDEX: t.Dict[str, "Filter[t.Any, t.Any]"] = {} + INDEX: dict[str, "Filter[t.Any, t.Any]"] = {} def __init__(self, name: str) -> None: self.name = name - self.callbacks: t.List[FilterCallback[T, P]] = [] + self.callbacks: list[FilterCallback[T, P]] = [] def __repr__(self) -> str: return f"{self.__class__.__name__}('{self.name}')" @@ -143,12 +145,12 @@ def clear(self, context: t.Optional[str] = None) -> None: # The methods below are specific to filters which take lists as first arguments def add_item( - self: "Filter[t.List[E], P]", item: E, priority: t.Optional[int] = None + self: "Filter[list[E], P]", item: E, priority: t.Optional[int] = None ) -> None: self.add_items([item], priority=priority) def add_items( - self: "Filter[t.List[E], P]", items: t.List[E], priority: t.Optional[int] = None + self: "Filter[list[E], P]", items: list[E], priority: t.Optional[int] = None ) -> None: # Unfortunately we have to type-ignore this line. If not, mypy complains with: # @@ -158,18 +160,16 @@ def add_items( # But we are unable to mark arguments positional-only (by adding / after values arg) in Python 3.7. # Get rid of this statement after Python 3.7 EOL. @self.add(priority=priority) # type: ignore - def callback( - values: t.List[E], *_args: P.args, **_kwargs: P.kwargs - ) -> t.List[E]: + def callback(values: list[E], *_args: P.args, **_kwargs: P.kwargs) -> list[E]: return values + items def iterate( - self: "Filter[t.List[E], P]", *args: P.args, **kwargs: P.kwargs + self: "Filter[list[E], P]", *args: P.args, **kwargs: P.kwargs ) -> t.Iterator[E]: yield from self.iterate_from_context(None, *args, **kwargs) def iterate_from_context( - self: "Filter[t.List[E], P]", + self: "Filter[list[E], P]", context: t.Optional[str], *args: P.args, **kwargs: P.kwargs, @@ -268,7 +268,7 @@ def add_item(name: str, item: T, priority: t.Optional[int] = None) -> None: get(name).add_item(item, priority=priority) -def add_items(name: str, items: t.List[T], priority: t.Optional[int] = None) -> None: +def add_items(name: str, items: list[T], priority: t.Optional[int] = None) -> None: """ Convenience function to add multiple item to a filter that returns a list of items. diff --git a/tutor/hooks/priorities.py b/tutor/hooks/priorities.py index 3c43d536a1..c493bdb54c 100644 --- a/tutor/hooks/priorities.py +++ b/tutor/hooks/priorities.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t from typing_extensions import Protocol @@ -14,7 +15,7 @@ class PrioritizedCallback(Protocol): TPrioritized = t.TypeVar("TPrioritized", bound=PrioritizedCallback) -def insert_callback(callback: TPrioritized, callbacks: t.List[TPrioritized]) -> None: +def insert_callback(callback: TPrioritized, callbacks: list[TPrioritized]) -> None: # I wish we could use bisect.insort_right here but the `key=` parameter # is unsupported in Python 3.9 position = 0 diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py index cc5c5d250f..fd137bee96 100644 --- a/tutor/plugins/__init__.py +++ b/tutor/plugins/__init__.py @@ -1,6 +1,7 @@ """ Provide API for plugin features. """ +from __future__ import annotations import typing as t from copy import deepcopy @@ -20,7 +21,7 @@ def _convert_plugin_patches() -> None: This action is run after plugins have been loaded. """ - patches: t.Iterable[t.Tuple[str, str]] = hooks.Filters.ENV_PATCHES.iterate() + patches: t.Iterable[tuple[str, str]] = hooks.Filters.ENV_PATCHES.iterate() for name, content in patches: hooks.Filters.ENV_PATCH(name).add_item(content) @@ -44,14 +45,14 @@ def iter_installed() -> t.Iterator[str]: yield from sorted(hooks.Filters.PLUGINS_INSTALLED.iterate()) -def iter_info() -> t.Iterator[t.Tuple[str, t.Optional[str]]]: +def iter_info() -> t.Iterator[tuple[str, t.Optional[str]]]: """ Iterate on the information of all installed plugins. Yields (, ) tuples. """ - def plugin_info_name(info: t.Tuple[str, t.Optional[str]]) -> str: + def plugin_info_name(info: tuple[str, t.Optional[str]]) -> str: return info[0] yield from sorted(hooks.Filters.PLUGINS_INFO.iterate(), key=plugin_info_name) diff --git a/tutor/serialize.py b/tutor/serialize.py index 415f75cf46..a838d40ab9 100644 --- a/tutor/serialize.py +++ b/tutor/serialize.py @@ -1,3 +1,4 @@ +from __future__ import annotations import re import typing as t @@ -36,7 +37,7 @@ def parse(v: t.Union[str, t.IO[str]]) -> t.Any: return v -def parse_key_value(text: str) -> t.Optional[t.Tuple[str, t.Any]]: +def parse_key_value(text: str) -> t.Optional[tuple[str, t.Any]]: """ Parse = command line arguments. diff --git a/tutor/templates/apps/openedx/settings/cms/production.py b/tutor/templates/apps/openedx/settings/cms/production.py index d09456e34d..03cae79b09 100644 --- a/tutor/templates/apps/openedx/settings/cms/production.py +++ b/tutor/templates/apps/openedx/settings/cms/production.py @@ -8,6 +8,7 @@ ENV_TOKENS.get("CMS_BASE"), "cms", ] +CORS_ORIGIN_WHITELIST.append("{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ CMS_HOST }}") # Authentication SOCIAL_AUTH_EDX_OAUTH2_KEY = "{{ CMS_OAUTH2_KEY_SSO }}" diff --git a/tutor/templates/apps/openedx/settings/lms/production.py b/tutor/templates/apps/openedx/settings/lms/production.py index b859b5feb7..6c2793b3bc 100644 --- a/tutor/templates/apps/openedx/settings/lms/production.py +++ b/tutor/templates/apps/openedx/settings/lms/production.py @@ -9,6 +9,7 @@ FEATURES["PREVIEW_LMS_BASE"], "lms", ] +CORS_ORIGIN_WHITELIST.append("{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}") {% if ENABLE_HTTPS %} # Properly set the "secure" attribute on session/csrf cookies. This is required in diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index fb39fa8a9d..d248b00a39 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -78,6 +78,10 @@ }, } +# The default Django contrib site is the one associated to the LMS domain name. 1 is +# usually "example.com", so it's the next available integer. +SITE_ID = 2 + # Contact addresses CONTACT_MAILING_ADDRESS = "{{ PLATFORM_NAME }} - {% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}" DEFAULT_FROM_EMAIL = ENV_TOKENS.get("DEFAULT_FROM_EMAIL", ENV_TOKENS["CONTACT_EMAIL"]) @@ -215,5 +219,18 @@ "user": None, } +# Upstream expects node_modules to be within edx-platform, but we put +# node_modules under /openedx/node_modules, so we must adjust any settings +# that hold a node_modules path. +NODE_MODULES_ROOT = "/openedx/node_modules/@edx" +STATICFILES_DIRS = [ + *[ + staticfiles_dir for staticfiles_dir in STATICFILES_DIRS + if "node_modules/@edx" not in staticfiles_dir + ], + NODE_MODULES_ROOT, +] +PIPELINE["UGLIFYJS_BINARY"] = "/openedx/node_modules/.bin/uglifyjs" + {{ patch("openedx-common-settings") }} ######## End of settings common to LMS and CMS diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index a1ba493803..27729c75e6 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -42,8 +42,12 @@ WORKDIR /openedx/edx-platform RUN git config --global user.email "tutor@overhang.io" \ && git config --global user.name "Tutor" +{%- if patch("openedx-dockerfile-git-patches-default") %} # Custom edx-platform patches {{ patch("openedx-dockerfile-git-patches-default") }} +{%- else %} +# Patch edx-platform +{%- endif %} {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/ | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} @@ -103,11 +107,11 @@ ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} RUN pip install nodeenv==1.7.0 RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt -# Install nodejs requirements +# Install nodejs requirements to /openedx/node_modules ARG NPM_REGISTRY={{ NPM_REGISTRY }} -COPY --from=code /openedx/edx-platform/package.json /openedx/edx-platform/package.json -COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/package-lock.json -WORKDIR /openedx/edx-platform +COPY --from=code /openedx/edx-platform/package.json /openedx/package.json +COPY --from=code /openedx/edx-platform/package-lock.json /openedx/package-lock.json +WORKDIR /openedx RUN npm install --verbose --registry=$NPM_REGISTRY ###### Production image with system and python requirements @@ -132,9 +136,9 @@ COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv -COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules +COPY --chown=app:app --from=nodejs-requirements /openedx/node_modules /openedx/node_modules -ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} +ENV PATH /openedx/venv/bin:/openedx/node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ WORKDIR /openedx/edx-platform diff --git a/tutor/templates/build/openedx/bin/openedx-assets b/tutor/templates/build/openedx/bin/openedx-assets index 1b89434d67..37e32e1c70 100755 --- a/tutor/templates/build/openedx/bin/openedx-assets +++ b/tutor/templates/build/openedx/bin/openedx-assets @@ -1,19 +1,43 @@ #! /usr/bin/env python -from __future__ import print_function +from __future__ import annotations + import argparse +import glob import os +import shlex import subprocess import sys import traceback +from datetime import datetime +from pathlib import Path -from path import Path - -from pavelib import assets +import sass # pylint: disable=import-error +from pavelib.assets import ( # pylint: disable=import-error + Observer, + SassWatcher, + debounce, + _compile_sass, +) DEFAULT_STATIC_ROOT = "/openedx/staticfiles" DEFAULT_THEMES_DIR = "/openedx/themes" +NODE_MODULES_PATH = Path("/openedx/node_modules") + +# Common lookup paths that are added to the lookup paths for all sass compilations +COMMON_LOOKUP_PATHS = [ + Path("common/static"), + Path("common/static/sass"), + NODE_MODULES_PATH / "@edx", + NODE_MODULES_PATH, +] + +# system specific lookup path additions, add sass dirs if one system depends on the sass files for other systems +SASS_LOOKUP_DEPENDENCIES = { + 'cms': [Path('lms') / 'static' / 'sass' / 'partials', ], +} + def main(): parser = argparse.ArgumentParser( @@ -21,10 +45,7 @@ def main(): ) subparsers = parser.add_subparsers() - npm = subparsers.add_parser("npm", help="Copy static assets from node_modules") - npm.set_defaults(func=run_npm) - - build = subparsers.add_parser("build", help="Build all assets") + build = subparsers.add_parser("build", help="Build all assets (npm+xmodule+webpack+common+themes)") build.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") build.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR]) build.add_argument("--themes", nargs="+", default=["all"]) @@ -32,6 +53,9 @@ def main(): build.add_argument("--systems", nargs="+", default=["lms", "cms"]) build.set_defaults(func=run_build) + npm = subparsers.add_parser("npm", help="Copy static assets from node_modules") + npm.set_defaults(func=run_npm) + xmodule = subparsers.add_parser("xmodule", help="Process assets from xmodule") xmodule.set_defaults(func=run_xmodule) @@ -98,6 +122,8 @@ def run_build(args): def run_xmodule(_args): + print(f"{sys.argv[0]}: Collecting xmodule assets") + # Collecting xmodule assets is incompatible with setting the django path, because # of an unfortunate call to settings.configure() django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE") @@ -105,7 +131,7 @@ def run_xmodule(_args): os.environ.pop("DJANGO_SETTINGS_MODULE") sys.argv[1:] = ["common/static/xmodule"] - import xmodule.static_content + import xmodule.static_content # pylint: disable=import-error xmodule.static_content.main() @@ -114,29 +140,32 @@ def run_xmodule(_args): def run_npm(_args): - assets.process_npm_assets() - + """ + Post-process npm assets. + """ + sh("/bin/sh", "scripts/assets/copy-node-modules.sh", "--node-modules", "/openedx/node_modules") def run_webpack(args): + print(f"{sys.argv[0]}: Executing webpack") os.environ["STATIC_ROOT_LMS"] = args.static_root os.environ["STATIC_ROOT_CMS"] = os.path.join(args.static_root, "studio") os.environ["NODE_ENV"] = {"prod": "production", "dev": "development"}[args.env] - subprocess.check_call( - [ - "webpack", - "--progress", - "--config=webpack.{env}.config.js".format(env=args.env), - ] + sh( + "webpack", + "--progress", + "--config=webpack.{env}.config.js".format(env=args.env), ) def run_common(args): + print(f"{sys.argv[0]}: Compiling sass assets from common theme") for system in args.systems: print("Compiling {} sass assets from common theme...".format(system)) - assets._compile_sass(system, None, False, False, []) + _compile_sass(system, None, False, False, []) def run_themes(args): + print(f"{sys.argv[0]}: Compiling sass assets for custom themes") for theme_dir in args.theme_dirs: local_themes = ( list_subdirectories(theme_dir) if "all" in args.themes else args.themes @@ -150,11 +179,12 @@ def run_themes(args): system, theme_path ) ) - assets._compile_sass(system, Path(theme_path), False, False, []) + _compile_sass(system, Path(theme_path), False, False, []) def run_collect(args): - assets.collect_assets(args.systems, args.settings) + print(f"{sys.argv[0]}: Collecting assets") + collect_assets(args.systems, args.settings) def run_watch_themes(args): @@ -168,7 +198,7 @@ def run_watch_themes(args): Note that this function will only work for watching assets in development mode. In production, watching changes does not make much sense anyway. """ - observer = assets.Observer() + observer = Observer() for theme_dir in args.theme_dirs: print("Watching changes in {}...".format(theme_dir)) ThemeWatcher(theme_dir).register(observer) @@ -188,7 +218,7 @@ def list_subdirectories(path): ] -class ThemeWatcher(assets.SassWatcher): +class ThemeWatcher(SassWatcher): def __init__(self, theme_dir): super(ThemeWatcher, self).__init__() self.theme_dir = theme_dir @@ -197,7 +227,7 @@ class ThemeWatcher(assets.SassWatcher): def register(self, observer): return super(ThemeWatcher, self).register(observer, [self.theme_dir]) - @assets.debounce() + @debounce() def on_any_event(self, event): components = os.path.relpath(event.src_path, self.theme_dir).split("/") try: @@ -208,11 +238,228 @@ class ThemeWatcher(assets.SassWatcher): try: print("Detected change:", event.src_path) print("\tRecompiling {} theme for {}".format(theme, system)) - assets._compile_sass(system, Path(self.theme_dir) / theme, False, False, []) + _compile_sass(system, Path(self.theme_dir) / theme, False, False, []) print("\tDone recompiling {} theme for {}".format(theme, system)) except Exception: # pylint: disable=broad-except traceback.print_exc() +def __compile_sass(system: str, theme: Path|None, debug: bool, force: bool, timing_info: list): + """ + Compile sass files for the given system and theme. + + Reimplementation of edx-platform's pavelib.assets:_compile_sass + + :param system: system to compile sass for e.g. 'lms', 'cms', 'common' + :param theme: absolute path of the theme to compile sass for. + :param debug: showing whether to display source comments in resulted css + :param force: showing whether to remove existing css files before generating new files + :param timing_info: (unused in this implementation) + """ + sass_dirs: list[dict] + if system == "common": + sass_dirs = [{ + "sass_source_dir": Path("common/static/sass"), + "css_destination_dir": Path("common/static/css"), + "lookup_paths": COMMON_LOOKUP_PATHS, + }] + elif theme: + sass_dirs = get_theme_sass_dirs(system, theme) + else: + sass_dirs = get_system_sass_dirs(system) + + # determine css out put style and source comments enabling + if debug: + source_comments = True + output_style = 'nested' + else: + source_comments = False + output_style = 'compressed' + + for dirs in sass_dirs: + start = datetime.now() + css_dir = str(dirs['css_destination_dir']) + sass_source_dir = str(dirs['sass_source_dir']) + lookup_paths: list[Path] = dirs['lookup_paths'] + + if not Path(sass_source_dir).is_dir(): + print("\033[91m Sass dir '{dir}' does not exists, skipping sass compilation for '{theme}' \033[00m".format( + dir=sass_source_dir, theme=theme or system, + )) + # theme doesn't override sass directory, so skip it + continue + + if force: + sh(f"rm", "-rf", f"{css_dir}/*.css") + + sass.compile( + dirname=(sass_source_dir, css_dir), + include_paths=[str(path) for path in COMMON_LOOKUP_PATHS + lookup_paths], + source_comments=source_comments, + output_style=output_style, + ) + + # For Sass files without explicit RTL versions, generate + # an RTL version of the CSS using the rtlcss library. + for sass_file in glob.glob(str(sass_source_dir) + '/**/*.scss'): + if should_generate_rtl_css_file(sass_file): + source_css_file = sass_file.replace(sass_source_dir, css_dir).replace('.scss', '.css') + target_css_file = source_css_file.replace('.css', '-rtl.css') + sh("rtlcss", source_css_file, target_css_file) + + +def get_theme_sass_dirs(system: str, theme_dir: Path) -> list[dict]: + """ + Return list of sass dirs that need to be compiled for the given theme. + """ + dirs = [] + + system_sass_dir = Path(system) / "static" / "sass" + sass_dir = theme_dir / system / "static" / "sass" + css_dir = theme_dir / system / "static" / "css" + certs_sass_dir = theme_dir / system / "static" / "certificates" / "sass" + certs_css_dir = theme_dir / system / "static" / "certificates" / "css" + + dependencies = SASS_LOOKUP_DEPENDENCIES.get(system, []) + if sass_dir.is_dir(): + css_dir.mkdir(parents=True, exist_ok=True) + + # first compile lms sass files and place css in theme dir + dirs.append({ + "sass_source_dir": system_sass_dir, + "css_destination_dir": css_dir, + "lookup_paths": dependencies + [ + sass_dir / "partials", + system_sass_dir / "partials", + system_sass_dir, + ], + }) + + # now compile theme sass files and override css files generated from lms + dirs.append({ + "sass_source_dir": sass_dir, + "css_destination_dir": css_dir, + "lookup_paths": dependencies + [ + sass_dir / "partials", + system_sass_dir / "partials", + system_sass_dir, + ], + }) + + # now compile theme sass files for certificate + if system == 'lms': + dirs.append({ + "sass_source_dir": certs_sass_dir, + "css_destination_dir": certs_css_dir, + "lookup_paths": [ + sass_dir / "partials", + sass_dir + ], + }) + + return dirs + + +def get_system_sass_dirs(system) -> list[dict]: + """ + Return list of sass dirs that need to be compiled for the given system. + """ + dirs = [] + sass_dir = Path(system) / "static" / "sass" + css_dir = Path(system) / "static" / "css" + dependencies = SASS_LOOKUP_DEPENDENCIES.get(system, []) + dirs.append({ + "sass_source_dir": sass_dir, + "css_destination_dir": css_dir, + "lookup_paths": dependencies + [ + sass_dir / "partials", + sass_dir, + ], + }) + if system == 'lms': + dirs.append({ + "sass_source_dir": Path(system) / "static" / "certificates" / "sass", + "css_destination_dir": Path(system) / "static" / "certificates" / "css", + "lookup_paths": [ + sass_dir / "partials", + sass_dir + ], + }) + return dirs + + +def should_generate_rtl_css_file(sass_file: str) -> bool: + """ + Returns true if a Sass file should have an RTL version generated. + """ + # Don't generate RTL CSS for partials + if Path(sass_file).name.startswith('_'): + return False + + # Don't generate RTL CSS if the file is itself an RTL version + if sass_file.endswith('-rtl.scss'): + return False + + # Don't generate RTL CSS if there is an explicit Sass version for RTL + rtl_sass_file = Path(sass_file.replace('.scss', '-rtl.scss')) + if rtl_sass_file.exists(): + return False + + return True + + +def collect_assets(systems: list[str], settings: str): + """ + Collect static assets, including Django pipeline processing. + + Reimplementation of edx-platform's pavelib.assets.collect_assets + """ + ignore_patterns = [ + # Karma test related files... + "fixtures", + "karma_*.js", + "spec", + "spec_helpers", + "spec-helpers", + "xmodule_js", # symlink for tests + + # Geo-IP data, only accessed in Python + "geoip", + + # We compile these out, don't need the source files in staticfiles + "sass", + ] + + ignore_args: list[str] = [ + arg + for pattern in ignore_patterns + for arg in [f"--ignore", pattern] + ] + for sys in systems: + if sys == "studio": + sys = "cms" + sh( + "./manage.py", + sys, + "collectstatic", + *ignore_args, + "--noinput", + "--settings", + settings, + ) + print(f"\t\tFinished collecting {sys} assets.") + + +def sh(*command): + """ + Run command in a shell. + + (if we ran it directly, globbing (*) wouldn't work) + """ + shell_command = shlex.join(command) + print(f"+{shell_command}") + return subprocess.check_call(shell_command, shell=True) + + if __name__ == "__main__": main() diff --git a/tutor/templates/build/openedx/settings/partials/assets.py b/tutor/templates/build/openedx/settings/partials/assets.py index fce97cdb85..bbd4e7f036 100644 --- a/tutor/templates/build/openedx/settings/partials/assets.py +++ b/tutor/templates/build/openedx/settings/partials/assets.py @@ -18,4 +18,17 @@ "default": {}, } +# Upstream expects node_modules to be within edx-platform, but we put +# node_modules under /openedx/node_modules, so we must adjust any settings +# that hold a node_modules path. +NODE_MODULES_ROOT = "/openedx/node_modules/@edx" +STATICFILES_DIRS = [ + *[ + staticfiles_dir for staticfiles_dir in STATICFILES_DIRS + if "node_modules/@edx" not in staticfiles_dir + ], + NODE_MODULES_ROOT, +] +PIPELINE["UGLIFYJS_BINARY"] = "/openedx/node_modules/.bin/uglifyjs" + {{ patch("openedx-common-assets-settings") }} diff --git a/tutor/types.py b/tutor/types.py index 4f772053f0..c6156b3173 100644 --- a/tutor/types.py +++ b/tutor/types.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" @@ -38,9 +40,9 @@ def cast_config(config: t.Any) -> Config: def get_typed( - config: t.Dict[str, t.Any], + config: dict[str, t.Any], key: str, - expected_type: t.Type[T], + expected_type: type[T], default: t.Optional[T] = None, ) -> T: value = config.get(key, default)