diff --git a/Makefile b/Makefile index cc6fa5591376..3dd8a9c63929 100644 --- a/Makefile +++ b/Makefile @@ -56,8 +56,14 @@ endif push_translations: ## push source strings to Transifex for translation i18n_tool transifex push +pull_plugin_translations: ## Pull translations from Transifex for edx_django_utils.plugins for both lms and cms + rm -rf conf/plugins-locale/plugins # Clean up existing atlas translations + mkdir -p conf/plugins-locale/plugins + python manage.py lms pull_plugin_translations --verbose $(ATLAS_OPTIONS) + python manage.py lms compile_plugin_translations + pull_xblock_translations: ## pull xblock translations via atlas - rm -rf conf/plugins-locale # Clean up existing atlas translations + rm -rf conf/plugins-locale/xblock.v1 # Clean up existing atlas translations rm -rf lms/static/i18n/xblock.v1 cms/static/i18n/xblock.v1 # Clean up existing xblock compiled translations mkdir -p conf/plugins-locale/xblock.v1/ lms/static/js/xblock.v1-i18n cms/static/js python manage.py lms pull_xblock_translations --verbose $(ATLAS_OPTIONS) @@ -76,6 +82,7 @@ ifeq ($(OPENEDX_ATLAS_PULL),) i18n_tool validate --verbose else make pull_xblock_translations + make pull_plugin_translations find conf/locale -mindepth 1 -maxdepth 1 -type d -exec rm -r {} \; atlas pull $(ATLAS_OPTIONS) translations/edx-platform/conf/locale:conf/locale i18n_tool generate diff --git a/common/djangoapps/xblock_django/management/commands/pull_xblock_translations.py b/common/djangoapps/xblock_django/management/commands/pull_xblock_translations.py index 3eb3f519e6be..5fd14767aec6 100644 --- a/common/djangoapps/xblock_django/management/commands/pull_xblock_translations.py +++ b/common/djangoapps/xblock_django/management/commands/pull_xblock_translations.py @@ -2,15 +2,13 @@ Download the translations via atlas for the XBlocks. """ -from django.core.management.base import BaseCommand, CommandError - -from openedx.core.djangoapps.plugins.i18n_api import ATLAS_ARGUMENTS +from openedx.core.djangoapps.plugins.i18n_api import BaseAtlasPullCommand from xmodule.modulestore import api as xmodule_api from ...translation import xblocks_atlas_pull -class Command(BaseCommand): +class Command(BaseAtlasPullCommand): """ Pull the XBlock translations via atlas for the XBlocks. @@ -19,33 +17,9 @@ class Command(BaseCommand): - https://github.com/openedx/openedx-atlas """ - def add_arguments(self, parser): - for argument in ATLAS_ARGUMENTS: - parser.add_argument(*argument.get_args(), **argument.get_kwargs()) - - parser.add_argument( - '--verbose|-v', - action='store_true', - default=False, - dest='verbose', - help='Verbose output using `--verbose` argument for `atlas pull`.', - ) - def handle(self, *args, **options): xblock_translations_root = xmodule_api.get_python_locale_root() - if list(xblock_translations_root.listdir()): - raise CommandError(f'"{xblock_translations_root}" should be empty before running atlas pull.') - - atlas_pull_options = [] - - for argument in ATLAS_ARGUMENTS: - option_value = options.get(argument.dest) - if option_value is not None: - atlas_pull_options += [argument.flag, option_value] - - if options['verbose']: - atlas_pull_options += ['--verbose'] - else: - atlas_pull_options += ['--silent'] + self.ensure_empty_directory(xblock_translations_root) + atlas_pull_options = self.get_atlas_pull_options(**options) xblocks_atlas_pull(pull_options=atlas_pull_options) diff --git a/openedx/core/djangoapps/plugins/api.py b/openedx/core/djangoapps/plugins/api.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/core/djangoapps/plugins/apps.py b/openedx/core/djangoapps/plugins/apps.py index b4bb2501c3e3..625a82966bd3 100644 --- a/openedx/core/djangoapps/plugins/apps.py +++ b/openedx/core/djangoapps/plugins/apps.py @@ -9,7 +9,8 @@ from django.conf import settings from edx_django_utils.plugins import connect_plugin_receivers -from openedx.core.djangoapps.plugins.constants import ProjectType +from openedx.core.djangoapps.plugins.constants import ProjectType, SettingsType +from edx_django_utils.plugins import PluginSettings class PluginsConfig(AppConfig): @@ -19,7 +20,16 @@ class PluginsConfig(AppConfig): name = 'openedx.core.djangoapps.plugins' - plugin_app = {} + plugin_app = { + PluginSettings.CONFIG: { + ProjectType.LMS: { + SettingsType.PRODUCTION: {PluginSettings.RELATIVE_PATH: 'settings.production'}, + }, + ProjectType.CMS: { + SettingsType.PRODUCTION: {PluginSettings.RELATIVE_PATH: 'settings.production'}, + }, + } + } def ready(self): """ diff --git a/openedx/core/djangoapps/plugins/constants.py b/openedx/core/djangoapps/plugins/constants.py index 17f559b103d5..a4dceee83fe5 100644 --- a/openedx/core/djangoapps/plugins/constants.py +++ b/openedx/core/djangoapps/plugins/constants.py @@ -37,3 +37,7 @@ class SettingsType(): COMMON = 'common' DEVSTACK = 'devstack' TEST = 'test' + + +# Locale root for IDA plugins for LMS and CMS, relative to settings.REPO_ROOT +plugins_locale_root = 'conf/plugins-locale/plugins' diff --git a/openedx/core/djangoapps/plugins/i18n_api.py b/openedx/core/djangoapps/plugins/i18n_api.py index 923b09e3854f..06bb715b22a5 100644 --- a/openedx/core/djangoapps/plugins/i18n_api.py +++ b/openedx/core/djangoapps/plugins/i18n_api.py @@ -3,10 +3,14 @@ """ from dataclasses import dataclass, asdict +from collections import defaultdict import os from pathlib import Path import subprocess +from django.core.management import BaseCommand, CommandError +from importlib_metadata import entry_points + @dataclass class ArgparseArgument: @@ -33,7 +37,7 @@ def get_args(self): return [self.flag] -# `atlas pull` arguments defintions. +# `atlas pull` arguments definitions. # # - https://github.com/openedx/openedx-atlas # @@ -53,11 +57,63 @@ def get_args(self): ArgparseArgument( flag='--branch', dest='branch', - help='Custom branch for "atlas pull" e.g. --branch=release/redwood . Default is "main".', + help='Deprecated option. Use --revision instead.', + ), + ArgparseArgument( + flag='--revision', + dest='revision', + help='Custom git revision for "atlas pull" e.g. --revision=release/redwood . Default is "main".', ), ] +class BaseAtlasPullCommand(BaseCommand): + """ + Base `atlas pull` Django command. + """ + + def add_arguments(self, parser): + """ + Configure Django command arguments. + """ + for argument in ATLAS_ARGUMENTS: + parser.add_argument(*argument.get_args(), **argument.get_kwargs()) + + parser.add_argument( + '--verbose|-v', + action='store_true', + default=False, + dest='verbose', + help='Verbose output using `--verbose` argument for `atlas pull`.', + ) + + def ensure_empty_directory(self, directory): + """ + Ensure the pull directory is empty before running atlas pull. + """ + plugin_translations_root = directory + if os.listdir(plugin_translations_root): + raise CommandError(f'"{plugin_translations_root}" should be empty before running atlas pull.') + + def get_atlas_pull_options(self, **options): + """ + Pass-through the Django command options to `atlas pull`. + """ + atlas_pull_options = [] + + for argument in ATLAS_ARGUMENTS: + option_value = options.get(argument.dest) + if option_value is not None: + atlas_pull_options += [argument.flag, option_value] + + if options['verbose']: + atlas_pull_options += ['--verbose'] + else: + atlas_pull_options += ['--silent'] + + return atlas_pull_options + + def atlas_pull_by_modules(module_names, locale_root, pull_options): """ Atlas pull translations by module name instead of repository name. @@ -91,3 +147,40 @@ def compile_po_files(root_dir): args=['msgfmt', '--check-format', '-o', str(po_file_path.with_suffix('.mo')), str(po_file_path)], check=True, ) + + +def get_installed_plugins_module_names(): + """ + Return the installed plugins Python module names. + + This function excludes the built-in edx-platform plugins such as `lms`, `cms` and `openedx`. + """ + # group (e.g 'lms.djangoapp') -> set for root module names (e.g {'edx_sga'}) + root_modules = defaultdict(set) + + for entry_point in entry_points(): + module_name = entry_point.value + root_module = module_name.split('.')[0] # e.g. `edx_sga` from `edx_sga.core.xblock` + root_modules[entry_point.group].add(root_module) + + return ( + # Return all lms.djangopapp and cms.djangoapp plugins + (root_modules['lms.djangoapp'] | root_modules['cms.djangoapp']) + # excluding the edx-platform built-in plugins which don't need atlas + - {'lms', 'cms', 'common', 'openedx', 'xmodule'} + # excluding XBlocks, which is handled by `pull_xblock_translations` command + - root_modules['xblock.v1'] + ) + + +def plugin_translations_atlas_pull(pull_options, locale_root): + """ + Atlas pull the translations for the installed non-XBlocks plugins. + """ + module_names = get_installed_plugins_module_names() + + atlas_pull_by_modules( + module_names=module_names, + locale_root=locale_root, + pull_options=pull_options, + ) diff --git a/openedx/core/djangoapps/plugins/management/__init__.py b/openedx/core/djangoapps/plugins/management/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/core/djangoapps/plugins/management/commands/__init__.py b/openedx/core/djangoapps/plugins/management/commands/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/core/djangoapps/plugins/management/commands/compile_plugin_translations.py b/openedx/core/djangoapps/plugins/management/commands/compile_plugin_translations.py new file mode 100644 index 000000000000..56322f104b1c --- /dev/null +++ b/openedx/core/djangoapps/plugins/management/commands/compile_plugin_translations.py @@ -0,0 +1,19 @@ +""" +Compile the translation files for the edx_django_utils.plugins. +""" + +from django.core.management.base import BaseCommand +from django.conf import settings + + +from ...constants import plugins_locale_root + +from ... import i18n_api + + +class Command(BaseCommand): + """ + Compile the translation files for the edx_django_utils.plugins. + """ + def handle(self, *args, **options): + i18n_api.compile_po_files(settings.REPO_ROOT / plugins_locale_root) diff --git a/openedx/core/djangoapps/plugins/management/commands/pull_plugin_translations.py b/openedx/core/djangoapps/plugins/management/commands/pull_plugin_translations.py new file mode 100644 index 000000000000..7ca4f443d079 --- /dev/null +++ b/openedx/core/djangoapps/plugins/management/commands/pull_plugin_translations.py @@ -0,0 +1,36 @@ +""" +Download the translations via atlas for the edx-platform plugins (edx_django_utils.plugins). + +For the XBlock command check the `pull_xblock_translations` command. +""" + +from django.conf import settings + +from openedx.core.djangoapps.plugins.i18n_api import BaseAtlasPullCommand + +from ...constants import plugins_locale_root + +from ...i18n_api import ( + plugin_translations_atlas_pull, +) + + +class Command(BaseAtlasPullCommand): + """ + Pull the edx_django_utils.plugins translations via atlas. + + For detailed information about atlas pull options check the atlas documentation: + + - https://github.com/openedx/openedx-atlas + """ + + def handle(self, *args, **options): + plugin_translations_root = settings.REPO_ROOT / plugins_locale_root + self.ensure_empty_directory(plugin_translations_root) + + atlas_pull_options = self.get_atlas_pull_options(**options) + + plugin_translations_atlas_pull( + pull_options=atlas_pull_options, + locale_root=plugin_translations_root, + ) diff --git a/openedx/core/djangoapps/plugins/settings/__init__.py b/openedx/core/djangoapps/plugins/settings/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/core/djangoapps/plugins/settings/production.py b/openedx/core/djangoapps/plugins/settings/production.py new file mode 100644 index 000000000000..7eb471fbd645 --- /dev/null +++ b/openedx/core/djangoapps/plugins/settings/production.py @@ -0,0 +1,17 @@ +""" +Production environment variables for `edx_django_utils.plugins` plugins. +""" + +from ..constants import plugins_locale_root + + +def plugin_settings(settings): + """ + Settings for the `edx_django_utils.plugins` plugins. + """ + locale_root = settings.REPO_ROOT / plugins_locale_root + if locale_root.isdir(): + for plugin_locale in locale_root.listdir(): + # Add the plugin locale directory only if it's a non-empty directory + if plugin_locale.isdir() and plugin_locale.listdir(): + settings.LOCALE_PATHS.append(plugin_locale) diff --git a/openedx/core/djangoapps/plugins/tests/test_commands.py b/openedx/core/djangoapps/plugins/tests/test_commands.py new file mode 100644 index 000000000000..65ce49b32adf --- /dev/null +++ b/openedx/core/djangoapps/plugins/tests/test_commands.py @@ -0,0 +1,49 @@ +""" +Tests for the plugins.i18n_api Django commands module. +""" +from unittest.mock import patch + +from django.core.management import call_command + + +def test_pull_plugin_translations_command(settings, tmp_path): + """ + Test the `pull_plugin_translations` Django command. + """ + plugins_locale_root = tmp_path / 'conf/plugins-locale/plugins' + plugins_locale_root.mkdir(parents=True) + settings.REPO_ROOT = tmp_path + + with patch('subprocess.run') as mock_run: + call_command( + 'pull_plugin_translations', + verbose=True, + filter='ar,es_ES', + repository='custom_repo', + ) + + assert mock_run.call_count == 1, 'Expected to call `subprocess.run` once' + call_kwargs = mock_run.call_args.kwargs + + assert call_kwargs['check'] is True + assert call_kwargs['cwd'] == plugins_locale_root + assert call_kwargs['args'][:8] == [ + 'atlas', 'pull', '--expand-glob', + '--filter', 'ar,es_ES', + '--repository', 'custom_repo', + '--verbose' + ], 'Pass arguments to atlas pull correctly' + + assert 'translations/*/edx_proctoring/conf/locale:edx_proctoring' in call_kwargs['args'], ( + 'Pull edx-proctoring translations by Python module name using the "--expand-glob" option' + ) + + +def test_compile_plugin_translations_command(settings): + """ + Test the `compile_plugin_translations` Django command. + """ + with patch('openedx.core.djangoapps.plugins.i18n_api.compile_po_files') as mock_compile_po_files: + call_command('compile_plugin_translations') + + mock_compile_po_files.assert_called_once_with(settings.REPO_ROOT / 'conf/plugins-locale/plugins') diff --git a/openedx/core/djangoapps/plugins/tests/test_i18n_api.py b/openedx/core/djangoapps/plugins/tests/test_i18n_api.py index 7e541799667b..0bbf77b600bb 100644 --- a/openedx/core/djangoapps/plugins/tests/test_i18n_api.py +++ b/openedx/core/djangoapps/plugins/tests/test_i18n_api.py @@ -1,12 +1,17 @@ """ Tests for the plugins.i18n_api module. """ - from unittest.mock import patch +import pytest +from django.core.management import CommandError + from ..i18n_api import ( ArgparseArgument, + BaseAtlasPullCommand, atlas_pull_by_modules, + compile_po_files, + get_installed_plugins_module_names, ) @@ -50,3 +55,61 @@ def test_atlas_pull_by_modules(): check=True, cwd=locale_root, ) + + +def test_compile_po_files(tmp_path): + """ + Test the compile_po_files recursive call to `msgfmt`. + """ + locale_root = tmp_path / 'locale' + locale_root.mkdir() + po_file_path = locale_root / 'test.po' + with open(po_file_path, 'w'): + # Creates an empty po file + pass + + with patch('subprocess.run') as mock_run: + compile_po_files(locale_root) + + mock_run.assert_called_once_with( + args=[ + 'msgfmt', '--check-format', + '-o', str(po_file_path.with_suffix('.mo')), + str(po_file_path), + ], + check=True, + ) + + +def test_base_atlas_pull_command(tmp_path): + """ + Test the BaseAtlasPullCommand's methods. + """ + command = BaseAtlasPullCommand() + + assert command.ensure_empty_directory(tmp_path) is None, 'Should not raise an exception if the directory is empty' + with pytest.raises(CommandError): + with open(tmp_path / 'test.txt', 'w'): + # Directory is not empty anymore + pass + command.ensure_empty_directory(tmp_path) + + assert command.get_atlas_pull_options( + filter='ar,jp_JP', + revision='custom_branch', + repository='my_org/custom_repo', + verbose=False, + ) == [ + '--filter', 'ar,jp_JP', '--repository', 'my_org/custom_repo', '--revision', 'custom_branch', '--silent', + ], 'Flatten out the options into a list of arguments for atlas pull' + + +def test_get_installed_plugins_module_names(): + """ + Test the get_installed_plugins_module_names helper. + """ + plugins = get_installed_plugins_module_names() + + assert 'drag_and_drop_v2' not in plugins, 'XBlocks have their own translation process' + assert 'edx_proctoring' in plugins, 'edx-proctoring should be included' + assert 'lms' not in plugins, 'lms and cms plugins are translated as part of the edx-platform itself'