From 0375fc72e904c6076c8d9ad5b814ffee58520a96 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Thu, 10 Oct 2024 18:09:47 +0200 Subject: [PATCH 1/2] Generate project object on demand This is a lot cleaner, and has better performance benefits. Commands like 'download' do not always require a project object. Now, if the code never touches 'obj.project', the object is never created. Apart from performance benefits, it also means that irrelevant errors spawned from creation of the project object are sidestepped, because the object is never instantiated (in some cases). Signed-off-by: Carmen Bianca BAKKER --- src/reuse/cli/annotate.py | 5 ++- src/reuse/cli/common.py | 65 ++++++++++++++++++++++++++--------- src/reuse/cli/convert_dep5.py | 6 ++-- src/reuse/cli/download.py | 10 ++---- src/reuse/cli/lint.py | 7 ++-- src/reuse/cli/lint_file.py | 8 ++--- src/reuse/cli/main.py | 33 ++---------------- src/reuse/cli/spdx.py | 8 ++--- 8 files changed, 67 insertions(+), 75 deletions(-) diff --git a/src/reuse/cli/annotate.py b/src/reuse/cli/annotate.py index 301593a47..c803e3341 100644 --- a/src/reuse/cli/annotate.py +++ b/src/reuse/cli/annotate.py @@ -44,7 +44,7 @@ ) from ..i18n import _ from ..project import Project -from .common import ClickObj, MutexOption, requires_project, spdx_identifier +from .common import ClickObj, MutexOption, spdx_identifier from .main import main _LOGGER = logging.getLogger(__name__) @@ -285,7 +285,6 @@ def get_reuse_info( ) -@requires_project @main.command(name="annotate", help=_HELP) @click.option( "--copyright", @@ -449,7 +448,7 @@ def annotate( paths: Sequence[Path], ) -> None: # pylint: disable=too-many-arguments,too-many-locals,missing-function-docstring - project = cast(Project, obj.project) + project = obj.project test_mandatory_option_required(copyrights, licenses, contributors) paths = all_paths(paths, recursive, project) diff --git a/src/reuse/cli/common.py b/src/reuse/cli/common.py index 88189bfcf..0e185f274 100644 --- a/src/reuse/cli/common.py +++ b/src/reuse/cli/common.py @@ -4,34 +4,67 @@ """Utilities that are common to multiple CLI commands.""" -from dataclasses import dataclass -from typing import Any, Callable, Mapping, Optional, TypeVar +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping, Optional import click from boolean.boolean import Expression, ParseError from license_expression import ExpressionError from .._util import _LICENSING +from ..global_licensing import GlobalLicensingParseError from ..i18n import _ -from ..project import Project +from ..project import GlobalLicensingConflict, Project +from ..vcs import find_root -F = TypeVar("F", bound=Callable) - -def requires_project(f: F) -> F: - """A decorator to mark subcommands that require a :class:`Project` object. - Make sure to apply this decorator _first_. - """ - setattr(f, "requires_project", True) - return f - - -@dataclass(frozen=True) +@dataclass() class ClickObj: """A dataclass holding necessary context and options.""" - no_multiprocessing: bool - project: Optional[Project] + root: Optional[Path] = None + include_submodules: bool = False + include_meson_subprojects: bool = False + no_multiprocessing: bool = True + + _project: Optional[Project] = field( + default=None, init=False, repr=False, compare=False + ) + + @property + def project(self) -> Project: + """Generate a project object on demand, and cache it.""" + if self._project: + return self._project + + root = self.root + if root is None: + root = find_root() + if root is None: + root = Path.cwd() + + try: + project = Project.from_directory( + root, + include_submodules=self.include_submodules, + include_meson_subprojects=self.include_meson_subprojects, + ) + # FileNotFoundError and NotADirectoryError don't need to be caught + # because argparse already made sure of these things. + except GlobalLicensingParseError as error: + raise click.UsageError( + _( + "'{path}' could not be parsed. We received the" + " following error message: {message}" + ).format(path=error.source, message=str(error)) + ) from error + + except (GlobalLicensingConflict, OSError) as error: + raise click.UsageError(str(error)) from error + + self._project = project + return project class MutexOption(click.Option): diff --git a/src/reuse/cli/convert_dep5.py b/src/reuse/cli/convert_dep5.py index c7bec0263..a4cb32f7c 100644 --- a/src/reuse/cli/convert_dep5.py +++ b/src/reuse/cli/convert_dep5.py @@ -12,8 +12,7 @@ from ..convert_dep5 import toml_from_dep5 from ..global_licensing import ReuseDep5 from ..i18n import _ -from ..project import Project -from .common import ClickObj, requires_project +from .common import ClickObj from .main import main _HELP = _( @@ -23,12 +22,11 @@ ) -@requires_project @main.command(name="convert-dep5", help=_HELP) @click.pass_obj def convert_dep5(obj: ClickObj) -> None: # pylint: disable=missing-function-docstring - project = cast(Project, obj.project) + project = obj.project if not (project.root / ".reuse/dep5").exists(): raise click.UsageError(_("No '.reuse/dep5' file.")) diff --git a/src/reuse/cli/download.py b/src/reuse/cli/download.py index 55d3998a9..b217eb78b 100644 --- a/src/reuse/cli/download.py +++ b/src/reuse/cli/download.py @@ -9,7 +9,7 @@ import sys from difflib import SequenceMatcher from pathlib import Path -from typing import IO, Collection, Optional, cast +from typing import IO, Collection, Optional from urllib.error import URLError import click @@ -17,10 +17,9 @@ from .._licenses import ALL_NON_DEPRECATED_MAP from ..download import _path_to_license_file, put_license_in_file from ..i18n import _ -from ..project import Project from ..report import ProjectReport from ..types import StrPath -from .common import ClickObj, MutexOption, requires_project +from .common import ClickObj, MutexOption from .main import main _LOGGER = logging.getLogger(__name__) @@ -113,7 +112,6 @@ def _successfully_downloaded(destination: StrPath) -> None: ) -@requires_project @main.command(name="download", help=_HELP) @click.option( "--all", @@ -166,9 +164,7 @@ def download( if all_: # TODO: This is fairly inefficient, but gets the job done. - report = ProjectReport.generate( - cast(Project, obj.project), do_checksum=False - ) + report = ProjectReport.generate(obj.project, do_checksum=False) licenses = report.missing_licenses.keys() if len(licenses) > 1 and output: diff --git a/src/reuse/cli/lint.py b/src/reuse/cli/lint.py index 0509ce0df..5a5cd3122 100644 --- a/src/reuse/cli/lint.py +++ b/src/reuse/cli/lint.py @@ -10,16 +10,14 @@ """Click code for lint subcommand.""" import sys -from typing import cast import click from .. import __REUSE_version__ from ..i18n import _ from ..lint import format_json, format_lines, format_plain -from ..project import Project from ..report import ProjectReport -from .common import ClickObj, MutexOption, requires_project +from .common import ClickObj, MutexOption from .main import main _OUTPUT_MUTEX = ["quiet", "json", "plain", "lines"] @@ -62,7 +60,6 @@ ) -@requires_project @main.command(name="lint", help=_HELP) @click.option( "--quiet", @@ -102,7 +99,7 @@ def lint( ) -> None: # pylint: disable=missing-function-docstring report = ProjectReport.generate( - cast(Project, obj.project), + obj.project, do_checksum=False, multiprocessing=not obj.no_multiprocessing, ) diff --git a/src/reuse/cli/lint_file.py b/src/reuse/cli/lint_file.py index b6e8bd81a..b87021550 100644 --- a/src/reuse/cli/lint_file.py +++ b/src/reuse/cli/lint_file.py @@ -9,15 +9,14 @@ import sys from pathlib import Path -from typing import Collection, cast +from typing import Collection import click from ..i18n import _ from ..lint import format_lines_subset -from ..project import Project from ..report import ProjectSubsetReport -from .common import ClickObj, MutexOption, requires_project +from .common import ClickObj, MutexOption from .main import main _OUTPUT_MUTEX = ["quiet", "lines"] @@ -29,7 +28,6 @@ ) -@requires_project @main.command(name="lint-file", help=_HELP) @click.option( "--quiet", @@ -58,7 +56,7 @@ def lint_file( obj: ClickObj, quiet: bool, lines: bool, files: Collection[Path] ) -> None: # pylint: disable=missing-function-docstring - project = cast(Project, obj.project) + project = obj.project subset_files = {Path(file_) for file_ in files} for file_ in subset_files: if not file_.resolve().is_relative_to(project.root.resolve()): diff --git a/src/reuse/cli/main.py b/src/reuse/cli/main.py index e85248906..b2cdd72d7 100644 --- a/src/reuse/cli/main.py +++ b/src/reuse/cli/main.py @@ -21,10 +21,7 @@ from .. import __REUSE_version__ from .._util import setup_logging -from ..global_licensing import GlobalLicensingParseError from ..i18n import _ -from ..project import GlobalLicensingConflict, Project -from ..vcs import find_root from .common import ClickObj _PACKAGE_PATH = os.path.dirname(__file__) @@ -146,33 +143,9 @@ def main( if not suppress_deprecation: warnings.filterwarnings("default", module="reuse") - project: Optional[Project] = None - if ctx.invoked_subcommand: - cmd = main.get_command(ctx, ctx.invoked_subcommand) - if getattr(cmd, "requires_project", False): - if root is None: - root = find_root() - if root is None: - root = Path.cwd() - - try: - project = Project.from_directory(root) - # FileNotFoundError and NotADirectoryError don't need to be caught - # because argparse already made sure of these things. - except GlobalLicensingParseError as error: - raise click.UsageError( - _( - "'{path}' could not be parsed. We received the" - " following error message: {message}" - ).format(path=error.source, message=str(error)) - ) from error - - except (GlobalLicensingConflict, OSError) as error: - raise click.UsageError(str(error)) from error - project.include_submodules = include_submodules - project.include_meson_subprojects = include_meson_subprojects - ctx.obj = ClickObj( + root=root, + include_submodules=include_submodules, + include_meson_subprojects=include_meson_subprojects, no_multiprocessing=no_multiprocessing, - project=project, ) diff --git a/src/reuse/cli/spdx.py b/src/reuse/cli/spdx.py index 9c69477bb..11a9933b5 100644 --- a/src/reuse/cli/spdx.py +++ b/src/reuse/cli/spdx.py @@ -8,15 +8,14 @@ import contextlib import logging import sys -from typing import Optional, cast +from typing import Optional import click from .. import _IGNORE_SPDX_PATTERNS from ..i18n import _ -from ..project import Project from ..report import ProjectReport -from .common import ClickObj, requires_project +from .common import ClickObj from .main import main _LOGGER = logging.getLogger(__name__) @@ -24,7 +23,6 @@ _HELP = _("Generate an SPDX bill of materials.") -@requires_project @main.command(name="spdx", help=_HELP) @click.option( "--output", @@ -103,7 +101,7 @@ def spdx( ) report = ProjectReport.generate( - cast(Project, obj.project), + obj.project, multiprocessing=not obj.no_multiprocessing, add_license_concluded=add_license_concluded, ) From 6d9c844349d214e8b23fcc2b5639d150b46fa8eb Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Thu, 10 Oct 2024 18:14:26 +0200 Subject: [PATCH 2/2] Set up Python in gettext workflow Without this, poetry is not available. Signed-off-by: Carmen Bianca BAKKER --- .github/workflows/gettext.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/gettext.yaml b/.github/workflows/gettext.yaml index b2d9c0c0c..6b2af961b 100644 --- a/.github/workflows/gettext.yaml +++ b/.github/workflows/gettext.yaml @@ -23,6 +23,10 @@ jobs: # exception to the branch protection, so we'll use that account's # token to push to the main branch. token: ${{ secrets.FSFE_SYSTEM_TOKEN }} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" - name: Install gettext and wlc run: sudo apt-get install -y gettext wlc # We mostly install reuse to install the click dependency.