diff --git a/dcontainer/__main__.py b/dcontainer/__main__.py index 9c16dfbd..97909b12 100644 --- a/dcontainer/__main__.py +++ b/dcontainer/__main__.py @@ -1,7 +1,7 @@ import typer -from dcontainer.cli.install import app as install_app from dcontainer.cli.generate import app as generate_app +from dcontainer.cli.install import app as install_app from dcontainer.utils.version import ( resolve_own_package_version, resolve_own_release_version, diff --git a/dcontainer/apt_get/apt_get_installer.py b/dcontainer/apt_get/apt_get_installer.py index 694098ff..150cb160 100644 --- a/dcontainer/apt_get/apt_get_installer.py +++ b/dcontainer/apt_get/apt_get_installer.py @@ -1,45 +1,23 @@ -from typing import List, Optional, Type import platform -import invoke -import sys - - -class InteractiveSudoInvoker: - class InteractiveSudoInvokerException(Exception): - def __init__(self, command: str, response) -> None: - self.command = command - self.response = response - - def __str__(self): - return f"The command '{self.command}' failed. return_code: {self.response.return_code}. see logs for details." - - @staticmethod - def invoke( - command: str, - exception_class: Type["InteractiveSudoInvoker.InteractiveSudoInvokerException"], - ) -> None: - response = invoke.sudo( - command, out_stream=sys.stdout, err_stream=sys.stderr, pty=True - ) - if not response.ok: - raise exception_class(command=command, response=response) +from typing import List, Optional +from dcontainer.utils.sudo_invoker import SudoInvoker class AptGetInstaller: class PPASOnNonUbuntu(Exception): pass - class AptGetUpdateFailed(InteractiveSudoInvoker.InteractiveSudoInvokerException): + class AptGetUpdateFailed(SudoInvoker.InteractiveSudoInvokerException): pass - class AddPPAsFailed(InteractiveSudoInvoker.InteractiveSudoInvokerException): + class AddPPAsFailed(SudoInvoker.InteractiveSudoInvokerException): pass - class RemovePPAsFailed(InteractiveSudoInvoker.InteractiveSudoInvokerException): + class RemovePPAsFailed(SudoInvoker.InteractiveSudoInvokerException): pass - class CleanUpFailed(InteractiveSudoInvoker.InteractiveSudoInvokerException): + class CleanUpFailed(SudoInvoker.InteractiveSudoInvokerException): pass @staticmethod @@ -68,29 +46,29 @@ def install( normalized_ppas = AptGetInstaller.normalize_ppas(ppas) try: - InteractiveSudoInvoker.invoke( + SudoInvoker.invoke( command="apt-get update -y", exception_class=AptGetInstaller.AptGetUpdateFailed, ) if ppas: - InteractiveSudoInvoker.invoke( + SudoInvoker.invoke( command="apt-get install -y software-properties-common", exception_class=AptGetInstaller.AddPPAsFailed, ) for ppa in normalized_ppas: - InteractiveSudoInvoker.invoke( + SudoInvoker.invoke( command=f"add-apt-repository -y {ppa}", exception_class=AptGetInstaller.AddPPAsFailed, ) - InteractiveSudoInvoker.invoke( + SudoInvoker.invoke( command="apt-get update -y", exception_class=AptGetInstaller.AptGetUpdateFailed, ) - InteractiveSudoInvoker.invoke( + SudoInvoker.invoke( command=f"apt-get install -y --no-install-recommends {' '.join(packages)}", exception_class=AptGetInstaller.AptGetUpdateFailed, ) @@ -98,13 +76,13 @@ def install( finally: if remove_ppas_on_completion: for ppa in normalized_ppas: - InteractiveSudoInvoker.invoke( + SudoInvoker.invoke( command=f"add-apt-repository -y --remove {ppa}", exception_class=AptGetInstaller.RemovePPAsFailed, ) if remove_cache_on_completion: - InteractiveSudoInvoker.invoke( + SudoInvoker.invoke( command="rm -rf /var/lib/apt/lists/*", exception_class=AptGetInstaller.CleanUpFailed, ) diff --git a/dcontainer/cli/generate.py b/dcontainer/cli/generate.py index 6e85f58b..6ddb4744 100644 --- a/dcontainer/cli/generate.py +++ b/dcontainer/cli/generate.py @@ -4,7 +4,6 @@ import typer - logger = logging.getLogger(__name__) app = typer.Typer(pretty_exceptions_show_locals=False, pretty_exceptions_short=False) @@ -14,20 +13,22 @@ @app.command("devcontainer-feature") def generate_command( - feature_definition: pathlib.Path, - output_dir: pathlib.Path, - release_version: Optional[str] = None + feature_definition: pathlib.Path, + output_dir: pathlib.Path, + release_version: Optional[str] = None, ) -> None: try: - from dcontainer.devcontainer.feature_generation.oci_feature_generator import OCIFeatureGenerator + from dcontainer.devcontainer.feature_generation.oci_feature_generator import ( + OCIFeatureGenerator, + ) except ImportError as e: logger.error( "Some imports required for feature generation are missing.\nMake sure you have included the generate extras during installation.\n eg. 'pip install dcontainer[generate]'" ) raise typer.Exit(code=1) from e - OCIFeatureGenerator.generate(feature_definition=feature_definition.as_posix(), - output_dir=output_dir.as_posix(), - release_version=release_version) - - + OCIFeatureGenerator.generate( + feature_definition=feature_definition.as_posix(), + output_dir=output_dir.as_posix(), + release_version=release_version, + ) diff --git a/dcontainer/cli/install.py b/dcontainer/cli/install.py index 99084693..65aae617 100644 --- a/dcontainer/cli/install.py +++ b/dcontainer/cli/install.py @@ -1,5 +1,5 @@ import logging -from typing import List, Optional, Dict +from typing import Dict, List, Optional import typer @@ -21,13 +21,12 @@ def _validate_args(value: Optional[List[str]]): @app.command("devcontainer-feature") def install_devcontainer_feature( - feature: str, - option: Optional[List[str]] = typer.Option(None, callback=_validate_args), - remote_user: Optional[str] = typer.Option(None, callback=_validate_args), - env: Optional[List[str]] = typer.Option(None, callback=_validate_args), - verbose: bool = False, + feature: str, + option: Optional[List[str]] = typer.Option(None, callback=_validate_args), + remote_user: Optional[str] = typer.Option(None, callback=_validate_args), + env: Optional[List[str]] = typer.Option(None, callback=_validate_args), + verbose: bool = False, ) -> None: - from dcontainer.devcontainer.oci_feature_installer import OCIFeatureInstaller def _key_val_arg_to_dict(args: Optional[List[str]]) -> Dict[str, str]: @@ -38,7 +37,7 @@ def _key_val_arg_to_dict(args: Optional[List[str]]) -> Dict[str, str]: for single_arg in args: single_arg = _strip_if_wrapped_around(single_arg, '"') arg_name = single_arg.split("=")[0] - arg_value = single_arg[len(arg_name) + 1:] + arg_value = single_arg[len(arg_name) + 1 :] arg_value = _strip_if_wrapped_around(arg_value, '"') args_dict[arg_name] = arg_value return args_dict @@ -56,22 +55,29 @@ def _strip_if_wrapped_around(value: str, char: str) -> str: options_dict = _key_val_arg_to_dict(option) envs_dict = _key_val_arg_to_dict(env) - OCIFeatureInstaller.install(feature_ref=feature, envs=envs_dict, options=options_dict, remote_user=remote_user, - verbose=verbose) + OCIFeatureInstaller.install( + feature_ref=feature, + envs=envs_dict, + options=options_dict, + remote_user=remote_user, + verbose=verbose, + ) @app.command("apt-get") def install_apt_get_packages( - package: List[str], - ppa: Optional[List[str]] = typer.Option(None), - force_ppas_on_non_ubuntu: bool = True, - remove_ppas_on_completion: bool = True, - remove_cache_on_completion: bool = True, + package: List[str], + ppa: Optional[List[str]] = typer.Option(None), + force_ppas_on_non_ubuntu: bool = True, + remove_ppas_on_completion: bool = True, + remove_cache_on_completion: bool = True, ) -> None: from dcontainer.apt_get.apt_get_installer import AptGetInstaller - - AptGetInstaller.install(packages=package, ppas=ppa, - force_ppas_on_non_ubuntu=force_ppas_on_non_ubuntu, - remove_ppas_on_completion=remove_ppas_on_completion, - remove_cache_on_completion=remove_cache_on_completion, - ) + + AptGetInstaller.install( + packages=package, + ppas=ppa, + force_ppas_on_non_ubuntu=force_ppas_on_non_ubuntu, + remove_ppas_on_completion=remove_ppas_on_completion, + remove_cache_on_completion=remove_cache_on_completion, + ) diff --git a/dcontainer/devcontainer/feature_generation/dir_models/src_dir.py b/dcontainer/devcontainer/feature_generation/dir_models/src_dir.py index d53ea1be..13638157 100644 --- a/dcontainer/devcontainer/feature_generation/dir_models/src_dir.py +++ b/dcontainer/devcontainer/feature_generation/dir_models/src_dir.py @@ -1,18 +1,27 @@ -from easyfs import Directory from typing import Optional -from dcontainer.devcontainer.feature_generation.file_models.dependencies_sh import DependenciesSH +from easyfs import Directory + +from dcontainer.devcontainer.feature_generation.file_models.dependencies_sh import ( + DependenciesSH, +) from dcontainer.devcontainer.feature_generation.file_models.devcontainer_feature_json import ( DevcontainerFeatureJson, ) -from dcontainer.devcontainer.feature_generation.file_models.install_command_sh import InstallCommandSH +from dcontainer.devcontainer.feature_generation.file_models.install_command_sh import ( + InstallCommandSH, +) from dcontainer.devcontainer.feature_generation.file_models.install_sh import InstallSH -from dcontainer.devcontainer.models.devcontainer_feature_definition import FeatureDefinition +from dcontainer.devcontainer.models.devcontainer_feature_definition import ( + FeatureDefinition, +) class SrcDir(Directory): @classmethod - def from_definition_model(cls, definition_model: FeatureDefinition, release_version: Optional[str] = None) -> "Directory": + def from_definition_model( + cls, definition_model: FeatureDefinition, release_version: Optional[str] = None + ) -> "Directory": feature_id = definition_model.id virtual_dir = {} diff --git a/dcontainer/devcontainer/feature_generation/dir_models/test_dir.py b/dcontainer/devcontainer/feature_generation/dir_models/test_dir.py index fa938e99..12ed5bbb 100644 --- a/dcontainer/devcontainer/feature_generation/dir_models/test_dir.py +++ b/dcontainer/devcontainer/feature_generation/dir_models/test_dir.py @@ -1,6 +1,8 @@ from easyfs import Directory -from dcontainer.devcontainer.feature_generation.file_models.scenarios_json import ScenariosJson +from dcontainer.devcontainer.feature_generation.file_models.scenarios_json import ( + ScenariosJson, +) from dcontainer.devcontainer.feature_generation.file_models.test_sh import TestSH from dcontainer.devcontainer.models.devcontainer_feature_definition import ( FeatureDefinition, diff --git a/dcontainer/devcontainer/feature_generation/file_models/dependencies_sh.py b/dcontainer/devcontainer/feature_generation/file_models/dependencies_sh.py index 56ca3140..97bb7549 100644 --- a/dcontainer/devcontainer/feature_generation/file_models/dependencies_sh.py +++ b/dcontainer/devcontainer/feature_generation/file_models/dependencies_sh.py @@ -1,5 +1,6 @@ -from typing import Dict, Optional, Union import logging +from typing import Dict, Optional, Union + from easyfs import File from dcontainer.devcontainer.models.devcontainer_feature import FeatureOption diff --git a/dcontainer/devcontainer/feature_generation/file_models/devcontainer_feature_json.py b/dcontainer/devcontainer/feature_generation/file_models/devcontainer_feature_json.py index 1614839a..03758dbe 100644 --- a/dcontainer/devcontainer/feature_generation/file_models/devcontainer_feature_json.py +++ b/dcontainer/devcontainer/feature_generation/file_models/devcontainer_feature_json.py @@ -1,6 +1,8 @@ from easyfs import File -from dcontainer.devcontainer.models.devcontainer_feature_definition import FeatureDefinition +from dcontainer.devcontainer.models.devcontainer_feature_definition import ( + FeatureDefinition, +) class DevcontainerFeatureJson(File): diff --git a/dcontainer/devcontainer/feature_generation/oci_feature_generator.py b/dcontainer/devcontainer/feature_generation/oci_feature_generator.py index 9216e464..cbac3fb8 100644 --- a/dcontainer/devcontainer/feature_generation/oci_feature_generator.py +++ b/dcontainer/devcontainer/feature_generation/oci_feature_generator.py @@ -4,22 +4,25 @@ from dcontainer.devcontainer.feature_generation.dir_models.src_dir import SrcDir from dcontainer.devcontainer.feature_generation.dir_models.test_dir import TestDir -from dcontainer.devcontainer.models.devcontainer_feature_definition import FeatureDefinition +from dcontainer.devcontainer.models.devcontainer_feature_definition import ( + FeatureDefinition, +) class OCIFeatureGenerator: - @staticmethod - def generate(feature_definition: str, - output_dir: str, - release_version: Optional[str] = None) -> None: - + def generate( + feature_definition: str, output_dir: str, release_version: Optional[str] = None + ) -> None: definition_model = FeatureDefinition.parse_file(feature_definition) # create virtual file systm directory using easyfs virtual_dir = Directory() - virtual_dir["src"] = SrcDir.from_definition_model(definition_model=definition_model, - release_version=release_version) - virtual_dir["test"] = TestDir.from_definition_model(definition_model=definition_model) + virtual_dir["src"] = SrcDir.from_definition_model( + definition_model=definition_model, release_version=release_version + ) + virtual_dir["test"] = TestDir.from_definition_model( + definition_model=definition_model + ) # manifesting the virtual directory into local filesystem virtual_dir.create(output_dir) diff --git a/dcontainer/devcontainer/oci_feature.py b/dcontainer/devcontainer/oci_feature.py index 6767de04..77bf7741 100644 --- a/dcontainer/devcontainer/oci_feature.py +++ b/dcontainer/devcontainer/oci_feature.py @@ -8,7 +8,6 @@ class OCIFeature: - DEVCONTAINER_JSON_FILENAME = "devcontainer-feature.json" DEVCONTAINER_FILE_NAME_ANNOTATION = "org.opencontainers.image.title" @@ -32,18 +31,28 @@ def download(oci_feature_ref: str, output_dir: Union[str, Path]) -> str: file_location = output_dir.joinpath(file_name) - OCIRegistry.download_layer(oci_input=oci_feature_ref, layer_num=0, output_file=file_location, ) + OCIRegistry.download_layer( + oci_input=oci_feature_ref, + layer_num=0, + output_file=file_location, + ) return file_location.as_posix() @staticmethod - def download_and_extract(oci_feature_ref: str, output_dir: Union[str, Path]) -> None: - OCIRegistry.download_and_extract_layer(oci_input=oci_feature_ref, layer_num=0, output_dir=output_dir) + def download_and_extract( + oci_feature_ref: str, output_dir: Union[str, Path] + ) -> None: + OCIRegistry.download_and_extract_layer( + oci_input=oci_feature_ref, layer_num=0, output_dir=output_dir + ) @staticmethod def get_devcontainer_feature_obj(oci_feature_ref: str) -> Feature: with tempfile.TemporaryDirectory() as extraction_dir: - OCIFeature.download_and_extract(oci_feature_ref=oci_feature_ref, output_dir=extraction_dir) + OCIFeature.download_and_extract( + oci_feature_ref=oci_feature_ref, output_dir=extraction_dir + ) return Feature.parse_file( os.path.join(extraction_dir, OCIFeature.DEVCONTAINER_JSON_FILENAME) diff --git a/dcontainer/devcontainer/oci_feature_installer.py b/dcontainer/devcontainer/oci_feature_installer.py index b18d989e..33e3429a 100644 --- a/dcontainer/devcontainer/oci_feature_installer.py +++ b/dcontainer/devcontainer/oci_feature_installer.py @@ -53,7 +53,9 @@ def install( if envs is None: envs = {} - feature_obj = OCIFeature.get_devcontainer_feature_obj(oci_feature_ref=feature_ref) + feature_obj = OCIFeature.get_devcontainer_feature_obj( + oci_feature_ref=feature_ref + ) options = cls._resolve_options(feature_obj=feature_obj, options=options) logger.info("resolved options: %s", str(options)) @@ -98,7 +100,9 @@ def install( ) with tempfile.TemporaryDirectory() as tempdir: - OCIFeature.download_and_extract(oci_feature_ref=feature_ref, output_dir=tempdir) + OCIFeature.download_and_extract( + oci_feature_ref=feature_ref, output_dir=tempdir + ) sys.stdout.reconfigure( encoding="utf-8" @@ -113,7 +117,7 @@ def install( sudo {env_variables_cmd} bash -i {'-x' if verbose else ''} ./{cls._FEATURE_ENTRYPOINT}", out_stream=sys.stdout, err_stream=sys.stderr, - pty=True + pty=True, ) if not response.ok: diff --git a/dcontainer/oci/oci_registry.py b/dcontainer/oci/oci_registry.py index 51171e00..3b1082de 100644 --- a/dcontainer/oci/oci_registry.py +++ b/dcontainer/oci/oci_registry.py @@ -140,7 +140,9 @@ def _attempt_request( return urllib.request.urlopen(request) # nosec @staticmethod - def download_layer(oci_input: str, layer_num: int, output_file: Union[str, Path]) -> None: + def download_layer( + oci_input: str, layer_num: int, output_file: Union[str, Path] + ) -> None: if isinstance(output_file, str): output_file = Path(output_file) @@ -157,21 +159,25 @@ def download_layer(oci_input: str, layer_num: int, output_file: Union[str, Path] f.write(OCIRegistry.get_blob(oci_input, blob_digest)) @staticmethod - def download_and_extract_layer(oci_input: str, output_dir: Union[str, Path], layer_num: int) -> None: + def download_and_extract_layer( + oci_input: str, output_dir: Union[str, Path], layer_num: int + ) -> None: if isinstance(output_dir, str): output_dir = Path(output_dir) if output_dir.is_file(): raise ValueError(f"{output_dir} is a file (should be an empty directory)") - + output_dir.mkdir(parents=True, exist_ok=True) - + if any(output_dir.iterdir()): raise ValueError(f"{output_dir} is not empty ") with tempfile.TemporaryDirectory() as download_dir: layer_file = Path(download_dir).joinpath("layer_file.tgz") - OCIRegistry.download_layer(oci_input=oci_input, layer_num=layer_num, output_file=layer_file) + OCIRegistry.download_layer( + oci_input=oci_input, layer_num=layer_num, output_file=layer_file + ) with tarfile.open(layer_file, "r") as tar: tar.extractall(output_dir) diff --git a/dcontainer/utils/sudo_invoker.py b/dcontainer/utils/sudo_invoker.py new file mode 100644 index 00000000..eb1af8a2 --- /dev/null +++ b/dcontainer/utils/sudo_invoker.py @@ -0,0 +1,34 @@ +import sys +from typing import Type + +import invoke + +sys.stdout.reconfigure( + encoding="utf-8" +) # some processes will print in utf-8 while original stdout accept only ascii, causing a "UnicodeEncodeError: 'ascii' codec can't encode characters" error +sys.stderr.reconfigure( + encoding="utf-8" +) # some processes will print in utf-8 while original stdout accept only ascii, causing a "UnicodeEncodeError: 'ascii' codec can't encode characters" error + + +class SudoInvoker: + class SudoInvokerException(Exception): + def __init__(self, command: str, response) -> None: + self.command = command + self.response = response + + def __str__(self): + return f"The command '{self.command}' failed. return_code: {self.response.return_code}. see logs for details." + + @staticmethod + def invoke( + command: str, + exception_class: Type[ + "SudoInvoker.SudoInvokerException" + ] = SudoInvokerException, + ) -> None: + response = invoke.sudo( + command, out_stream=sys.stdout, err_stream=sys.stderr, pty=True + ) + if not response.ok: + raise exception_class(command=command, response=response) diff --git a/test/apt_get/test_apt_get_installer.py b/test/apt_get/test_apt_get_installer.py index ceaa4d32..f4ad077d 100644 --- a/test/apt_get/test_apt_get_installer.py +++ b/test/apt_get/test_apt_get_installer.py @@ -50,7 +50,7 @@ def generate_testing_devcontainer_feature(release_version: str, command: str, t @pytest.mark.parametrize( "packages,ppas,release_version,image", [ - (["neovim"], ["ppa:neovim-ppa/stable"], "v0.3.0rc2", "mcr.microsoft.com/devcontainers/base:ubuntu") + (["neovim"], ["ppa:neovim-ppa/stable"], "v0.3.0rc3", "mcr.microsoft.com/devcontainers/base:ubuntu") ], ) def test_apt_get_install(