diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8cb61d..6dedb84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,23 +1,24 @@ --- -'.test_common': &job_test_common +'.review': script: + - 'export TOXENV="${CI_JOB_NAME##review}"' - 'python3 -m pip install tox' - 'python3 -m tox' - 'python3 -m tox -e package' -'test py36': - <<: *job_test_common +'review py36': + extends: '.review' image: 'python:3.6' - variables: - 'TOXENV': 'py36' -'test py37': - <<: *job_test_common +'review py37': + extends: '.review' image: 'python:3.7' - variables: - 'TOXENV': 'py37' + +'review py38': + extends: '.review' + image: 'python:3.8' ... EOF diff --git a/.travis.yml b/.travis.yml index f9ee8fc..f894449 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ language: 'python' python: - '3.6' - '3.7' + - '3.8' install: - 'python3 -m pip install tox tox-travis tox-venv' diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bfb63c0..4615b62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,18 @@ .. Keep the current version number on line number 5 +0.0.6 +===== + +2020-04-20 + +* Add support for 'requirements.txt' files +* Fix call to main entry point of 'shiv', somewhat revert the change introduced + in version '0.0.5'. +* Show summary in CLI's help output +* Add test against Python3.8 in Travis CI and GitLab + + 0.0.5 ===== diff --git a/README.rst b/README.rst index b16127d..7b1edfc 100644 --- a/README.rst +++ b/README.rst @@ -44,13 +44,21 @@ By default this tool looks for a configuration file at the following location: [toolmaker.tool.pex:http] entry_point = http.server - requirements = [toolmaker.tool.shiv:shiv] entry_point = shiv.cli:main requirements = shiv + [toolmaker.tool.zapp:something] + entry_point = something.cli:main + requirements = + --no-index + SomeRandomProject --find-links /path/to/location + requirements_txts = + requirements.txt + more.txt + Action ------ @@ -59,8 +67,8 @@ The action can be specified on the command line. Either one of: * ``--build``, ``-b`` to build (already existing tools are skipped); * ``--rebuild``, ``-r`` to rebuild (already existing tools are rebuilt); -* ``--delete``, ``-d`` to delete (tool target file is deleted if it exists, then - its parent directory is deleted if it is empty). +* ``--delete``, ``-d`` to delete (tool target file is deleted if it exists, + then its parent directory is deleted if it is empty). The default action when no flag is specified is to build the tools. diff --git a/example.cfg b/example.cfg deleted file mode 100644 index 773707a..0000000 --- a/example.cfg +++ /dev/null @@ -1,22 +0,0 @@ -# - - -[toolmaker.tool.defaults] -tools_directory = - -[toolmaker.tool.zapp:deptree] -entry_point = deptree.cli:main -requirements = - deptree - -[toolmaker.tool.pex:http] -entry_point = http.server -requirements = - -[toolmaker.tool.shiv:shiv] -entry_point = shiv.cli:main -requirements = - shiv - - -# EOF diff --git a/examples/bin/.gitignore b/examples/bin/.gitignore new file mode 100644 index 0000000..f8b73e6 --- /dev/null +++ b/examples/bin/.gitignore @@ -0,0 +1,8 @@ +# + + +* +!.gitignore + + +# EOF diff --git a/examples/requirements.txt b/examples/requirements.txt new file mode 100644 index 0000000..f994d06 --- /dev/null +++ b/examples/requirements.txt @@ -0,0 +1 @@ +toolmaker diff --git a/examples/toolmaker.cfg b/examples/toolmaker.cfg new file mode 100644 index 0000000..f90e785 --- /dev/null +++ b/examples/toolmaker.cfg @@ -0,0 +1,41 @@ +# + + +[toolmaker.tool.defaults] +tools_directory = examples/bin + +[toolmaker.tool.pex:toolmaker_pex_req] +entry_point = toolmaker.cli:main +requirements = + toolmaker + +[toolmaker.tool.pex:toolmaker_pex_txt] +entry_point = toolmaker.cli:main +requirements_txts = + requirements.txt + +[toolmaker.tool.pex:http_pex] +entry_point = http.server + +[toolmaker.tool.shiv:toolmaker_shiv_req] +entry_point = toolmaker.cli:main +requirements = + toolmaker + +[toolmaker.tool.shiv:toolmaker_shiv_txt] +entry_point = toolmaker.cli:main +requirements_txts = + requirements.txt + +[toolmaker.tool.zapp:toolmaker_zapp_req] +entry_point = toolmaker.cli:main +requirements = + toolmaker + +[toolmaker.tool.zapp:toolmaker_zapp_txt] +entry_point = toolmaker.cli:main +requirements_txts = + requirements.txt + + +# EOF diff --git a/setup.cfg b/setup.cfg index 42a7eb3..55893c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ strict = 1 name = toolmaker author = sinoroc author_email = sinoroc.code+python@gmail.com -description = toolmaker application +description = Make single-file builds of Python tools using zapp, shiv, or pex license = Apache-2.0 license_file = LICENSE.txt long_description = file: README.rst diff --git a/src/toolmaker/_meta.py b/src/toolmaker/_meta.py index 1508bd6..0fdf06b 100644 --- a/src/toolmaker/_meta.py +++ b/src/toolmaker/_meta.py @@ -9,7 +9,10 @@ PROJECT_NAME = 'toolmaker' -VERSION = importlib_metadata.version(PROJECT_NAME) +_DISTRIBUTION_METADATA = importlib_metadata.metadata(PROJECT_NAME) + +SUMMARY = _DISTRIBUTION_METADATA['Summary'] +VERSION = _DISTRIBUTION_METADATA['Version'] # EOF diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index 174146b..f231455 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -17,6 +17,7 @@ def _create_args_parser(default_config_path, tools_names=None): args_parser = argparse.ArgumentParser( allow_abbrev=False, + description=_meta.SUMMARY, ) args_parser.add_argument( '--version', diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index 926e6da..9bed065 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -11,6 +11,7 @@ import os import pathlib import platform +import tempfile import pex.bin.pex import shiv @@ -28,65 +29,100 @@ class ConfigurationFileError(configparser.Error): """Configuration file error""" -def _pex(requirements, entry_point, output_file_path): - cmd = [ +def _pex(requirements_txts, entry_point, output_file_path): + command = [ '--entry-point={}'.format(entry_point), '--output-file={}'.format(str(output_file_path)), - ] + requirements - pex.bin.pex.main(cmd) + ] + [ + '--requirement={}'.format(requirements_txt) + for requirements_txt + in requirements_txts + ] + pex.bin.pex.main(command) -def _shiv(requirements, entry_point, output_file_path): +def _shiv(requirements_txts, entry_point, output_file_path): + pip_args = [] + for requirements_txt in requirements_txts: + pip_args.extend( + [ + '--requirement', + requirements_txt, + ], + ) # Since it is decorated by 'click', the 'main' function is not callable # with its original arguments. The original function is "hidden" under - # 'shiv.cli.main.callback', and 'shiv.cli.main' takes the equivalent of - # 'sys.argv' instead. - shiv.cli.main( # pylint: disable=no-value-for-parameter - [ - '--output-file', str(output_file_path), - '--entry-point', entry_point, # entry_point - '--python', '/usr/bin/env python3', - ] + requirements, + # 'shiv.cli.main.callback'. And 'shiv.cli.main' takes the equivalent of + # 'sys.argv' instead, but running it causes the whole application to exit + # at the end of the 'shiv.cli.main' function call. + shiv.cli.main.callback( + output_file=str(output_file_path), + entry_point=entry_point, + console_script=None, + python='/usr/bin/env python3', + site_packages=None, + compressed=False, + compile_pyc=False, + extend_pythonpath=False, + reproducible=False, + pip_args=pip_args, ) -def _zapp(requirements, entry_point, output_file_path): +def _zapp(requirements_txts, entry_point, output_file_path): zapp.core.build_zapp( output_file_path, entry_point, - requirements=requirements, + requirements_txts=requirements_txts, ) -def _get_requirements(config): - requirements = [ - req.strip() - for req in config['requirements'].splitlines() - if req.strip() - ] - return requirements +def _get_requirements_txts(tool_config, temp_requirements_txt): + requirements_txts = [] + # + requirements = tool_config.get('requirements', None) + if requirements: + temp_requirements_txt.write(requirements) + temp_requirements_txt.flush() + requirements_txts.append(temp_requirements_txt.name) + # + for line in tool_config.get('requirements_txts', '').splitlines(): + stripped_line = line.strip() + if stripped_line: + requirements_txt_path = pathlib.Path(stripped_line) + requirements_txts.append( + str(requirements_txt_path) + if requirements_txt_path.is_absolute() + else str( + tool_config['configuration_directory'].joinpath( + requirements_txt_path, + ) + ) + ) + # + return requirements_txts -def _get_file_path(config): +def _get_file_path(tool_config): path = ( pathlib.Path( - os.path.expandvars(config['tools_directory']), + os.path.expandvars(tool_config['tools_directory']), ).expanduser() - if 'tools_directory' in config + if 'tools_directory' in tool_config else pathlib.Path.cwd() ) - if 'tool_directory' in config: - tool_directory_name = config['tool_directory'] + if 'tool_directory' in tool_config: + tool_directory_name = tool_config['tool_directory'] if tool_directory_name: path = path.joinpath(tool_directory_name) else: - path = path.joinpath(config['name']) + path = path.joinpath(tool_config['name']) - if 'tool_file' in config: - path = path.joinpath(config['tool_file']) + if 'tool_file' in tool_config: + path = path.joinpath(tool_config['tool_file']) else: - path = path.joinpath(config['name']) + path = path.joinpath(tool_config['name']) if platform.system() == 'Windows': path = path.with_suffix('.pyz') @@ -94,47 +130,59 @@ def _get_file_path(config): return path -def _build_pex(config, force): +def _build_pex(tool_config, force): """ Build pex """ - tool_name = config['name'] - output_file_path = _get_file_path(config) + tool_name = tool_config['name'] + output_file_path = _get_file_path(tool_config) if force or not output_file_path.exists(): LOGGER.info("Building pex tool '%s'...", tool_name) - requirements = _get_requirements(config) - entry_point = config['entry_point'] + entry_point = tool_config['entry_point'] output_file_path.parent.mkdir(exist_ok=True) - _pex(requirements, entry_point, output_file_path) + with tempfile.NamedTemporaryFile('w') as temp_requirements_txt: + requirements_txts = _get_requirements_txts( + tool_config, + temp_requirements_txt, + ) + _pex(requirements_txts, entry_point, output_file_path) else: LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def _build_shiv(config, force): +def _build_shiv(tool_config, force): """ Build shiv """ - tool_name = config['name'] - output_file_path = _get_file_path(config) + tool_name = tool_config['name'] + output_file_path = _get_file_path(tool_config) if force or not output_file_path.exists(): LOGGER.info("Building shiv tool '%s'...", tool_name) - requirements = _get_requirements(config) - entry_point = config['entry_point'] + entry_point = tool_config['entry_point'] output_file_path.parent.mkdir(exist_ok=True) - _shiv(requirements, entry_point, output_file_path) + with tempfile.NamedTemporaryFile('w') as temp_requirements_txt: + requirements_txts = _get_requirements_txts( + tool_config, + temp_requirements_txt, + ) + _shiv(requirements_txts, entry_point, output_file_path) else: LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def _build_zapp(config, force): +def _build_zapp(tool_config, force): """ Build zapp """ - tool_name = config['name'] - output_file_path = _get_file_path(config) + tool_name = tool_config['name'] + output_file_path = _get_file_path(tool_config) if force or not output_file_path.exists(): LOGGER.info("Building zapp tool '%s'...", tool_name) - requirements = _get_requirements(config) - entry_point = config['entry_point'] + entry_point = tool_config['entry_point'] output_file_path.parent.mkdir(exist_ok=True) - _zapp(requirements, entry_point, output_file_path) + with tempfile.NamedTemporaryFile('w') as temp_requirements_txt: + requirements_txts = _get_requirements_txts( + tool_config, + temp_requirements_txt, + ) + _zapp(requirements_txts, entry_point, output_file_path) else: LOGGER.info("Tool '%s' already exists, build skipped", tool_name) @@ -176,12 +224,17 @@ def parse_config(config_file): ) raise ConfigurationFileError(str(config_error)) else: + config_directory_path = pathlib.Path(config_file.name).resolve().parent config = { 'tools': {}, } for section_name in raw_config.sections(): - tool_config = _parse_tool_config(raw_config, section_name) + tool_config = _parse_tool_config( + raw_config, + section_name, + ) if tool_config: + tool_config['configuration_directory'] = config_directory_path config['tools'][tool_config['name']] = tool_config return config