From 235a0563699e3d718db874f3819a5f4cc976784f Mon Sep 17 00:00:00 2001 From: Arijit De Date: Sat, 25 Nov 2023 21:34:36 +0530 Subject: [PATCH 1/2] Added option to get resolved packages in a requirements.txt file Fixes #135 Created a new option --resolved-output that finds resolved packages and writes those into the given filename in a typical requirements.txt file format. Also added test test_resolved_cli in test_cli.py Signed-off-by: Arijit De --- src/python_inspector/resolve_cli.py | 13 ++++++ src/python_inspector/utils.py | 14 ++++++ .../resolved-pinned-requirements-expected.txt | 28 +++++++++++ tests/test_cli.py | 46 +++++++++++++++++++ 4 files changed, 101 insertions(+) create mode 100644 tests/data/resolved-pinned-requirements-expected.txt diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index cdd5abc..44d6f44 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -16,6 +16,7 @@ from python_inspector import utils_pypi from python_inspector.cli_utils import FileOptionType from python_inspector.utils import write_output_in_file +from python_inspector.utils import write_resolved_packages TRACE = False @@ -115,6 +116,15 @@ def print_version(ctx, param, value): help="Write output as pretty-printed JSON to FILE as a tree in the style of pipdeptree. " "Use the special '-' file name to print results on screen/stdout.", ) +@click.option( + "--resolved-output", + "resolved_output", + type=FileOptionType(mode="w", encoding="utf-8", lazy=True), + required=False, + metavar="FILE", + help="Write the packages that are resolved after inspecting, into a TEXT file." + "The packages are written in a typical requirements file format that can be used by pip.", +) @click.option( "-n", "--netrc", @@ -194,6 +204,7 @@ def resolve_dependencies( index_urls, json_output, pdt_output, + resolved_output, netrc_file, max_rounds, use_cached_index=False, @@ -285,6 +296,8 @@ def resolve_dependencies( output=output, location=json_output or pdt_output, ) + if resolved_output: + write_resolved_packages(files, resolved_output) except Exception: import traceback diff --git a/src/python_inspector/utils.py b/src/python_inspector/utils.py index 9bdf622..d7e89f8 100644 --- a/src/python_inspector/utils.py +++ b/src/python_inspector/utils.py @@ -54,6 +54,20 @@ def write_output_in_file(output, location): return output +def write_resolved_packages(package_list, requirements_file): + """ + Write the resolved package names and versions into ``requirements_file_path`` + """ + dependencies = package_list[0]["package_data"][0]["dependencies"] + resolved_packages = [] + for dependency in dependencies: + if dependency["is_resolved"]: + package = dependency["extracted_requirement"] + resolved_packages.append(package) + for package in resolved_packages: + requirements_file.write(package + "\n") + + class Candidate(NamedTuple): """ A candidate is a package that can be installed. diff --git a/tests/data/resolved-pinned-requirements-expected.txt b/tests/data/resolved-pinned-requirements-expected.txt new file mode 100644 index 0000000..fd9569c --- /dev/null +++ b/tests/data/resolved-pinned-requirements-expected.txt @@ -0,0 +1,28 @@ +aboutcode-toolkit==7.0.2 +attrs==21.4.0 +beautifulsoup4==4.11.1 +certifi==2022.5.18.1 +charset-normalizer==2.0.12 +click==8.0.4 +colorama==0.4.4 +commoncode==30.2.0 +dparse2==0.6.1 +idna==3.3 +importlib-metadata==4.8.3 +intbitset==3.0.1 +packageurl-python==0.9.9 +packaging==21.3 +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 +pyparsing==3.0.9 +PyYAML==6.0 +requests==2.27.1 +resolvelib==0.8.1 +saneyaml==0.5.2 +soupsieve==2.3.2.post1 +text-unidecode==1.3 +toml==0.10.2 +typing==3.6.6 +typing_extensions==4.1.1 +urllib3==1.26.9 +zipp==3.6.0 diff --git a/tests/test_cli.py b/tests/test_cli.py index 323e4bb..2401eaf 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -444,6 +444,52 @@ def test_passing_of_unsupported_os(): assert message in result.output +@pytest.mark.online +def test_resolved_cli(): + """ + Tests the '--resolved-output' option of resolve_dependencies() function. + The '--resolved-output' option takes in a filename and writes. + The resolved package dependencies into a .txt file with the given filename. + """ + requirements_file = test_env.get_test_loc("pinned-requirements.txt") + expected_file = test_env.get_test_loc( + "resolved-pinned-requirements-expected.txt", must_exist=False + ) + result_txt_file = test_env.get_temp_file("txt") + result_json_file = test_env.get_temp_file("json") + options = [ + "--requirement", + requirements_file, + "--resolved-output", + result_txt_file, + "--json", + result_json_file, + ] + run_cli(options=options) + check_resolved_packages_results(result_txt_file, expected_file) + + +def check_resolved_packages_results(result_file, expected_file, regen=REGEN_TEST_FIXTURES): + """ + Check the ``result_file`` data against the ``expected_file`` expected results. + + If ``regen`` is True the expected_file WILL BE overwritten with the new + results from ``results_file``. This is convenient for updating tests + expectations. + """ + with open(result_file, "r") as res_file: + results = res_file.readlines() + + if regen: + with open(expected_file, "w") as exo: + exo.writelines(results) + expected = results + else: + with open(expected_file) as reso: + expected = reso.readlines() + assert results == expected + + def check_requirements_resolution( requirements_file, expected_file, From 4cb1002bbb8e69b93146b435638571ba43219a88 Mon Sep 17 00:00:00 2001 From: Arijit De Date: Fri, 5 Jan 2024 15:41:53 +0530 Subject: [PATCH 2/2] Added option to get resolved packages in a requirements.txt file Fixes #135 Added unit tests for the function write_resolved_packages in test_utils.py Signed-off-by: Arijit De --- src/python_inspector/utils.py | 28 +++++++++++++++++++--------- tests/test_cli.py | 4 ++-- tests/test_utils.py | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/python_inspector/utils.py b/src/python_inspector/utils.py index d7e89f8..d79b7ee 100644 --- a/src/python_inspector/utils.py +++ b/src/python_inspector/utils.py @@ -56,16 +56,26 @@ def write_output_in_file(output, location): def write_resolved_packages(package_list, requirements_file): """ - Write the resolved package names and versions into ``requirements_file_path`` + Write the resolved package names and versions into `requirements_file` """ - dependencies = package_list[0]["package_data"][0]["dependencies"] - resolved_packages = [] - for dependency in dependencies: - if dependency["is_resolved"]: - package = dependency["extracted_requirement"] - resolved_packages.append(package) - for package in resolved_packages: - requirements_file.write(package + "\n") + print(package_list) + # if package list is a python list and not empty then proceed + if package_list and len(package_list) > 0: + try: + dependencies = package_list[0]["package_data"][0]["dependencies"] + resolved_packages = [] + for dependency in dependencies: + if dependency.get("is_resolved"): + package = dependency.get("extracted_requirement") + if package: + resolved_packages.append(package) + except KeyError: + return "Could not find dependencies key in package_list" + if len(resolved_packages) > 0: + for package in resolved_packages: + requirements_file.write(package + "\n") + else: + return "Package list is not a list or empty" class Candidate(NamedTuple): diff --git a/tests/test_cli.py b/tests/test_cli.py index 2401eaf..086486a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -448,8 +448,8 @@ def test_passing_of_unsupported_os(): def test_resolved_cli(): """ Tests the '--resolved-output' option of resolve_dependencies() function. - The '--resolved-output' option takes in a filename and writes. - The resolved package dependencies into a .txt file with the given filename. + The '--resolved-output' option takes in a filename and writes + the resolved package dependencies into a .txt file with the given filename. """ requirements_file = test_env.get_test_loc("pinned-requirements.txt") expected_file = test_env.get_test_loc( diff --git a/tests/test_utils.py b/tests/test_utils.py index b60f67a..9c85a71 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -20,6 +20,7 @@ from _packagedcode.pypi import SetupCfgHandler from python_inspector.resolution import fetch_and_extract_sdist from python_inspector.utils import get_netrc_auth +from python_inspector.utils import write_resolved_packages from python_inspector.utils_pypi import PypiSimpleRepository from python_inspector.utils_pypi import valid_python_version @@ -111,3 +112,21 @@ def test_parse_reqs_with_setup_requires_and_python_requires(): def test_valid_python_version(): assert valid_python_version("3.8", ">3.1") assert not valid_python_version("3.8.1", ">3.9") + + +def test_write_resolved_packages_with_empty_package(): + package_list = [] + requirements_file = test_env.get_test_loc("pinned-requirements.txt") + assert ( + write_resolved_packages(package_list, requirements_file) + == "Package list is not a list or empty" + ) + + +def test_write_resolved_packages_with_corrupt_package(): + package_list = [{"test": "wrong_value"}, {"list": ["list", "of", "values"]}] + requirements_file = test_env.get_test_loc("pinned-requirements.txt") + assert ( + write_resolved_packages(package_list, requirements_file) + == "Could not find dependencies key in package_list" + )