Merge pull request aboutcode-org#125 from nexB/fix_124

Fix resolution of setup files which partially have dynamic dependencies
bennati · Apr 12, 2023 · cb19215 · cb19215
2 parents 271921d + 3bf0a3a
commit cb19215
Show file tree

Hide file tree

Showing 11 changed files with 350 additions and 431 deletions.
diff --git a/src/python_inspector/api.py b/src/python_inspector/api.py
@@ -37,6 +37,7 @@
 from python_inspector.resolution import get_environment_marker_from_environment
 from python_inspector.resolution import get_package_list
 from python_inspector.resolution import get_python_version_from_env_tag
+from python_inspector.resolution import get_reqs_insecurely
 from python_inspector.resolution import get_requirements_from_python_manifest
 from python_inspector.utils_pypi import PLATFORMS_BY_OS
 from python_inspector.utils_pypi import PYPI_SIMPLE_URL
@@ -175,22 +176,30 @@ def resolve_dependencies(
                 f"is not compatible with setup.py {setup_py_file} "
                 f"python_requires {python_requires}",
             )
-
-        setup_py_file_deps = package_data.dependencies
-        for dep in package_data.dependencies:
-            # TODO : we need to handle to all the scopes
-            if dep.scope == "install":
-                direct_dependencies.append(dep)
-
-        if not package_data.dependencies:
-            reqs = get_requirements_from_python_manifest(
-                sdist_location=os.path.dirname(setup_py_file),
-                setup_py_location=setup_py_file,
-                files=[setup_py_file],
-                analyze_setup_py_insecurely=analyze_setup_py_insecurely,
+        if analyze_setup_py_insecurely:
+            reqs = list(
+                get_reqs_insecurely(
+                    setup_py_location=setup_py_file,
+                )
             )
             setup_py_file_deps = list(get_dependent_packages_from_reqs(reqs))
             direct_dependencies.extend(setup_py_file_deps)
+        else:
+            setup_py_file_deps = package_data.dependencies
+            for dep in package_data.dependencies:
+                # TODO : we need to handle to all the scopes
+                if dep.scope == "install":
+                    direct_dependencies.append(dep)
+
+            if not package_data.dependencies:
+                reqs = get_requirements_from_python_manifest(
+                    sdist_location=os.path.dirname(setup_py_file),
+                    setup_py_location=setup_py_file,
+                    files=[setup_py_file],
+                    analyze_setup_py_insecurely=analyze_setup_py_insecurely,
+                )
+                setup_py_file_deps = list(get_dependent_packages_from_reqs(reqs))
+                direct_dependencies.extend(setup_py_file_deps)
 
         package_data.dependencies = setup_py_file_deps
         file_package_data = [package_data.to_dict()]

diff --git a/src/python_inspector/resolution.py b/src/python_inspector/resolution.py
@@ -493,25 +493,30 @@ def get_requirements_for_package_from_pypi_simple(
                 "setup.cfg",
             )
 
-            requirements = list(
-                get_setup_requirements(
-                    sdist_location=sdist_location,
+            if self.analyze_setup_py_insecurely:
+                yield from get_reqs_insecurely(
                     setup_py_location=setup_py_location,
-                    setup_cfg_location=setup_cfg_location,
                 )
-            )
-            if requirements:
-                yield from requirements
             else:
-                # Look in requirements file if and only if thy are refered in setup.py or setup.cfg
-                # And no deps have been yielded by requirements file
-
-                yield from get_requirements_from_python_manifest(
-                    sdist_location=sdist_location,
-                    setup_py_location=setup_py_location,
-                    files=[setup_cfg_location, setup_py_location],
-                    analyze_setup_py_insecurely=self.analyze_setup_py_insecurely,
+                requirements = list(
+                    get_setup_requirements(
+                        sdist_location=sdist_location,
+                        setup_py_location=setup_py_location,
+                        setup_cfg_location=setup_cfg_location,
+                    )
                 )
+                if requirements:
+                    yield from requirements
+                else:
+                    # Look in requirements file if and only if thy are refered in setup.py or setup.cfg
+                    # And no deps have been yielded by requirements file
+
+                    yield from get_requirements_from_python_manifest(
+                        sdist_location=sdist_location,
+                        setup_py_location=setup_py_location,
+                        files=[setup_cfg_location, setup_py_location],
+                        analyze_setup_py_insecurely=self.analyze_setup_py_insecurely,
+                    )
 
     def get_requirements_for_package_from_pypi_json_api(
         self, purl: PackageURL

diff --git a/tests/data/azure-devops.req-310-expected.json b/tests/data/azure-devops.req-310-expected.json
diff --git a/tests/data/azure-devops.req-38-expected.json b/tests/data/azure-devops.req-38-expected.json
diff --git a/tests/data/insecure-setup-2/setup.py-expected.json b/tests/data/insecure-setup-2/setup.py-expected.json
diff --git a/tests/data/partial-setup.py b/tests/data/partial-setup.py
@@ -0,0 +1,12 @@
+from setuptools import setup
+
+semver_version = "2.13.0"
+
+setup(
+    name="example",
+    version="0.0.1",
+    install_requires=[
+        f"semver @ git+https://github.com/python-semver/python-semver.git@{semver_version}",
+    ],
+    extras_require={"test": ["botocore==1.27.76"]},
+)
diff --git a/tests/data/setup/spdx-setup.py-expected.json b/tests/data/setup/spdx-setup.py-expected.json
@@ -75,7 +75,7 @@
               "purl": "pkg:pypi/ply",
               "extracted_requirement": "ply",
               "scope": "install",
-              "is_runtime": true,
+              "is_runtime": false,
               "is_optional": false,
               "is_resolved": false,
               "resolved_package": {},
@@ -85,7 +85,7 @@
               "purl": "pkg:pypi/rdflib",
               "extracted_requirement": "rdflib",
               "scope": "install",
-              "is_runtime": true,
+              "is_runtime": false,
               "is_optional": false,
               "is_resolved": false,
               "resolved_package": {},
@@ -95,7 +95,7 @@
               "purl": "pkg:pypi/six",
               "extracted_requirement": "six",
               "scope": "install",
-              "is_runtime": true,
+              "is_runtime": false,
               "is_optional": false,
               "is_resolved": false,
               "resolved_package": {},

diff --git a/tests/data/single-url-except-simple-expected.json b/tests/data/single-url-except-simple-expected.json
@@ -276,12 +276,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "importlib-metadata",
-      "version": "6.1.0",
+      "version": "6.3.0",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "Read metadata from Python packages\n.. image:: https://img.shields.io/pypi/v/importlib_metadata.svg\n   :target: https://pypi.org/project/importlib_metadata\n\n.. image:: https://img.shields.io/pypi/pyversions/importlib_metadata.svg\n\n.. image:: https://github.com/python/importlib_metadata/workflows/tests/badge.svg\n   :target: https://github.com/python/importlib_metadata/actions?query=workflow%3A%22tests%22\n   :alt: tests\n\n.. image:: https://img.shields.io/badge/code%20style-black-000000.svg\n   :target: https://github.com/psf/black\n   :alt: Code style: Black\n\n.. image:: https://readthedocs.org/projects/importlib-metadata/badge/?version=latest\n   :target: https://importlib-metadata.readthedocs.io/en/latest/?badge=latest\n\n.. image:: https://img.shields.io/badge/skeleton-2023-informational\n   :target: https://blog.jaraco.com/skeleton\n\n.. image:: https://tidelift.com/badges/package/pypi/importlib-metadata\n   :target: https://tidelift.com/subscription/pkg/pypi-importlib-metadata?utm_source=pypi-importlib-metadata&utm_medium=readme\n\nLibrary to access the metadata for a Python package.\n\nThis package supplies third-party access to the functionality of\n`importlib.metadata <https://docs.python.org/3/library/importlib.metadata.html>`_\nincluding improvements added to subsequent Python versions.\n\n\nCompatibility\n=============\n\nNew features are introduced in this third-party library and later merged\ninto CPython. The following table indicates which versions of this library\nwere contributed to different versions in the standard library:\n\n.. list-table::\n   :header-rows: 1\n\n   * - importlib_metadata\n     - stdlib\n   * - 5.0\n     - 3.12\n   * - 4.13\n     - 3.11\n   * - 4.6\n     - 3.10\n   * - 1.4\n     - 3.8\n\n\nUsage\n=====\n\nSee the `online documentation <https://importlib_metadata.readthedocs.io/>`_\nfor usage details.\n\n`Finder authors\n<https://docs.python.org/3/reference/import.html#finders-and-loaders>`_ can\nalso add support for custom package installers.  See the above documentation\nfor details.\n\n\nCaveats\n=======\n\nThis project primarily supports third-party packages installed by PyPA\ntools (or other conforming packages). It does not support:\n\n- Packages in the stdlib.\n- Packages installed without metadata.\n\nProject details\n===============\n\n * Project home: https://github.com/python/importlib_metadata\n * Report bugs at: https://github.com/python/importlib_metadata/issues\n * Code hosting: https://github.com/python/importlib_metadata\n * Documentation: https://importlib_metadata.readthedocs.io/\n\nFor Enterprise\n==============\n\nAvailable as part of the Tidelift Subscription.\n\nThis project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.\n\n`Learn more <https://tidelift.com/subscription/pkg/pypi-importlib-metadata?utm_source=pypi-importlib-metadata&utm_medium=referral&utm_campaign=github>`_.\n\nSecurity Contact\n================\n\nTo report a security vulnerability, please use the\n`Tidelift security contact <https://tidelift.com/security>`_.\nTidelift will coordinate the fix and disclosure.",
-      "release_date": "2023-03-18T17:10:47",
+      "release_date": "2023-04-10T02:27:40",
       "parties": [
         {
           "type": "person",
@@ -298,11 +298,11 @@
         "Programming Language :: Python :: 3 :: Only"
       ],
       "homepage_url": "https://github.com/python/importlib_metadata",
-      "download_url": "https://files.pythonhosted.org/packages/f8/7d/e3adad613703c86d62aa991b45d6f090cf59975078a8c8100b50a0c86948/importlib_metadata-6.1.0-py3-none-any.whl",
-      "size": 21918,
+      "download_url": "https://files.pythonhosted.org/packages/af/15/544ee37359dd4d8e490d1846062015f9d7d59b0f11e2e8e629917608e592/importlib_metadata-6.3.0-py3-none-any.whl",
+      "size": 22533,
       "sha1": null,
-      "md5": "a952f41348b249647623b1f32c152c65",
-      "sha256": "ff80f3b5394912eb1b108fcfd444dc78b7f1f3e16b16188054bd01cb9cb86f09",
+      "md5": "fa927011905878792c54886f35e59bca",
+      "sha256": "8f8bd2af397cf33bd344d35cfe7f489219b7d14fc79a3f854b75b8417e9226b0",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": null,
@@ -321,20 +321,20 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/importlib-metadata/6.1.0/json",
+      "api_data_url": "https://pypi.org/pypi/importlib-metadata/6.3.0/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/importlib-metadata@6.1.0"
+      "purl": "pkg:pypi/importlib-metadata@6.3.0"
     },
     {
       "type": "pypi",
       "namespace": null,
       "name": "importlib-metadata",
-      "version": "6.1.0",
+      "version": "6.3.0",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "Read metadata from Python packages\n.. image:: https://img.shields.io/pypi/v/importlib_metadata.svg\n   :target: https://pypi.org/project/importlib_metadata\n\n.. image:: https://img.shields.io/pypi/pyversions/importlib_metadata.svg\n\n.. image:: https://github.com/python/importlib_metadata/workflows/tests/badge.svg\n   :target: https://github.com/python/importlib_metadata/actions?query=workflow%3A%22tests%22\n   :alt: tests\n\n.. image:: https://img.shields.io/badge/code%20style-black-000000.svg\n   :target: https://github.com/psf/black\n   :alt: Code style: Black\n\n.. image:: https://readthedocs.org/projects/importlib-metadata/badge/?version=latest\n   :target: https://importlib-metadata.readthedocs.io/en/latest/?badge=latest\n\n.. image:: https://img.shields.io/badge/skeleton-2023-informational\n   :target: https://blog.jaraco.com/skeleton\n\n.. image:: https://tidelift.com/badges/package/pypi/importlib-metadata\n   :target: https://tidelift.com/subscription/pkg/pypi-importlib-metadata?utm_source=pypi-importlib-metadata&utm_medium=readme\n\nLibrary to access the metadata for a Python package.\n\nThis package supplies third-party access to the functionality of\n`importlib.metadata <https://docs.python.org/3/library/importlib.metadata.html>`_\nincluding improvements added to subsequent Python versions.\n\n\nCompatibility\n=============\n\nNew features are introduced in this third-party library and later merged\ninto CPython. The following table indicates which versions of this library\nwere contributed to different versions in the standard library:\n\n.. list-table::\n   :header-rows: 1\n\n   * - importlib_metadata\n     - stdlib\n   * - 5.0\n     - 3.12\n   * - 4.13\n     - 3.11\n   * - 4.6\n     - 3.10\n   * - 1.4\n     - 3.8\n\n\nUsage\n=====\n\nSee the `online documentation <https://importlib_metadata.readthedocs.io/>`_\nfor usage details.\n\n`Finder authors\n<https://docs.python.org/3/reference/import.html#finders-and-loaders>`_ can\nalso add support for custom package installers.  See the above documentation\nfor details.\n\n\nCaveats\n=======\n\nThis project primarily supports third-party packages installed by PyPA\ntools (or other conforming packages). It does not support:\n\n- Packages in the stdlib.\n- Packages installed without metadata.\n\nProject details\n===============\n\n * Project home: https://github.com/python/importlib_metadata\n * Report bugs at: https://github.com/python/importlib_metadata/issues\n * Code hosting: https://github.com/python/importlib_metadata\n * Documentation: https://importlib_metadata.readthedocs.io/\n\nFor Enterprise\n==============\n\nAvailable as part of the Tidelift Subscription.\n\nThis project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.\n\n`Learn more <https://tidelift.com/subscription/pkg/pypi-importlib-metadata?utm_source=pypi-importlib-metadata&utm_medium=referral&utm_campaign=github>`_.\n\nSecurity Contact\n================\n\nTo report a security vulnerability, please use the\n`Tidelift security contact <https://tidelift.com/security>`_.\nTidelift will coordinate the fix and disclosure.",
-      "release_date": "2023-03-18T17:10:49",
+      "release_date": "2023-04-10T02:27:42",
       "parties": [
         {
           "type": "person",
@@ -351,11 +351,11 @@
         "Programming Language :: Python :: 3 :: Only"
       ],
       "homepage_url": "https://github.com/python/importlib_metadata",
-      "download_url": "https://files.pythonhosted.org/packages/e2/d8/3d431bade4598ad9e33be9da41d15e6607b878008e922d122659ab01b077/importlib_metadata-6.1.0.tar.gz",
-      "size": 50209,
+      "download_url": "https://files.pythonhosted.org/packages/c2/84/ab374b7e05fbdeecf867294660ac0fdb23aa286aca68a31d587f67d181ad/importlib_metadata-6.3.0.tar.gz",
+      "size": 52838,
       "sha1": null,
-      "md5": "c92d5a03615c6ef97d283d71fb92cf1b",
-      "sha256": "43ce9281e097583d758c2c708c4376371261a02c34682491a8e98352365aad20",
+      "md5": "f06a844f0917a1f4db1a1050f750c4fb",
+      "sha256": "23c2bcae4762dfb0bbe072d358faec24957901d75b6c4ab11172c0c982532402",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": null,
@@ -374,9 +374,9 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/importlib-metadata/6.1.0/json",
+      "api_data_url": "https://pypi.org/pypi/importlib-metadata/6.3.0/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/importlib-metadata@6.1.0"
+      "purl": "pkg:pypi/importlib-metadata@6.3.0"
     },
     {
       "type": "pypi",
@@ -1006,14 +1006,14 @@
       "package": "pkg:pypi/[email protected]",
       "dependencies": [
         "pkg:pypi/[email protected]",
-        "pkg:pypi/importlib-metadata@6.1.0",
+        "pkg:pypi/importlib-metadata@6.3.0",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]"
       ]
     },
     {
-      "package": "pkg:pypi/importlib-metadata@6.1.0",
+      "package": "pkg:pypi/importlib-metadata@6.3.0",
       "dependencies": [
         "pkg:pypi/[email protected]"
       ]