Add parsing and checking of requires and tool_requires in the require…

…ments method of conanfile.py and testing for it

Add python 3.12 support and testing for it
Update pre-commit hooks
Fix black and ruff warnings and errors

.pre-commit-config.yaml

@@ -1,21 +1,21 @@
 repos:
   - repo: "https://github.com/pre-commit/pre-commit-hooks"
-    rev: v4.3.0
+    rev: v4.6.0
     hooks:
       - id: end-of-file-fixer
         exclude: ^tests/output/
       - id: trailing-whitespace
         exclude: ^tests/output/
   - repo: "https://github.com/psf/black"
-    rev: 22.12.0
+    rev: 24.3.0
     hooks:
       - id: black
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.258
+    rev: v0.3.5
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
   - repo: "https://github.com/pre-commit/mirrors-mypy"
-    rev: v0.991
+    rev: v1.9.0
     hooks:
       - id: mypy

pyproject.toml

@@ -71,7 +71,7 @@ line-length = 100
 
 [tool.ruff]
 line-length = 100
-select = [
+lint.select = [
     "F", # pyflakes
     "E", "W", # pycodestyle
     "I", # isort
@@ -87,11 +87,11 @@ select = [
     "PL", # pylint
     "RUF", # ruff specific rules
 ]
-ignore = [
+lint.ignore = [
     "PLR0911", # too many return statements
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "tests/*" = [
     "PLR0913", # too many arguments
     "PLR2004", # magic value
@@ -125,7 +125,7 @@ envlist =
     black
     ruff
     mypy
-    py{37, 38, 39, 310, 311}-{v1, v2}
+    py{37, 38, 39, 310, 311, 312}-{v1, v2}
     coverage-report
 
 [testenv:black]

src/conan_check_updates/color.py

@@ -61,7 +61,7 @@ class AnsiCodes(IntEnum):
     BG_DEFAULT = 49
 
     def __str__(self) -> str:
-        return f"\033[{str(self.value)}m"
+        return f"\033[{self.value!s}m"
 
 
 def colored(text: str, *codes: AnsiCodes, force_color: bool = False) -> str:

src/conan_check_updates/conan.py

@@ -95,13 +95,13 @@ def find_conanfile(path: Path) -> Path:
     if path.is_file():
         if path.name in filenames:
             return path
-        raise ValueError(f"Path is not a conanfile: {str(path)}")
+        raise ValueError(f"Path is not a conanfile: {path!s}")
     if path.is_dir():
         for filepath in (path / filename for filename in filenames):
             if filepath.exists():
                 return filepath
-        raise ValueError(f"Could not find conanfile in path: {str(path)}")
-    raise ValueError(f"Invalid path: {str(path)}")
+        raise ValueError(f"Could not find conanfile in path: {path!s}")
+    raise ValueError(f"Invalid path: {path!s}")
 
 
 # https://docs.conan.io/en/1.55/reference/conanfile/attributes.html#name
@@ -134,8 +134,8 @@ class ConanReference:
     channel: Optional[str] = None
 
     @classmethod
-    def parse(cls, reference: str):
-        reference = reference.strip()
+    def parse(cls, reference):
+        reference = reference.strip() if isinstance(reference, str) else reference["ref"].strip()
         match = _PATTERN_CONAN_REFERENCE.fullmatch(reference)
         if not match:
             raise ValueError(f"Invalid Conan reference '{reference}'")
@@ -173,6 +173,26 @@ def version_str() -> str:
 _REQUIRES_ATTRIBUTES = ("requires", "build_requires", "tool_requires", "test_requires")
 
 
+def inspect_requirements_conanfile_py(conanfile: Path) -> List[ConanReference]:
+    """Get requirements from requirements() method of conanfile.py"""
+    assert conanfile.name == "conanfile.py"
+    refs = []
+    with open(conanfile, mode="r", encoding="utf-8") as file:
+        for line_orig in file:
+            line = line_orig.strip()
+            # strip end of line comment
+            line = line.partition(" #")[0].strip()
+            # ignore empty line or line comments
+            if not line or line.startswith("#"):
+                continue
+            res = re.search(r'self\.(?:tool_)*requires\("(.*)"\)', line)
+            if res:
+                ref = res.group(1)
+                if len(ref) > 0:
+                    refs.append(ref)
+    return list(map(ConanReference.parse, refs))
+
+
 def inspect_requires_conanfile_py(conanfile: Path) -> List[ConanReference]:
     """Get requirements of conanfile.py with `conan inspect`."""
     assert conanfile.name == "conanfile.py"
@@ -183,7 +203,7 @@ def get_command():
             return ("conan", "inspect", str(conanfile), *args)
         if conan_version().major == 2:  # noqa: PLR2004
             return ("conan", "inspect", str(conanfile))
-        raise RuntimeError(f"Conan version {str(conan_version())} not supported")
+        raise RuntimeError(f"Conan version {conan_version()!s} not supported")
 
     stdout, _ = _run_capture(*get_command(), timeout=TIMEOUT)
 
@@ -239,10 +259,12 @@ def gen_requires():
 def inspect_requires_conanfile(conanfile: Path) -> List[ConanReference]:
     """Get requirements of conanfile.py/conanfile.py"""
     if conanfile.name == "conanfile.py":
-        return inspect_requires_conanfile_py(conanfile)
+        return inspect_requires_conanfile_py(conanfile) + inspect_requirements_conanfile_py(
+            conanfile
+        )
     if conanfile.name == "conanfile.txt":
         return inspect_requires_conanfile_txt(conanfile)
-    raise ValueError(f"Invalid conanfile: {str(conanfile)}")
+    raise ValueError(f"Invalid conanfile: {conanfile!s}")
 
 
 async def search(
@@ -261,7 +283,7 @@ def get_command():
             return ("conan", "search", pattern, "--remote", "all", "--raw")
         if conan_version().major == 2:  # noqa: PLR2004
             return ("conan", "search", pattern)
-        raise RuntimeError(f"Conan version {str(conan_version())} not supported")
+        raise RuntimeError(f"Conan version {conan_version()!s} not supported")
 
     stdout, _ = await _run_capture_async(*get_command(), timeout=timeout)
     return [

src/conan_check_updates/main.py

@@ -45,8 +45,7 @@ class CheckUpdateResult:
 
 
 class ProgressCallback(Protocol):
-    def __call__(self, done: int, total: int):
-        ...
+    def __call__(self, done: int, total: int): ...
 
 
 async def check_updates(
@@ -119,11 +118,9 @@ def upgrade_conanfile(conanfile: Path, update_results: Sequence[CheckUpdateResul
 
         occurrences = content.count(str(result.ref))
         if occurrences < 1:
-            raise RuntimeError(f"Reference '{str(result.ref)}' not found in conanfile")
+            raise RuntimeError(f"Reference '{result.ref!s}' not found in conanfile")
         if occurrences > 1:
-            raise RuntimeError(
-                f"Multiple occurrences of reference '{str(result.ref)}' in conanfile"
-            )
+            raise RuntimeError(f"Multiple occurrences of reference '{result.ref!s}' in conanfile")
 
         # generate new reference with update version
         new_ref = replace(

tests/conanfile.py

@@ -13,3 +13,10 @@ class Example(ConanFile):
     )
     tool_requires = "ninja/[^1.10]"
     generators = "cmake"
+
+    def requirements(self):
+        self.requires("openssl/3.2.0")
+        self.requires("nanodbc/2.13.0")
+        self.requires("ms-gsl/3.1.0")
+        self.tool_requires("cmake/3.27.7")
+        # self.requires("quill/3.6.0")

tests/conanfile.py.json

@@ -0,0 +1,22 @@
+{
+    "requires": [
+      "boost/1.79.0",
+      "catch2/3.2.0",
+      "fmt/9.0.0",
+      "nlohmann_json/3.10.0"
+    ],
+    "tool_requires": [
+      "ninja/[^1.10]"
+    ],
+    "generators": [
+      "cmake"
+    ],
+    "requirements": [
+        "openssl/3.2.0",
+        "nanodbc/2.13.0",
+        "ms-gsl/3.1.0"
+    ],
+    "tool_requirements": [
+        "cmake/3.27.7"
+    ]
+  }

tests/conanfile.txt

@@ -4,9 +4,14 @@ catch2/3.2.0
 # line comment
 fmt/9.0.0  # end of line comment
 nlohmann_json/3.10.0
+openssl/3.2.0
+nanodbc/2.13.0
+ms-gsl/3.1.0
+# quill/3.6.0
 
 [tool_requires]
 ninja/[^1.10]
+cmake/3.27.7
 
 [generators]
 cmake

tests/conanfile.json → tests/conanfile.txt.json

@@ -3,10 +3,14 @@
     "boost/1.79.0",
     "catch2/3.2.0",
     "fmt/9.0.0",
-    "nlohmann_json/3.10.0"
+    "nlohmann_json/3.10.0",
+    "openssl/3.2.0",
+    "nanodbc/2.13.0",
+    "ms-gsl/3.1.0"
   ],
   "tool_requires": [
-    "ninja/[^1.10]"
+    "ninja/[^1.10]",
+    "cmake/3.27.7"
   ],
   "generators": [
     "cmake"

tests/generate_output.py

@@ -10,7 +10,7 @@
 def run_and_save_output(name: str, cmd: str):
     print("Run", name)
     # pylint: disable=subprocess-run-check
-    proc = run(cmd, capture_output=True, shell=True, cwd=HERE)
+    proc = run(cmd, capture_output=True, shell=True, cwd=HERE, check=False)
     if proc.returncode != 0:
         raise RuntimeError(f"Error running '{cmd}':\n\n{proc.stderr.decode()}")
     (OUTPUT_DIR / f"{name}_stdout.txt").write_bytes(proc.stdout)

tests/test_conan.py

@@ -1,21 +1,21 @@
 import asyncio
+import contextlib
 import json
 import subprocess
 import sys
 from pathlib import Path
 from typing import List
 from unittest.mock import Mock, patch
 
-try:
+with contextlib.suppress(ImportError):
     from unittest.mock import AsyncMock
-except ImportError:
-    ...  # skip tests for Python < 3.8
 
 import pytest
 from conan_check_updates.conan import (
     ConanError,
     ConanReference,
     find_conanfile,
+    inspect_requirements_conanfile_py,
     inspect_requires_conanfile_py,
     inspect_requires_conanfile_txt,
     search,
@@ -127,7 +127,14 @@ def parse_requires_conanfile_json(path: Path) -> List[ConanReference]:
     obj = json.loads(path.read_bytes())
 
     def gen_requires():
-        for attr in ("requires", "build_requires", "tool_requires", "test_requires"):
+        for attr in (
+            "requires",
+            "build_requires",
+            "tool_requires",
+            "test_requires",
+            "requirements",
+            "tool_requirements",
+        ):
             yield from obj.get(attr, [])
 
     return list(map(ConanReference.parse, gen_requires()))
@@ -152,13 +159,15 @@ def test_inspect_requires_conanfile_py(mock_process, stdout, stderr):
     mock_process.stdout = stdout
     mock_process.stderr = stderr
 
-    expected = parse_requires_conanfile_json(HERE / "conanfile.json")
-    requires = inspect_requires_conanfile_py(HERE / "conanfile.py")
+    expected = parse_requires_conanfile_json(HERE / "conanfile.py.json")
+    requires = inspect_requires_conanfile_py(
+        HERE / "conanfile.py"
+    ) + inspect_requirements_conanfile_py(HERE / "conanfile.py")
     assert requires == expected
 
 
 def test_inspect_requires_conanfile_txt():
-    expected = parse_requires_conanfile_json(HERE / "conanfile.json")
+    expected = parse_requires_conanfile_json(HERE / "conanfile.txt.json")
     requires = inspect_requires_conanfile_txt(HERE / "conanfile.txt")
     assert requires == expected
 

tests/test_conan_e2e.py

@@ -28,10 +28,10 @@ def test_conan_version_fail():
         conan_version()
 
 
-@pytest.mark.parametrize("conanfile", ["conanfile.py", "conanfile.txt"])
-def test_inspect_requires_conanfile(conanfile):
-    expected = parse_requires_conanfile_json(HERE / "conanfile.json")
-    requires = inspect_requires_conanfile(HERE / conanfile)
+@pytest.mark.parametrize("conanfileext", ["py", "txt"])
+def test_inspect_requires_conanfile(conanfileext):
+    expected = parse_requires_conanfile_json(HERE / f"conanfile.{conanfileext}.json")
+    requires = inspect_requires_conanfile(HERE / f"conanfile.{conanfileext}")
     assert requires == expected