fix: project.unpack() to include full contracts directory (#2123)

pyproject.toml

@@ -12,7 +12,7 @@ plugins = ["pydantic.mypy"]
 # This version is purposely set to really high minor so that it should always work
 # with newer, stricter plugin releases.
 # NOTE: This should be bumped with every minor release!
-fallback_version = "0.7.999"
+fallback_version = "0.8.999"
 write_to = "src/ape/version.py"
 
 # NOTE: you have to use single-quoted strings in TOML for regular expressions.
@@ -21,7 +21,7 @@ write_to = "src/ape/version.py"
 # character.
 [tool.black]
 line-length = 100
-target-version = ['py38', 'py39', 'py310', 'py311', 'py312']
+target-version = ['py39', 'py310', 'py311', 'py312']
 include = '\.pyi?$'
 
 [tool.pytest.ini_options]

src/ape/cli/arguments.py

@@ -1,5 +1,4 @@
 from collections.abc import Iterable
-from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Union
 
@@ -64,7 +63,7 @@ def callback(cls, ctx, param, value) -> set[Path]:
         project = ctx.params.get("project")
         return cls(value, project=project).filtered_paths
 
-    @cached_property
+    @property
     def filtered_paths(self) -> set[Path]:
         """
         Get the filtered set of paths.
@@ -78,9 +77,10 @@ def filtered_paths(self) -> set[Path]:
         elif not value or value == "*":
             # Get all file paths in the project.
             return {p for p in self.project.sources.paths}
-        else:
-            # Given a sequence of paths.
+        elif isinstance(value, Iterable):
             contract_paths = value
+        else:
+            raise BadArgumentUsage(f"Not a path or iter[Path]: {value}")
 
         # Convert source IDs or relative paths to absolute paths.
         path_set = self.lookup(contract_paths)
@@ -120,8 +120,9 @@ def compiler_is_unknown(self, path: Union[Path, str]) -> bool:
 
     def lookup(self, path_iter: Iterable, path_set: Optional[set] = None) -> set[Path]:
         path_set = path_set or set()
+        given_paths = [p for p in path_iter]  # Handle iterators w/o losing it.
 
-        for path_id in path_iter:
+        for path_id in given_paths:
             path = Path(path_id)
             contracts_folder = self.project.contracts_folder
             if (
@@ -148,13 +149,8 @@ def lookup(self, path_iter: Iterable, path_set: Optional[set] = None) -> set[Pat
                     # NOTE: ^ Also tracks.
                     continue
 
-                suffix = get_full_extension(resolved_path)
-                if suffix in self.compiler_manager.registered_compilers:
-                    # File exists and is compile-able.
-                    path_set.add(resolved_path)
-
-                elif suffix:
-                    raise BadArgumentUsage(f"Source file '{resolved_path.name}' not found.")
+                # We know here that the compiler is known.
+                path_set.add(resolved_path)
 
             else:
                 raise BadArgumentUsage(f"Source file '{path.name}' not found.")

src/ape/managers/compilers.py

@@ -122,7 +122,6 @@ def compile(
 
         pm = project or self.local_project
         files_by_ext = defaultdict(list)
-
         if isinstance(contract_filepaths, (str, Path)):
             contract_filepaths = (contract_filepaths,)
 

src/ape/managers/project.py

@@ -77,7 +77,7 @@ def __len__(self) -> int:
         if self._path_cache is not None:
             return len(self._path_cache)
 
-        # Will set _path_cache, eliminates need to iterte (perf).
+        # Will set _path_cache, eliminates need to iterate (perf).
         return len(list(self.paths))
 
     def __iter__(self) -> Iterator[str]:
@@ -216,9 +216,11 @@ def is_excluded(self, path: Path) -> bool:
         for excl in self.exclude_globs:
             if isinstance(excl, Pattern):
                 for opt in options:
-                    if excl.match(opt):
-                        self._exclude_cache[source_id] = True
-                        return True
+                    if not excl.match(opt):
+                        continue
+
+                    self._exclude_cache[source_id] = True
+                    return True
 
             else:
                 # perf: Check parent directory first to exclude faster by marking them all.
@@ -280,13 +282,10 @@ def find_in_dir(dir_path: Path, path: Path) -> Optional[Path]:
                     for file in full_path.parent.iterdir():
                         if not file.is_file():
                             continue
-                        elif not (file_ext := get_full_extension(file)):
-                            continue
 
                         # Check exact match w/o extension.
-                        prefix = file_ext.join(str(file).split(file_ext)[:-1])
-
-                        if str(full_path) == prefix:
+                        prefix = str(file.with_suffix("")).strip(" /\\")
+                        if str(full_path).strip(" /\\") == prefix:
                             return file
 
                 # Look for stem-only matches (last resort).
@@ -1786,6 +1785,7 @@ def extract_manifest(self) -> PackageManifest:
 
     def clean(self):
         self._manifest.contract_types = None
+        self._config_override = {}
 
 
 class DeploymentManager(ManagerAccessMixin):
@@ -2246,7 +2246,17 @@ def isolate_in_tempdir(self, **config_override) -> Iterator["LocalProject"]:
                 yield project
 
     def unpack(self, destination: Path, config_override: Optional[dict] = None) -> "LocalProject":
-        project = super().unpack(destination, config_override=config_override)
+        config_override = {**self._config_override, **(config_override or {})}
+
+        # Unpack contracts.
+        if self.contracts_folder.is_dir():
+            contracts_path = get_relative_path(self.contracts_folder, self.path)
+            contracts_destination = destination / contracts_path
+            shutil.copytree(self.contracts_folder, contracts_destination, dirs_exist_ok=True)
+
+        # Unpack config file.
+        if not (destination / "ape-config.yaml").is_file():
+            self.config.write_to_disk(destination / "ape-config.yaml")
 
         # Unpack scripts folder.
         if self.scripts_folder.is_dir():
@@ -2265,7 +2275,7 @@ def unpack(self, destination: Path, config_override: Optional[dict] = None) -> "
             interfaces_destination.parent.mkdir(parents=True, exist_ok=True)
             shutil.copytree(self.interfaces_folder, interfaces_destination, dirs_exist_ok=True)
 
-        return project
+        return LocalProject(destination, config_override=config_override)
 
     def load_manifest(self) -> PackageManifest:
         """

src/ape_compile/__init__.py

@@ -2,7 +2,7 @@
 from re import Pattern
 from typing import Union
 
-from pydantic import field_validator
+from pydantic import field_serializer, field_validator
 
 from ape import plugins
 from ape.api.config import ConfigEnum, PluginConfig
@@ -74,6 +74,21 @@ def validate_exclude(cls, value):
         # Include defaults.
         return {*given_values, *SOURCE_EXCLUDE_PATTERNS}
 
+    @field_serializer("exclude", when_used="json")
+    def serialize_exclude(self, exclude, info):
+        """
+        Exclude is put back with the weird r-prefix so we can
+        go to-and-from.
+        """
+        result: list[str] = []
+        for excl in exclude:
+            if isinstance(excl, Pattern):
+                result.append(f'r"{excl.pattern}"')
+            else:
+                result.append(excl)
+
+        return result
+
 
 @plugins.register(plugins.Config)
 def config_class():

src/ape_compile/_cli.py

@@ -6,6 +6,7 @@
 
 from ape.cli.arguments import contract_file_paths_argument
 from ape.cli.options import ape_cli_context, config_override_option, project_option
+from ape.utils.os import clean_path
 
 
 def _include_dependencies_callback(ctx, param, value):
@@ -92,7 +93,8 @@ def cli(
                 _display_byte_code_sizes(cli_ctx, contract_types)
 
     if not compiled:
-        cli_ctx.logger.warning("Nothing to compile.")
+        folder = clean_path(project.contracts_folder)
+        cli_ctx.logger.warning(f"Nothing to compile ({folder}).")
 
     if errored:
         # Ensure exit code.

tests/conftest.py

@@ -79,8 +79,8 @@ def validate_cwd(start_dir):
         os.chdir(start_dir)
 
 
-@pytest.fixture(scope="session")
-def project(config):
+@pytest.fixture
+def project():
     path = "functional/data/contracts/local"
     with ape.project.temp_config(contracts_folder=path):
         yield ape.project

tests/functional/data/projects/ApeProject/contracts/Path.with.sub.json

@@ -0,0 +1 @@
+{}

tests/functional/test_compilers.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from re import Pattern
 from typing import cast
 
 import pytest
@@ -7,6 +8,7 @@
 from ape.contracts import ContractContainer
 from ape.exceptions import APINotImplementedError, CompilerError, ContractLogicError, CustomError
 from ape.types import AddressType
+from ape_compile import Config
 
 
 def test_get_imports(project, compilers):
@@ -162,3 +164,21 @@ def test_enrich_error_custom_error_with_inputs(compilers, setup_custom_error):
     assert actual.__class__.__name__ == "AllowanceExpired"
     assert actual.inputs["deadline"] == deadline
     assert repr(actual) == f"AllowanceExpired(deadline={deadline})"
+
+
+def test_config_exclude_regex_serialize():
+    """
+    Show we can to-and-fro with exclude regexes.
+    """
+    raw_value = 'r"FooBar"'
+    cfg = Config(exclude=[raw_value])
+    excl = [x for x in cfg.exclude if isinstance(x, Pattern)]
+    assert len(excl) == 1
+    assert excl[0].pattern == "FooBar"
+    # NOTE: Use json mode to ensure we can go from most minimum value back.
+    model_dump = cfg.model_dump(mode="json", by_alias=True)
+    assert raw_value in model_dump.get("exclude", [])
+    new_cfg = Config.model_validate(cfg.model_dump(mode="json", by_alias=True))
+    excl = [x for x in new_cfg.exclude if isinstance(x, Pattern)]
+    assert len(excl) == 1
+    assert excl[0].pattern == "FooBar"

tests/functional/test_coverage.py

@@ -173,7 +173,7 @@ def src(self):
     def contract_source(self, vyper_contract_type, src):
         return ContractSource(contract_type=vyper_contract_type, source=src)
 
-    @pytest.fixture(scope="class")
+    @pytest.fixture
     def coverage_data(self, project, contract_source):
         return CoverageData(project, (contract_source,))
 

tests/functional/test_project.py

@@ -13,6 +13,7 @@
 from ape.contracts import ContractContainer
 from ape.exceptions import ProjectError
 from ape.logging import LogLevel
+from ape.utils import create_tempdir
 from ape_pm import BrownieProject, FoundryProject
 from tests.conftest import skip_if_plugin_installed
 
@@ -415,6 +416,15 @@ def test_clean(tmp_project):
     assert tmp_project.sources._path_cache is None
 
 
+def test_unpack(project_with_source_files_contract):
+    with create_tempdir() as path:
+        project_with_source_files_contract.unpack(path)
+        assert (path / "contracts" / "Contract.json").is_file()
+
+        # Show that even non-sources end up in the unpacked destination.
+        assert (path / "contracts" / "Path.with.sub.json").is_file()
+
+
 def test_add_compiler_data(project_with_dependency_config):
     # NOTE: Using different project than default to lessen
     #   chance of race-conditions from multi-process test runners.

tests/functional/utils/test_misc.py

@@ -1,3 +1,5 @@
+import asyncio
+
 import pytest
 from eth_pydantic_types import HexBytes
 from packaging.version import Version
@@ -73,6 +75,8 @@ def test_run_until_complete_coroutine():
     async def foo():
         return 3
 
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
     actual = run_until_complete(foo())
     assert actual == 3
 
@@ -84,6 +88,8 @@ async def foo():
     async def bar():
         return 4
 
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
     actual = run_until_complete(foo(), bar())
     assert actual == [3, 4]
 

tests/integration/cli/conftest.py

@@ -85,7 +85,7 @@ def pytest_collection_modifyitems(session, config, items):
     items[:] = modified_items
 
 
-@pytest.fixture(autouse=True)
+@pytest.fixture(autouse=True, scope="session")
 def project_dir_map():
     """
     Ensure only copying projects once to prevent `TooManyOpenFilesError`.
@@ -97,9 +97,12 @@ class ProjectDirCache:
         def load(self, name: str) -> Path:
             base_path = Path(__file__).parent / "projects"
             if name in self.project_map:
-                # Already copied.
-                return self.project_map[name]
+                res = self.project_map[name]
+                if res.is_dir():
+                    # Already copied and still exists!
+                    return res
 
+            # Either re-copy or copy for the first time.
             project_source_dir = __projects_directory__ / name
             project_dest_dir = base_path / project_source_dir.name
             project_dest_dir.parent.mkdir(exist_ok=True, parents=True)
@@ -114,11 +117,17 @@ def load(self, name: str) -> Path:
 
 
 @pytest.fixture(autouse=True, params=__project_names__)
-def project(request, config, project_dir_map):
+def integ_project(request, project_dir_map):
     project_dir = project_dir_map.load(request.param)
+    if not project_dir.is_dir():
+        # Should not happen because of logic in fixture,
+        # but just in case!
+        pytest.fail("Setup failed - project dir not exists.")
+
     root_project = Project(project_dir)
     # Using tmp project so no .build folder get kept around.
     with root_project.isolate_in_tempdir(name=request.param) as tmp_project:
+        assert tmp_project.path.is_dir(), "Setup failed - tmp-project dir not exists"
         yield tmp_project