refactor: move missing compiler check to contract-filepaths callback (#…

…2043)

docs/userguides/clis.md

@@ -217,3 +217,29 @@ def cli_1(alias):
 my_accounts = [accounts.load("me"), accounts.load("me2")]
 selected_account = get_user_selected_account(account_type=my_accounts)
 ```
+
+# Contract File Paths
+
+Does your CLI interact with contract source files?
+(Think `ape compile`).
+
+If so, use the `@contract_file_paths_argument()` decorator in your CLI.
+
+```python
+from pathlib import Path
+import click
+
+from ape.cli import contract_file_paths_argument
+
+@click.command()
+@contract_file_paths_argument()
+def cli(file_paths: set[Path]):
+   # Loop through all source files given (or all source files in the project).
+    for path in file_paths:
+        click.echo(f"Source found: {path}")
+```
+
+When using the `@contract_file_paths_argument()` decorator, you can pass any number of source files as arguments.
+When not passing any source file(s), `@contract_file_paths_argument()` defaults to all sources in the local project.
+That is why `ape compile` compiles the full project and `ape compile MySource.vy` only compiles `MySource.vy` (and whatever else it needs / imports).
+Use `@contract_file_paths_argument()` for any similar use-case involving contract source files.

src/ape/api/projects.py

@@ -340,18 +340,21 @@ class DependencyAPI(ExtraAttributesMixin, BaseInterfaceModel):
     The version of the dependency. Omit to use the latest.
     """
 
+    # TODO: Remove in 0.8.
     contracts_folder: str = "contracts"
     """
     The name of the dependency's ``contracts/`` directory.
     This is where ``ape`` will look for source files when compiling
     the manifest for this dependency.
 
-    **NOTE**: This must be the name of a directory in the project.
+    **Deprecated**: Use ``config_override:contracts_folder``.
     """
 
+    # TODO: Remove in 0.8.
     exclude: List[str] = ["package.json", "package-lock.json", "**/.build/**/*.json"]
     """
     A list of glob-patterns for excluding files in dependency projects.
+    **Deprecated**: Use ``config_override:compile:exclude``.
     """
 
     config_override: Dict = {}
@@ -535,13 +538,16 @@ def _extract_local_manifest(
         elif project_path.parent.is_dir():
             project_path = project_path.parent
 
+        # TODO: In 0.8, delete self.contracts_folder and rely on cfg override.
+        contracts_folder = self.config_override.get("contracts_folder", self.contracts_folder)
+
         # NOTE: Dependencies are not compiled here. Instead, the sources are packaged
         # for later usage via imports. For legacy reasons, many dependency-esque projects
         # are not meant to compile on their own.
 
         with self.config_manager.using_project(
             project_path,
-            contracts_folder=(project_path / self.contracts_folder).expanduser().resolve(),
+            contracts_folder=(project_path / contracts_folder).expanduser().resolve(),
         ) as pm:
             project = pm.local_project
             if sources := self._get_sources(project):
@@ -578,8 +584,14 @@ def _get_sources(self, project: ProjectAPI) -> List[Path]:
         pattern = rf".*({extension_pattern})"
         all_sources = get_all_files_in_directory(project.contracts_folder, pattern=pattern)
 
+        # TODO: In 0.8, delete self.exclude and only use config override.
+        exclude = [
+            *(self.exclude or []),
+            *(self.config_override.get("compile", {}).get("exclude", []) or []),
+        ]
+
         excluded_files = set()
-        for pattern in set(self.exclude):
+        for pattern in set(exclude):
             excluded_files.update({f for f in project.contracts_folder.glob(pattern)})
 
         return [s for s in all_sources if s not in excluded_files]

src/ape/cli/arguments.py

@@ -1,14 +1,17 @@
-from itertools import chain
+from fnmatch import fnmatch
+from functools import cached_property
+from pathlib import Path
+from typing import Iterable, List, Set, Union
 
 import click
+from click import BadArgumentUsage
 
 from ape.cli.choices import _ACCOUNT_TYPE_FILTER, Alias
-from ape.cli.paramtype import AllFilePaths
+from ape.logging import logger
 from ape.utils.basemodel import ManagerAccessMixin
+from ape.utils.os import get_all_files_in_directory, get_full_extension
 from ape.utils.validators import _validate_account_alias
 
-_flatten = chain.from_iterable
-
 
 def _alias_callback(ctx, param, value):
     return _validate_account_alias(value)
@@ -40,21 +43,127 @@ def non_existing_alias_argument(**kwargs):
     return click.argument("alias", callback=callback, **kwargs)
 
 
-def _create_contracts_paths(ctx, param, value):
-    contract_paths = _flatten(value)
+class _ContractPaths(ManagerAccessMixin):
+    """
+    Helper callback class for handling CLI-given contract paths.
+    """
 
-    def _raise_bad_arg(name):
-        raise click.BadArgumentUsage(f"Contract '{name}' not found.")
+    def __init__(self, value):
+        self.value = value
+        self._path_set = set()
+        self.missing_compilers = set()
+        self.exclude_list = {}
+
+    @classmethod
+    def callback(cls, ctx, param, value) -> Set[Path]:
+        """
+        Use this for click.option / argument callbacks.
+        """
+        return cls(value).filtered_paths
+
+    @cached_property
+    def filtered_paths(self) -> Set[Path]:
+        """
+        Get the filtered set of paths.
+        """
+        value = self.value
+        contract_paths: Iterable[Path]
+
+        if value and isinstance(value, (list, tuple, set)):
+            # Given a single list of paths.
+            contract_paths = value
+
+        elif value and isinstance(value, (Path, str)):
+            # Given single path.
+            contract_paths = (Path(value),)
+
+        elif not value or value == "*":
+            # Get all file paths in the project.
+            contract_paths = get_all_files_in_directory(self.project_manager.contracts_folder)
 
-    resolved_contract_paths = set()
-    for contract_path in contract_paths:
-        # Adds missing absolute path as well as extension.
-        if resolved_contract_path := ManagerAccessMixin.project_manager.lookup_path(contract_path):
-            resolved_contract_paths.add(resolved_contract_path)
         else:
-            _raise_bad_arg(contract_path.name)
-
-    return resolved_contract_paths
+            raise ValueError(f"Unknown contracts-paths value '{value}'.")
+
+        self.lookup(contract_paths)
+
+        # Handle missing compilers.
+        if self.missing_compilers:
+            # Craft a nice message for all missing compilers.
+            missing_ext = ", ".join(sorted(self.missing_compilers))
+            message = (
+                f"Missing compilers for the following file types: '{missing_ext}'. "
+                "Possibly, a compiler plugin is not installed or is "
+                "installed but not loading correctly."
+            )
+            if ".vy" in self.missing_compilers:
+                message = f"{message} Is 'ape-vyper' installed?"
+            if ".sol" in self.missing_compilers:
+                message = f"{message} Is 'ape-solidity' installed?"
+
+            logger.warning(message)
+
+        return self._path_set
+
+    @property
+    def exclude_patterns(self) -> List[str]:
+        return self.config_manager.get_config("compile").exclude or []
+
+    def do_exclude(self, path: Union[Path, str]) -> bool:
+        name = path if isinstance(path, str) else path.name
+        if path not in self.exclude_list:
+            self.exclude_list[path] = any(fnmatch(name, p) for p in self.exclude_patterns)
+
+        return self.exclude_list[path]
+
+    def compiler_is_unknown(self, path: Union[Path, str]) -> bool:
+        path = Path(path)
+        if self.do_exclude(path):
+            return False
+
+        ext = get_full_extension(path)
+        unknown_compiler = ext and ext not in self.compiler_manager.registered_compilers
+        if unknown_compiler and ext not in self.missing_compilers:
+            self.missing_compilers.add(ext)
+
+        return bool(unknown_compiler)
+
+    def lookup(self, path_iter):
+        for path in path_iter:
+            path = Path(path)
+            if self.do_exclude(path):
+                continue
+
+            contracts_folder = self.project_manager.contracts_folder
+            if (
+                self.project_manager.path / path.name
+            ) == contracts_folder or path.name == contracts_folder.name:
+                # Was given the path to the contracts folder.
+                self.lookup(p for p in self.project_manager.source_paths)
+
+            elif (self.project_manager.path / path).is_dir():
+                # Was given sub-dir in the project folder.
+                self.lookup(p for p in (self.project_manager.path / path).iterdir())
+
+            elif (contracts_folder / path.name).is_dir():
+                # Was given sub-dir in the contracts folder.
+                self.lookup(p for p in (contracts_folder / path.name).iterdir())
+
+            elif resolved_path := self.project_manager.lookup_path(path):
+                # Check compiler missing.
+                if self.compiler_is_unknown(resolved_path):
+                    # NOTE: ^ Also tracks.
+                    continue
+
+                suffix = get_full_extension(resolved_path)
+                if suffix in self.compiler_manager.registered_compilers:
+                    # File exists and is compile-able.
+                    self._path_set.add(resolved_path)
+
+                elif suffix:
+                    raise BadArgumentUsage(f"Source file '{resolved_path.name}' not found.")
+
+            else:
+                raise BadArgumentUsage(f"Source file '{path.name}' not found.")
 
 
 def contract_file_paths_argument():
@@ -66,9 +175,4 @@ def contract_file_paths_argument():
     source file-paths.
     """
 
-    return click.argument(
-        "file_paths",
-        nargs=-1,
-        type=AllFilePaths(resolve_path=True),
-        callback=_create_contracts_paths,
-    )
+    return click.argument("file_paths", nargs=-1, callback=_ContractPaths.callback)

src/ape/cli/paramtype.py

@@ -21,6 +21,8 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
 
+# TODO: Delete for 0.8 (list of lists is weird and we
+#  are no longer using this).
 class AllFilePaths(Path):
     """
     Either all the file paths in the given directory,

src/ape/managers/project/manager.py

@@ -683,7 +683,7 @@ def lookup_path(self, key_contract_path: Union[Path, str]) -> Optional[Path]:
         input_path = Path(key_contract_path)
         if input_path.is_file():
             # Already given an existing file.
-            return input_path
+            return input_path.absolute()
 
         input_stem = input_path.stem
         input_extension = get_full_extension(input_path) or None

src/ape_compile/__init__.py

@@ -7,6 +7,18 @@
 from ape.api import PluginConfig
 
 DEFAULT_CACHE_FOLDER_NAME = ".cache"  # default relative to contracts/
+EXCLUDE_PATTERNS = [
+    "*package.json",
+    "*package-lock.json",
+    "*tsconfig.json",
+    "*.md",
+    "*.rst",
+    "*.txt",
+    "*.py[a-zA-Z]?",
+    "*.html",
+    "*.css",
+    "*.adoc",
+]
 
 
 class Config(PluginConfig):
@@ -21,7 +33,7 @@ class Config(PluginConfig):
     should configure ``include_dependencies`` to be ``True``.
     """
 
-    exclude: List[str] = ["*package.json", "*package-lock.json", "*tsconfig.json"]
+    exclude: List[str] = []
     """
     Source exclusion globs across all file types.
     """
@@ -41,7 +53,7 @@ def base_path(self) -> Path:
         assert self.cache_folder is not None
 
         # If the dependency cache folder is configured, to be outside of the contracts dir, we want
-        # to use the projects folder to be the base dir for copmilation.
+        # to use the projects folder to be the base dir for compilation.
         if self._config_manager.contracts_folder not in self.cache_folder.parents:
             return self._config_manager.PROJECT_FOLDER
 
@@ -79,7 +91,8 @@ def validate_cache_folder(self):
     @field_validator("exclude", mode="before")
     @classmethod
     def validate_exclude(cls, value):
-        return value or []
+        excl = [*(value or []), *EXCLUDE_PATTERNS]
+        return list(set(excl))
 
 
 @plugins.register(plugins.Config)

src/ape_compile/_cli.py

@@ -5,7 +5,6 @@
 from ethpm_types import ContractType
 
 from ape.cli import ape_cli_context, contract_file_paths_argument
-from ape.utils.os import get_full_extension
 
 
 def _include_dependencies_callback(ctx, param, value):
@@ -52,34 +51,6 @@ def cli(cli_ctx, file_paths: Set[Path], use_cache: bool, display_size: bool, inc
         cli_ctx.logger.warning("Nothing to compile.")
         return
 
-    ext_given = [get_full_extension(p) for p in file_paths if p]
-
-    # Filter out common files that we know are not files you can compile anyway,
-    # like documentation files. NOTE: Nothing prevents a CompilerAPI from using these
-    # extensions, we just don't warn about missing compilers here. The warning is really
-    # meant to help guide users when the vyper, solidity, or cairo plugins are not installed.
-    general_extensions = {".md", ".rst", ".txt", ".py", ".html", ".css", ".adoc"}
-
-    ext_with_missing_compilers = {
-        x
-        for x in cli_ctx.project_manager.extensions_with_missing_compilers(ext_given)
-        if x not in general_extensions
-    }
-    if ext_with_missing_compilers:
-        if len(ext_with_missing_compilers) > 1:
-            # NOTE: `sorted` to increase reproducibility.
-            extensions_str = ", ".join(sorted(ext_with_missing_compilers))
-            message = f"Missing compilers for the following file types: '{extensions_str}'."
-        else:
-            message = f"Missing a compiler for {ext_with_missing_compilers.pop()} file types."
-
-        message = (
-            f"{message} "
-            f"Possibly, a compiler plugin is not installed "
-            f"or is installed but not loading correctly."
-        )
-        cli_ctx.logger.warning(message)
-
     contract_types = cli_ctx.project_manager.load_contracts(
         file_paths=file_paths, use_cache=use_cache
     )

tests/functional/data/projects/ApeProject/contracts/subdir/ApeContractNested.json

@@ -0,0 +1,3 @@
+[
+    {"name":"ApeContract","type":"fallback", "stateMutability":"nonpayable"}
+]