diff --git a/docs/userguides/clis.md b/docs/userguides/clis.md index ac7d5cd740..6df5e1bb03 100644 --- a/docs/userguides/clis.md +++ b/docs/userguides/clis.md @@ -217,3 +217,29 @@ def cli_1(alias): my_accounts = [accounts.load("me"), accounts.load("me2")] selected_account = get_user_selected_account(account_type=my_accounts) ``` + +# Contract File Paths + +Does your CLI interact with contract source files? +(Think `ape compile`). + +If so, use the `@contract_file_paths_argument()` decorator in your CLI. + +```python +from pathlib import Path +import click + +from ape.cli import contract_file_paths_argument + +@click.command() +@contract_file_paths_argument() +def cli(file_paths: set[Path]): + # Loop through all source files given (or all source files in the project). + for path in file_paths: + click.echo(f"Source found: {path}") +``` + +When using the `@contract_file_paths_argument()` decorator, you can pass any number of source files as arguments. +When not passing any source file(s), `@contract_file_paths_argument()` defaults to all sources in the local project. +That is why `ape compile` compiles the full project and `ape compile MySource.vy` only compiles `MySource.vy` (and whatever else it needs / imports). +Use `@contract_file_paths_argument()` for any similar use-case involving contract source files. diff --git a/src/ape/api/projects.py b/src/ape/api/projects.py index 2689079fb8..39693acc44 100644 --- a/src/ape/api/projects.py +++ b/src/ape/api/projects.py @@ -340,18 +340,21 @@ class DependencyAPI(ExtraAttributesMixin, BaseInterfaceModel): The version of the dependency. Omit to use the latest. """ + # TODO: Remove in 0.8. contracts_folder: str = "contracts" """ The name of the dependency's ``contracts/`` directory. This is where ``ape`` will look for source files when compiling the manifest for this dependency. - **NOTE**: This must be the name of a directory in the project. + **Deprecated**: Use ``config_override:contracts_folder``. """ + # TODO: Remove in 0.8. exclude: List[str] = ["package.json", "package-lock.json", "**/.build/**/*.json"] """ A list of glob-patterns for excluding files in dependency projects. + **Deprecated**: Use ``config_override:compile:exclude``. """ config_override: Dict = {} @@ -535,13 +538,16 @@ def _extract_local_manifest( elif project_path.parent.is_dir(): project_path = project_path.parent + # TODO: In 0.8, delete self.contracts_folder and rely on cfg override. + contracts_folder = self.config_override.get("contracts_folder", self.contracts_folder) + # NOTE: Dependencies are not compiled here. Instead, the sources are packaged # for later usage via imports. For legacy reasons, many dependency-esque projects # are not meant to compile on their own. with self.config_manager.using_project( project_path, - contracts_folder=(project_path / self.contracts_folder).expanduser().resolve(), + contracts_folder=(project_path / contracts_folder).expanduser().resolve(), ) as pm: project = pm.local_project if sources := self._get_sources(project): @@ -578,8 +584,14 @@ def _get_sources(self, project: ProjectAPI) -> List[Path]: pattern = rf".*({extension_pattern})" all_sources = get_all_files_in_directory(project.contracts_folder, pattern=pattern) + # TODO: In 0.8, delete self.exclude and only use config override. + exclude = [ + *(self.exclude or []), + *(self.config_override.get("compile", {}).get("exclude", []) or []), + ] + excluded_files = set() - for pattern in set(self.exclude): + for pattern in set(exclude): excluded_files.update({f for f in project.contracts_folder.glob(pattern)}) return [s for s in all_sources if s not in excluded_files] diff --git a/src/ape/cli/arguments.py b/src/ape/cli/arguments.py index 5b1e99be58..03574e2c74 100644 --- a/src/ape/cli/arguments.py +++ b/src/ape/cli/arguments.py @@ -1,14 +1,17 @@ -from itertools import chain +from fnmatch import fnmatch +from functools import cached_property +from pathlib import Path +from typing import Iterable, List, Set, Union import click +from click import BadArgumentUsage from ape.cli.choices import _ACCOUNT_TYPE_FILTER, Alias -from ape.cli.paramtype import AllFilePaths +from ape.logging import logger from ape.utils.basemodel import ManagerAccessMixin +from ape.utils.os import get_all_files_in_directory, get_full_extension from ape.utils.validators import _validate_account_alias -_flatten = chain.from_iterable - def _alias_callback(ctx, param, value): return _validate_account_alias(value) @@ -40,21 +43,127 @@ def non_existing_alias_argument(**kwargs): return click.argument("alias", callback=callback, **kwargs) -def _create_contracts_paths(ctx, param, value): - contract_paths = _flatten(value) +class _ContractPaths(ManagerAccessMixin): + """ + Helper callback class for handling CLI-given contract paths. + """ - def _raise_bad_arg(name): - raise click.BadArgumentUsage(f"Contract '{name}' not found.") + def __init__(self, value): + self.value = value + self._path_set = set() + self.missing_compilers = set() + self.exclude_list = {} + + @classmethod + def callback(cls, ctx, param, value) -> Set[Path]: + """ + Use this for click.option / argument callbacks. + """ + return cls(value).filtered_paths + + @cached_property + def filtered_paths(self) -> Set[Path]: + """ + Get the filtered set of paths. + """ + value = self.value + contract_paths: Iterable[Path] + + if value and isinstance(value, (list, tuple, set)): + # Given a single list of paths. + contract_paths = value + + elif value and isinstance(value, (Path, str)): + # Given single path. + contract_paths = (Path(value),) + + elif not value or value == "*": + # Get all file paths in the project. + contract_paths = get_all_files_in_directory(self.project_manager.contracts_folder) - resolved_contract_paths = set() - for contract_path in contract_paths: - # Adds missing absolute path as well as extension. - if resolved_contract_path := ManagerAccessMixin.project_manager.lookup_path(contract_path): - resolved_contract_paths.add(resolved_contract_path) else: - _raise_bad_arg(contract_path.name) - - return resolved_contract_paths + raise ValueError(f"Unknown contracts-paths value '{value}'.") + + self.lookup(contract_paths) + + # Handle missing compilers. + if self.missing_compilers: + # Craft a nice message for all missing compilers. + missing_ext = ", ".join(sorted(self.missing_compilers)) + message = ( + f"Missing compilers for the following file types: '{missing_ext}'. " + "Possibly, a compiler plugin is not installed or is " + "installed but not loading correctly." + ) + if ".vy" in self.missing_compilers: + message = f"{message} Is 'ape-vyper' installed?" + if ".sol" in self.missing_compilers: + message = f"{message} Is 'ape-solidity' installed?" + + logger.warning(message) + + return self._path_set + + @property + def exclude_patterns(self) -> List[str]: + return self.config_manager.get_config("compile").exclude or [] + + def do_exclude(self, path: Union[Path, str]) -> bool: + name = path if isinstance(path, str) else path.name + if path not in self.exclude_list: + self.exclude_list[path] = any(fnmatch(name, p) for p in self.exclude_patterns) + + return self.exclude_list[path] + + def compiler_is_unknown(self, path: Union[Path, str]) -> bool: + path = Path(path) + if self.do_exclude(path): + return False + + ext = get_full_extension(path) + unknown_compiler = ext and ext not in self.compiler_manager.registered_compilers + if unknown_compiler and ext not in self.missing_compilers: + self.missing_compilers.add(ext) + + return bool(unknown_compiler) + + def lookup(self, path_iter): + for path in path_iter: + path = Path(path) + if self.do_exclude(path): + continue + + contracts_folder = self.project_manager.contracts_folder + if ( + self.project_manager.path / path.name + ) == contracts_folder or path.name == contracts_folder.name: + # Was given the path to the contracts folder. + self.lookup(p for p in self.project_manager.source_paths) + + elif (self.project_manager.path / path).is_dir(): + # Was given sub-dir in the project folder. + self.lookup(p for p in (self.project_manager.path / path).iterdir()) + + elif (contracts_folder / path.name).is_dir(): + # Was given sub-dir in the contracts folder. + self.lookup(p for p in (contracts_folder / path.name).iterdir()) + + elif resolved_path := self.project_manager.lookup_path(path): + # Check compiler missing. + if self.compiler_is_unknown(resolved_path): + # NOTE: ^ Also tracks. + continue + + suffix = get_full_extension(resolved_path) + if suffix in self.compiler_manager.registered_compilers: + # File exists and is compile-able. + self._path_set.add(resolved_path) + + elif suffix: + raise BadArgumentUsage(f"Source file '{resolved_path.name}' not found.") + + else: + raise BadArgumentUsage(f"Source file '{path.name}' not found.") def contract_file_paths_argument(): @@ -66,9 +175,4 @@ def contract_file_paths_argument(): source file-paths. """ - return click.argument( - "file_paths", - nargs=-1, - type=AllFilePaths(resolve_path=True), - callback=_create_contracts_paths, - ) + return click.argument("file_paths", nargs=-1, callback=_ContractPaths.callback) diff --git a/src/ape/cli/paramtype.py b/src/ape/cli/paramtype.py index 860bb20ace..e85760c656 100644 --- a/src/ape/cli/paramtype.py +++ b/src/ape/cli/paramtype.py @@ -21,6 +21,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) +# TODO: Delete for 0.8 (list of lists is weird and we +# are no longer using this). class AllFilePaths(Path): """ Either all the file paths in the given directory, diff --git a/src/ape/managers/project/manager.py b/src/ape/managers/project/manager.py index 975019e3a9..8af4c8c862 100644 --- a/src/ape/managers/project/manager.py +++ b/src/ape/managers/project/manager.py @@ -683,7 +683,7 @@ def lookup_path(self, key_contract_path: Union[Path, str]) -> Optional[Path]: input_path = Path(key_contract_path) if input_path.is_file(): # Already given an existing file. - return input_path + return input_path.absolute() input_stem = input_path.stem input_extension = get_full_extension(input_path) or None diff --git a/src/ape_compile/__init__.py b/src/ape_compile/__init__.py index ed9c64653f..48264bb66f 100644 --- a/src/ape_compile/__init__.py +++ b/src/ape_compile/__init__.py @@ -7,6 +7,18 @@ from ape.api import PluginConfig DEFAULT_CACHE_FOLDER_NAME = ".cache" # default relative to contracts/ +EXCLUDE_PATTERNS = [ + "*package.json", + "*package-lock.json", + "*tsconfig.json", + "*.md", + "*.rst", + "*.txt", + "*.py[a-zA-Z]?", + "*.html", + "*.css", + "*.adoc", +] class Config(PluginConfig): @@ -21,7 +33,7 @@ class Config(PluginConfig): should configure ``include_dependencies`` to be ``True``. """ - exclude: List[str] = ["*package.json", "*package-lock.json", "*tsconfig.json"] + exclude: List[str] = [] """ Source exclusion globs across all file types. """ @@ -41,7 +53,7 @@ def base_path(self) -> Path: assert self.cache_folder is not None # If the dependency cache folder is configured, to be outside of the contracts dir, we want - # to use the projects folder to be the base dir for copmilation. + # to use the projects folder to be the base dir for compilation. if self._config_manager.contracts_folder not in self.cache_folder.parents: return self._config_manager.PROJECT_FOLDER @@ -79,7 +91,8 @@ def validate_cache_folder(self): @field_validator("exclude", mode="before") @classmethod def validate_exclude(cls, value): - return value or [] + excl = [*(value or []), *EXCLUDE_PATTERNS] + return list(set(excl)) @plugins.register(plugins.Config) diff --git a/src/ape_compile/_cli.py b/src/ape_compile/_cli.py index ad6f2011ff..c57e67e55c 100644 --- a/src/ape_compile/_cli.py +++ b/src/ape_compile/_cli.py @@ -5,7 +5,6 @@ from ethpm_types import ContractType from ape.cli import ape_cli_context, contract_file_paths_argument -from ape.utils.os import get_full_extension def _include_dependencies_callback(ctx, param, value): @@ -52,34 +51,6 @@ def cli(cli_ctx, file_paths: Set[Path], use_cache: bool, display_size: bool, inc cli_ctx.logger.warning("Nothing to compile.") return - ext_given = [get_full_extension(p) for p in file_paths if p] - - # Filter out common files that we know are not files you can compile anyway, - # like documentation files. NOTE: Nothing prevents a CompilerAPI from using these - # extensions, we just don't warn about missing compilers here. The warning is really - # meant to help guide users when the vyper, solidity, or cairo plugins are not installed. - general_extensions = {".md", ".rst", ".txt", ".py", ".html", ".css", ".adoc"} - - ext_with_missing_compilers = { - x - for x in cli_ctx.project_manager.extensions_with_missing_compilers(ext_given) - if x not in general_extensions - } - if ext_with_missing_compilers: - if len(ext_with_missing_compilers) > 1: - # NOTE: `sorted` to increase reproducibility. - extensions_str = ", ".join(sorted(ext_with_missing_compilers)) - message = f"Missing compilers for the following file types: '{extensions_str}'." - else: - message = f"Missing a compiler for {ext_with_missing_compilers.pop()} file types." - - message = ( - f"{message} " - f"Possibly, a compiler plugin is not installed " - f"or is installed but not loading correctly." - ) - cli_ctx.logger.warning(message) - contract_types = cli_ctx.project_manager.load_contracts( file_paths=file_paths, use_cache=use_cache ) diff --git a/tests/functional/data/projects/ApeProject/contracts/subdir/ApeContractNested.json b/tests/functional/data/projects/ApeProject/contracts/subdir/ApeContractNested.json new file mode 100644 index 0000000000..95edf12aff --- /dev/null +++ b/tests/functional/data/projects/ApeProject/contracts/subdir/ApeContractNested.json @@ -0,0 +1,3 @@ +[ + {"name":"ApeContract","type":"fallback", "stateMutability":"nonpayable"} +] diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index b4349ed714..c3999a0702 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -23,7 +23,7 @@ from ape.cli.commands import get_param_from_ctx, parse_network from ape.exceptions import AccountsError from ape.logging import logger -from tests.conftest import geth_process_test +from tests.conftest import geth_process_test, skip_if_plugin_installed OUTPUT_FORMAT = "__TEST__{0}:{1}:{2}_" OTHER_OPTION_VALUE = "TEST_OTHER_OPTION" @@ -74,6 +74,19 @@ def cmd(ecosystem, network, provider): return cmd +@pytest.fixture +def contracts_paths_cmd(): + expected = "EXPECTED {}" + + @click.command() + @contract_file_paths_argument() + def cmd(file_paths): + output = ", ".join(x.name for x in sorted(file_paths)) + click.echo(expected.format(output)) + + return cmd + + def _setup_temp_acct_number_change(accounts, num_accounts: int): if "containers" in accounts.__dict__: del accounts.__dict__["containers"] @@ -412,14 +425,78 @@ def test_account_prompt_name(): assert option.name == "account_z" -def test_contract_file_paths_argument(runner): - @click.command() - @contract_file_paths_argument() - def cmd(file_paths): - pass +def test_contract_file_paths_argument_given_source_id( + project_with_source_files_contract, runner, contracts_paths_cmd +): + pm = project_with_source_files_contract + src_id = next(iter(pm.sources)) + result = runner.invoke(contracts_paths_cmd, src_id) + assert f"EXPECTED {src_id}" in result.output + + +def test_contract_file_paths_argument_given_name( + project_with_source_files_contract, runner, contracts_paths_cmd +): + pm = project_with_source_files_contract + src_stem = next(iter(pm.sources)).split(".")[0] + result = runner.invoke(contracts_paths_cmd, src_stem) + assert f"EXPECTED {src_stem}" in result.output + - result = runner.invoke(cmd, ["path0", "path1"]) - assert "Contract 'path0' not found" in result.output +def test_contract_file_paths_argument_given_contracts_folder( + project_with_contract, runner, contracts_paths_cmd +): + pm = project_with_contract + result = runner.invoke(contracts_paths_cmd, pm.contracts_folder.as_posix()) + all_paths = ", ".join(x.name for x in sorted(pm.source_paths)) + assert f"EXPECTED {all_paths}" in result.output + + +def test_contract_file_paths_argument_given_contracts_folder_name( + project_with_contract, runner, contracts_paths_cmd +): + pm = project_with_contract + result = runner.invoke(contracts_paths_cmd, "contracts") + all_paths = ", ".join(x.name for x in sorted(pm.source_paths)) + assert f"EXPECTED {all_paths}" in result.output + + +@pytest.mark.parametrize("name", ("contracts/subdir", "subdir")) +def test_contract_file_paths_argument_given_subdir_relative_to_path( + project_with_contract, runner, contracts_paths_cmd, name +): + pm = project_with_contract + result = runner.invoke(contracts_paths_cmd, name) + all_paths = ", ".join(x.name for x in sorted(pm.source_paths) if x.parent.name == "subdir") + assert f"EXPECTED {all_paths}" in result.output + + +@skip_if_plugin_installed("vyper") +def test_contract_file_paths_argument_missing_vyper( + project_with_source_files_contract, runner, contracts_paths_cmd +): + name = "VyperContract" + result = runner.invoke(contracts_paths_cmd, name) + expected = ( + "Missing compilers for the following file types: '.vy'. " + "Possibly, a compiler plugin is not installed or is installed " + "but not loading correctly. Is 'ape-vyper' installed?" + ) + assert expected in result.output + + +@skip_if_plugin_installed("solidity") +def test_contract_file_paths_argument_missing_solidity( + project_with_source_files_contract, runner, contracts_paths_cmd +): + name = "SolidityContract" + result = runner.invoke(contracts_paths_cmd, name) + expected = ( + "Missing compilers for the following file types: '.sol'. " + "Possibly, a compiler plugin is not installed or is installed " + "but not loading correctly. Is 'ape-solidity' installed?" + ) + assert expected in result.output def test_existing_alias_option(runner): @@ -428,7 +505,7 @@ def test_existing_alias_option(runner): def cmd(alias): click.echo(alias) - result = runner.invoke(cmd, ["TEST::0"]) + result = runner.invoke(cmd, "TEST::0") assert "TEST::0" in result.output @@ -443,7 +520,7 @@ def custom_callback(*args, **kwargs): def cmd(alias): click.echo(alias) - result = runner.invoke(cmd, ["TEST::0"]) + result = runner.invoke(cmd, "TEST::0") assert magic_value in result.output @@ -453,7 +530,7 @@ def test_non_existing_alias_option(runner): def cmd(alias): click.echo(alias) - result = runner.invoke(cmd, ["non-exists"]) + result = runner.invoke(cmd, "non-exists") assert "non-exists" in result.output @@ -468,7 +545,7 @@ def custom_callback(*args, **kwargs): def cmd(alias): click.echo(alias) - result = runner.invoke(cmd, ["non-exists"]) + result = runner.invoke(cmd, "non-exists") assert magic_value in result.output @@ -489,7 +566,7 @@ def test_connected_provider_command_invalid_value(runner): def cmd(): pass - result = runner.invoke(cmd, ["--network", "OOGA_BOOGA"], catch_exceptions=False) + result = runner.invoke(cmd, ("--network", "OOGA_BOOGA"), catch_exceptions=False) assert result.exit_code != 0 assert "Invalid value for '--network'" in result.output diff --git a/tests/functional/test_project.py b/tests/functional/test_project.py index 844704953d..4dd1ed0f65 100644 --- a/tests/functional/test_project.py +++ b/tests/functional/test_project.py @@ -462,6 +462,19 @@ def clean(): clean() +def test_lookup_path_includes_contracts_prefix(project_with_source_files_contract): + """ + Show we can include the `contracts/` prefix. + """ + project = project_with_source_files_contract + actual_from_str = project.lookup_path("contracts/ContractA.sol") + actual_from_path = project.lookup_path(Path("contracts/ContractA.sol")) + expected = project.contracts_folder / "ContractA.sol" + assert actual_from_str == actual_from_path == expected + assert actual_from_str.is_absolute() + assert actual_from_path.is_absolute() + + def test_sources(project_with_source_files_contract): project = project_with_source_files_contract assert "ApeContract0.json" in project.sources diff --git a/tests/integration/cli/test_compile.py b/tests/integration/cli/test_compile.py index 81cde55360..b3e7ab21a5 100644 --- a/tests/integration/cli/test_compile.py +++ b/tests/integration/cli/test_compile.py @@ -38,26 +38,30 @@ def test_compile_missing_contracts_dir(ape_cli, runner, project): @skip_projects_except("bad-contracts") -def test_skip_contracts_and_missing_compilers(ape_cli, runner, project, switch_config): +def test_compile_skip_contracts_and_missing_compilers(ape_cli, runner, project, switch_config): result = runner.invoke(ape_cli, ("compile", "--force")) + + # Default exclude test. assert "INFO: Compiling 'subdir/tsconfig.json'." not in result.output assert "INFO: Compiling 'package.json'." not in result.output - # NOTE: `.md` should NOT appear in this list! + # Ensure extensions from exclude (such as .md) don't appear in missing-compilers. assert ( "WARNING: Missing compilers for the following file types: '.foo, .foobar, .test'. " "Possibly, a compiler plugin is not installed or is installed but not loading correctly." ) in result.output - # Simulate configuring Ape to not ignore tsconfig.json for some reason. + # Show we can include custom excludes. content = """ compile: exclude: - - "*package.json" + - "*Contract2.foo" """ with switch_config(project, content): result = runner.invoke(ape_cli, ("compile", "--force")) - assert "INFO: Compiling 'subdir/tsconfig.json'." in result.output + + # Show our custom exclude is not mentioned in missing compilers. + assert "pes: '.foo," not in result.output @skip_non_compilable_projects @@ -224,11 +228,11 @@ def test_compile_specified_contracts(ape_cli, runner, project, contract_path, cl @skip_projects_except("multiple-interfaces") def test_compile_unknown_extension_does_not_compile(ape_cli, runner, project, clean_cache): - result = runner.invoke( - ape_cli, ("compile", "Interface.js"), catch_exceptions=False - ) # Suffix to existing extension + name = "Interface.js" + result = runner.invoke(ape_cli, ("compile", name), catch_exceptions=False) + expected = f"Source file '{name}' not found." assert result.exit_code == 2, result.output - assert "Error: Contract 'Interface.js' not found." in result.output + assert expected in result.output @skip_projects_except()