perf: lazily load plugins [APE-1422]

src/ape/_cli.py

@@ -1,7 +1,7 @@
 import difflib
 import re
 import sys
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional
 
 import click
 import importlib_metadata as metadata
@@ -29,7 +29,8 @@ def display_config(ctx, param, value):
 
 
 class ApeCLI(click.MultiCommand):
-    _commands = None
+    _commands: Optional[Dict] = None
+    _CLI_GROUP_NAME = "ape_cli_subcommands"
 
     def invoke(self, ctx) -> Any:
         try:
@@ -48,11 +49,14 @@ def _suggest_cmd(usage_error):
         if usage_error.message is None:
             raise usage_error
 
-        match = re.match("No such command '(.*)'.", usage_error.message)
-        if not match:
+        elif not (match := re.match("No such command '(.*)'.", usage_error.message)):
             raise usage_error
 
-        bad_arg = match.groups()[0]
+        groups = match.groups()
+        if len(groups) < 1:
+            raise usage_error
+
+        bad_arg = groups[0]
         suggested_commands = difflib.get_close_matches(
             bad_arg, list(usage_error.ctx.command.commands.keys()), cutoff=_DIFFLIB_CUT_OFF
         )
@@ -66,30 +70,22 @@ def _suggest_cmd(usage_error):
 
     @property
     def commands(self) -> Dict:
-        group_name = "ape_cli_subcommands"
-        if not self._commands:
-            try:
-                entry_points = metadata.entry_points(group=group_name)
-            except TypeError:
-                entry_points = metadata.entry_points()
-                entry_points = (
-                    entry_points[group_name] if group_name in entry_points else []  # type: ignore
-                )
-
-            if not entry_points:
-                raise Abort("Missing registered cli subcommands")
+        if self._commands:
+            return self._commands
 
-            self._commands = {
-                clean_plugin_name(entry_point.name): entry_point.load
-                for entry_point in entry_points
-            }
+        entry_points = metadata.entry_points(group=self._CLI_GROUP_NAME)
+        if not entry_points:
+            raise Abort("Missing registered CLI subcommands.")
 
+        self._commands = {
+            clean_plugin_name(entry_point.name): entry_point.load for entry_point in entry_points
+        }
         return self._commands
 
-    def list_commands(self, ctx):
+    def list_commands(self, ctx) -> List[str]:
         return list(sorted(self.commands))
 
-    def get_command(self, ctx, name):
+    def get_command(self, ctx, name) -> Optional[click.Command]:
         if name in self.commands:
             try:
                 return self.commands[name]()
@@ -99,6 +95,7 @@ def get_command(self, ctx, name):
                 )
 
         # NOTE: don't return anything so Click displays proper error
+        return None
 
 
 @click.command(cls=ApeCLI, context_settings=dict(help_option_names=["-h", "--help"]))

src/ape/cli/arguments.py

@@ -55,13 +55,11 @@ def _raise_bad_arg(name):
     resolved_contract_paths = set()
     for contract_path in contract_paths:
         # Adds missing absolute path as well as extension.
-        pm = project if ctx.obj is None else ctx.obj.project_manager
-        resolved_contract_path = pm.lookup_path(contract_path)
-        if not resolved_contract_path:
+        if resolved_contract_path := project.lookup_path(contract_path):
+            resolved_contract_paths.add(resolved_contract_path)
+        else:
             _raise_bad_arg(contract_path.name)
 
-        resolved_contract_paths.add(resolved_contract_path)
-
     return resolved_contract_paths
 
 

src/ape/cli/choices.py

@@ -1,13 +1,15 @@
 import re
 from enum import Enum
-from typing import Any, List, Optional, Type, Union
+from functools import lru_cache
+from typing import Any, Iterator, List, Optional, Sequence, Type, Union
 
 import click
 from click import Choice, Context, Parameter
 
 from ape import accounts, networks
 from ape.api.accounts import AccountAPI
 from ape.exceptions import AccountsError
+from ape.types import _LazySequence
 
 ADHOC_NETWORK_PATTERN = re.compile(r"\w*:\w*:https?://\w*.*")
 
@@ -34,18 +36,15 @@ def __init__(self, account_type: Optional[Type[AccountAPI]] = None):
         # NOTE: we purposely skip the constructor of `Choice`
         self.case_sensitive = False
         self._account_type = account_type
+        self.choices = _LazySequence(self._choices_iterator)
 
     @property
-    def choices(self) -> List[str]:  # type: ignore
-        """
-        The aliases available to choose from.
-
-        Returns:
-            List[str]: A list of account aliases the user may choose from.
-        """
+    def _choices_iterator(self) -> Iterator[str]:
+        for acct in _get_account_by_type(self._account_type):
+            if acct.alias is None:
+                continue
 
-        options = _get_account_by_type(self._account_type)
-        return [a.alias for a in options if a.alias is not None]
+            yield acct.alias
 
 
 class PromptChoice(click.ParamType):
@@ -67,7 +66,7 @@ def cmd(choice):
             click.echo(f"__expected_{choice}")
     """
 
-    def __init__(self, choices, name: Optional[str] = None):
+    def __init__(self, choices: Sequence[str], name: Optional[str] = None):
         self.choices = choices
         # Since we purposely skip the super() constructor, we need to make
         # sure the class still has a name.
@@ -159,6 +158,7 @@ def __init__(
         self._account_type = account_type
         self._prompt_message = prompt_message or "Select an account"
         self.name = name
+        self.choices = _LazySequence(self._choices_iterator)
 
     def convert(
         self, value: Any, param: Optional[Parameter], ctx: Optional[Context]
@@ -201,21 +201,15 @@ def print_choices(self):
             click.echo()
 
     @property
-    def choices(self) -> List[str]:
-        """
-        All the account aliases.
+    def _choices_iterator(self) -> Iterator[str]:
+        # Yield real accounts.
+        for account in _get_account_by_type(self._account_type):
+            if account and (alias := account.alias):
+                yield alias
 
-        Returns:
-            List[str]: A list of all the account aliases.
-        """
-
-        _accounts = [
-            a.alias
-            for a in _get_account_by_type(self._account_type)
-            if a is not None and a.alias is not None
-        ]
-        _accounts.extend([f"TEST::{i}" for i, _ in enumerate(accounts.test_accounts)])
-        return _accounts
+        # Yield test accounts (at the end).
+        for idx, _ in enumerate(accounts.test_accounts):
+            yield f"TEST::{idx}"
 
     def get_user_selected_account(self) -> AccountAPI:
         """
@@ -239,6 +233,53 @@ def fail_from_invalid_choice(self, param):
         return self.fail("Invalid choice. Type the number or the alias.", param=param)
 
 
+_NETWORK_FILTER = Optional[Union[List[str], str]]
+
+
+def get_networks(
+    ecosystem: _NETWORK_FILTER = None,
+    network: _NETWORK_FILTER = None,
+    provider: _NETWORK_FILTER = None,
+) -> _LazySequence:
+    # NOTE: Use str-keys and lru_cache.
+    return _get_networks_sequence_from_cache(
+        _network_filter_to_key(ecosystem),
+        _network_filter_to_key(network),
+        _network_filter_to_key(provider),
+    )
+
+
+@lru_cache(maxsize=None)
+def _get_networks_sequence_from_cache(ecosystem_key: str, network_key: str, provider_key: str):
+    return _LazySequence(
+        networks.get_network_choices(
+            ecosystem_filter=_key_to_network_filter(ecosystem_key),
+            network_filter=_key_to_network_filter(network_key),
+            provider_filter=_key_to_network_filter(provider_key),
+        )
+    )
+
+
+def _network_filter_to_key(filter_: _NETWORK_FILTER) -> str:
+    if filter_ is None:
+        return "__none__"
+
+    elif isinstance(filter_, list):
+        return ",".join(filter_)
+
+    return filter_
+
+
+def _key_to_network_filter(key: str) -> _NETWORK_FILTER:
+    if key == "__none__":
+        return None
+
+    elif "," in key:
+        return [n.strip() for n in key.split(",")]
+
+    return key
+
+
 class NetworkChoice(click.Choice):
     """
     A ``click.Choice`` to provide network choice defaults for the active project.
@@ -252,17 +293,12 @@ class NetworkChoice(click.Choice):
     def __init__(
         self,
         case_sensitive=True,
-        ecosystem: Optional[Union[List[str], str]] = None,
-        network: Optional[Union[List[str], str]] = None,
-        provider: Optional[Union[List[str], str]] = None,
+        ecosystem: _NETWORK_FILTER = None,
+        network: _NETWORK_FILTER = None,
+        provider: _NETWORK_FILTER = None,
     ):
         super().__init__(
-            list(
-                networks.get_network_choices(
-                    ecosystem_filter=ecosystem, network_filter=network, provider_filter=provider
-                )
-            ),
-            case_sensitive,
+            get_networks(ecosystem=ecosystem, network=network, provider=provider), case_sensitive
         )
 
     def get_metavar(self, param):
@@ -306,7 +342,7 @@ def output_format_choice(options: Optional[List[OutputFormat]] = None) -> Choice
         :class:`click.Choice`
     """
 
-    options = options or [o for o in OutputFormat]
+    options = options or list(OutputFormat)
 
     # Uses `str` form of enum for CLI choices.
     return click.Choice([o.value for o in options], case_sensitive=False)

src/ape/cli/options.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, NoReturn, Optional, Union
+from typing import Callable, Dict, List, NoReturn, Optional, Union
 
 import click
 from ethpm_types import ContractType
@@ -27,7 +27,6 @@ class ApeCliContextObject(ManagerAccessMixin):
 
     def __init__(self):
         self.logger = logger
-        self.config_manager.load()
 
     @staticmethod
     def abort(msg: str, base_error: Optional[Exception] = None) -> NoReturn:
@@ -96,7 +95,7 @@ def decorator(f):
 
 
 def network_option(
-    default: Optional[str] = "auto",
+    default: Optional[Union[str, Callable]] = "auto",
     ecosystem: Optional[Union[List[str], str]] = None,
     network: Optional[Union[List[str], str]] = None,
     provider: Optional[Union[List[str], str]] = None,
@@ -126,8 +125,13 @@ def network_option(
     if auto and not required:
         if ecosystem:
             default = ecosystem[0] if isinstance(ecosystem, (list, tuple)) else ecosystem
+
         else:
-            default = networks.default_ecosystem.name
+            # NOTE: Use a function as the default so it is calculated lazily
+            def fn():
+                return networks.default_ecosystem.name
+
+            default = fn
 
     elif auto:
         default = None

src/ape/plugins/__init__.py

@@ -121,6 +121,10 @@ class PluginManager:
     _unimplemented_plugins: List[str] = []
 
     def __init__(self) -> None:
+        self.__registered = False
+
+    @functools.cached_property
+    def _plugin_modules(self) -> Tuple[str, ...]:
         # NOTE: Unable to use pkgutil.iter_modules() for installed plugins
         # because it does not work with editable installs.
         # See https://github.com/python/cpython/issues/99805.
@@ -134,23 +138,18 @@ def __init__(self) -> None:
         core_plugin_module_names = {
             n for _, n, ispkg in pkgutil.iter_modules() if n.startswith("ape_")
         }
-        module_names = installed_plugin_module_names.union(core_plugin_module_names)
-
-        for module_name in module_names:
-            try:
-                module = importlib.import_module(module_name)
-                plugin_manager.register(module)
-            except Exception as err:
-                if module_name in __modules__:
-                    # Always raise core plugin registration errors.
-                    raise
 
-                logger.warn_from_exception(err, f"Error loading plugin package '{module_name}'.")
+        # NOTE: Returns tuple because this shouldn't change.
+        return tuple(installed_plugin_module_names.union(core_plugin_module_names))
 
     def __repr__(self):
         return f"<{self.__class__.__name__}>"
 
     def __getattr__(self, attr_name: str) -> Iterator[Tuple[str, Tuple]]:
+        # NOTE: The first time this method is called, the actual
+        #  plugin registration occurs. Registration only happens once.
+        self._register_plugins()
+
         if not hasattr(plugin_manager.hook, attr_name):
             raise ApeAttributeError(f"{self.__class__.__name__} has no attribute '{attr_name}'.")
 
@@ -174,6 +173,23 @@ def get_plugin_name_and_hookfn(h):
                     if validated_plugin:
                         yield validated_plugin
 
+    def _register_plugins(self):
+        if self.__registered:
+            return
+
+        for module_name in self._plugin_modules:
+            try:
+                module = importlib.import_module(module_name)
+                plugin_manager.register(module)
+            except Exception as err:
+                if module_name in __modules__:
+                    # Always raise core plugin registration errors.
+                    raise
+
+                logger.warn_from_exception(err, f"Error loading plugin package '{module_name}'.")
+
+        self.__registered = True
+
     def _validate_plugin(self, plugin_name: str, plugin_cls) -> Optional[Tuple[str, Tuple]]:
         if valid_impl(plugin_cls):
             return clean_plugin_name(plugin_name), plugin_cls