feat: networks run CLI [APE-981] (#1576)

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
         name: black
 
 -   repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
     -   id: flake8
 

docs/userguides/networks.md

@@ -108,3 +108,18 @@ Some reasons for this include:
 3. Response differences in uncommon blocks, such as the `"pending"` block or the genesis block.
 4. Revert messages and exception-handling differences.
 5. You are limited to using `web3.py` and EVM-based chains.
+
+## Running a Network Process
+
+To run a network with a process, use the `ape networks run` command:
+
+```shell
+ape networks run
+```
+
+By default, `ape networks run` runs a development Geth process.
+To use a different network, such as `hardhat` or Anvil nodes, use the `--network` flag:
+
+```shell
+ape networks run --network ethereum:local:foundry
+```

setup.py

@@ -25,7 +25,7 @@
         "types-setuptools",  # Needed due to mypy typeshed
         "pandas-stubs==1.2.0.62",  # Needed due to mypy typeshed
         "types-SQLAlchemy>=1.4.49",  # Needed due to mypy typeshed
-        "flake8>=6.0.0,<7",  # Style linter
+        "flake8>=6.1.0,<7",  # Style linter
         "flake8-breakpoint>=1.1.0,<2",  # detect breakpoints left in code
         "flake8-print>=4.0.1,<5",  # detect print statements left in code
         "isort>=5.10.1,<6",  # Import sorting linter

src/ape/api/providers.py

@@ -1623,9 +1623,8 @@ def start(self, timeout: int = 20):
             self.stderr_queue = JoinableQueue()
             self.stdout_queue = JoinableQueue()
             out_file = PIPE if logger.level <= LogLevel.DEBUG else DEVNULL
-            self.process = Popen(
-                self.build_command(), preexec_fn=pre_exec_fn, stdout=out_file, stderr=out_file
-            )
+            cmd = self.build_command()
+            self.process = Popen(cmd, preexec_fn=pre_exec_fn, stdout=out_file, stderr=out_file)
             spawn(self.produce_stdout_queue)
             spawn(self.produce_stderr_queue)
             spawn(self.consume_stdout_queue)

src/ape/cli/options.py

@@ -47,16 +47,18 @@ def abort(msg: str, base_error: Optional[Exception] = None) -> NoReturn:
         raise Abort(msg)
 
 
-def verbosity_option(cli_logger: Optional[CliLogger] = None):
+def verbosity_option(cli_logger: Optional[CliLogger] = None, default: str = DEFAULT_LOG_LEVEL):
     """A decorator that adds a `--verbosity, -v` option to the decorated
     command.
     """
     _logger = cli_logger or logger
-    kwarguments = _create_verbosity_kwargs(_logger=_logger)
+    kwarguments = _create_verbosity_kwargs(_logger=_logger, default=default)
     return lambda f: click.option(*_VERBOSITY_VALUES, **kwarguments)(f)
 
 
-def _create_verbosity_kwargs(_logger: Optional[CliLogger] = None) -> Dict:
+def _create_verbosity_kwargs(
+    _logger: Optional[CliLogger] = None, default: str = DEFAULT_LOG_LEVEL
+) -> Dict:
     cli_logger = _logger or logger
 
     def set_level(ctx, param, value):
@@ -66,23 +68,27 @@ def set_level(ctx, param, value):
     names_str = f"{', '.join(level_names[:-1])}, or {level_names[-1]}"
     return {
         "callback": set_level,
-        "default": DEFAULT_LOG_LEVEL,
+        "default": default or DEFAULT_LOG_LEVEL,
         "metavar": "LVL",
         "expose_value": False,
         "help": f"One of {names_str}",
         "is_eager": True,
     }
 
 
-def ape_cli_context():
+def ape_cli_context(default_log_level: str = DEFAULT_LOG_LEVEL):
     """
     A ``click`` context object with helpful utilities.
     Use in your commands to get access to common utility features,
     such as logging or accessing managers.
+
+    Args:
+        default_log_level (str): The log-level value to pass to
+          :meth:`~ape.cli.options.verbosity_option`.
     """
 
     def decorator(f):
-        f = verbosity_option(logger)(f)
+        f = verbosity_option(logger, default=default_log_level)(f)
         f = click.make_pass_decorator(ApeCliContextObject, ensure=True)(f)
         return f
 

src/ape/logging.py

@@ -19,6 +19,7 @@ class LogLevel(IntEnum):
 logging.addLevelName(LogLevel.SUCCESS.value, LogLevel.SUCCESS.name)
 logging.SUCCESS = LogLevel.SUCCESS.value  # type: ignore
 DEFAULT_LOG_LEVEL = LogLevel.INFO.name
+DEFAULT_LOG_FORMAT = "%(levelname)s: %(message)s"
 
 
 def success(self, message, *args, **kws):
@@ -61,8 +62,9 @@ def _isatty(stream: IO) -> bool:
 
 
 class ApeColorFormatter(logging.Formatter):
-    def __init__(self):
-        super().__init__(fmt="%(levelname)s: %(message)s")
+    def __init__(self, fmt: Optional[str] = None):
+        fmt = fmt or DEFAULT_LOG_FORMAT
+        super().__init__(fmt=fmt)
 
     def format(self, record):
         if _isatty(sys.stdout) and _isatty(sys.stderr):
@@ -95,17 +97,43 @@ def emit(self, record):
 class CliLogger:
     _mentioned_verbosity_option = False
 
-    def __init__(self):
-        _logger = _get_logger("ape")
+    def __init__(
+        self,
+        _logger: logging.Logger,
+        fmt: str,
+        web3_request_logger: Optional[logging.Logger] = None,
+        web3_http_logger: Optional[logging.Logger] = None,
+    ):
         self.error = _logger.error
         self.warning = _logger.warning
         self.success = getattr(_logger, "success", _logger.info)
         self.info = _logger.info
         self.debug = _logger.debug
         self._logger = _logger
-        self._web3_request_manager_logger = _get_logger("web3.RequestManager")
-        self._web3_http_provider_logger = _get_logger("web3.providers.HTTPProvider")
+        self._web3_request_manager_logger = web3_request_logger
+        self._web3_http_provider_logger = web3_http_logger
         self._load_from_sys_argv()
+        self.fmt = fmt
+
+    @classmethod
+    def create(cls, fmt: Optional[str] = None, third_party: bool = True) -> "CliLogger":
+        fmt = fmt or DEFAULT_LOG_FORMAT
+        kwargs = {}
+        if third_party:
+            kwargs["web3_request_logger"] = _get_logger("web3.RequestManager", fmt=fmt)
+            kwargs["web3_http_logger"] = _get_logger("web3.providers.HTTPProvider", fmt=fmt)
+
+        _logger = _get_logger("ape", fmt=fmt)
+        return cls(_logger, fmt, **kwargs)
+
+    def format(self, fmt: Optional[str] = None):
+        self.fmt = fmt or DEFAULT_LOG_FORMAT
+        fmt = fmt or DEFAULT_LOG_FORMAT
+        _format_logger(self._logger, fmt)
+        if req_log := self._web3_request_manager_logger:
+            _format_logger(req_log, fmt)
+        if prov_log := self._web3_http_provider_logger:
+            _format_logger(prov_log, fmt)
 
     def _load_from_sys_argv(self, default: Optional[Union[str, int]] = None):
         """
@@ -126,9 +154,7 @@ def _load_from_sys_argv(self, default: Optional[Union[str, int]] = None):
                     self._logger.error(f"Must be one of '{names_str}', not '{level}'.")
                     sys.exit(2)
 
-        self._logger.setLevel(log_level)
-        self._web3_request_manager_logger.setLevel(log_level)
-        self._web3_http_provider_logger.setLevel(log_level)
+        self.set_level(log_level)
 
     @property
     def level(self) -> int:
@@ -145,9 +171,13 @@ def set_level(self, level: Union[str, int]):
         if level == self._logger.level:
             return
 
-        self._logger.setLevel(level)
-        self._web3_request_manager_logger.setLevel(level)
-        self._web3_http_provider_logger.setLevel(level)
+        for log in (
+            self._logger,
+            self._web3_http_provider_logger,
+            self._web3_request_manager_logger,
+        ):
+            if obj := log:
+                obj.setLevel(level)
 
     def log_error(self, err: Exception):
         """
@@ -190,14 +220,29 @@ def log_debug_stack_trace(self):
         stack_trace = traceback.format_exc()
         self._logger.debug(stack_trace)
 
+    def _clear_web3_loggers(self):
+        self._web3_request_manager_logger = None
+        self._web3_http_provider_logger = None
 
-def _get_logger(name: str) -> logging.Logger:
-    """Get a logger with the given ``name`` and configure it for usage with Click."""
-    cli_logger = logging.getLogger(name)
+
+def _format_logger(_logger: logging.Logger, fmt: str):
     handler = ClickHandler(echo_kwargs=CLICK_ECHO_KWARGS)
-    handler.setFormatter(ApeColorFormatter())
-    cli_logger.handlers = [handler]
-    return cli_logger
+    formatter = ApeColorFormatter(fmt=fmt)
+    handler.setFormatter(formatter)
+
+    # Remove existing handler(s)
+    for existing_handler in _logger.handlers[:]:
+        if isinstance(existing_handler, ClickHandler):
+            _logger.removeHandler(existing_handler)
+
+    _logger.addHandler(handler)
+
+
+def _get_logger(name: str, fmt: Optional[str] = None) -> logging.Logger:
+    """Get a logger with the given ``name`` and configure it for usage with Click."""
+    obj = logging.getLogger(name)
+    _format_logger(obj, fmt=fmt or DEFAULT_LOG_FORMAT)
+    return obj
 
 
 def _get_level(level: Optional[Union[str, int]] = None) -> str:
@@ -209,7 +254,7 @@ def _get_level(level: Optional[Union[str, int]] = None) -> str:
     return level
 
 
-logger = CliLogger()
+logger = CliLogger.create()
 
 
 __all__ = ["DEFAULT_LOG_LEVEL", "logger", "LogLevel"]

src/ape_geth/provider.py

@@ -36,7 +36,14 @@
 from web3.providers.auto import load_provider_from_environment
 from yarl import URL
 
-from ape.api import PluginConfig, TestProviderAPI, TransactionAPI, UpstreamProvider, Web3Provider
+from ape.api import (
+    PluginConfig,
+    SubprocessProvider,
+    TestProviderAPI,
+    TransactionAPI,
+    UpstreamProvider,
+    Web3Provider,
+)
 from ape.exceptions import APINotImplementedError, ProviderError
 from ape.logging import LogLevel, logger
 from ape.types import CallTreeNode, SnapshotID, SourceTraceback, TraceFrame
@@ -135,6 +142,11 @@ def make_logs_paths(stream_name: str):
             stderr_logfile_path=make_logs_paths("stderr"),
         )
 
+        if logger.level <= LogLevel.DEBUG:
+            # Show process output.
+            self.register_stdout_callback(lambda x: logger.debug)
+            self.register_stderr_callback(lambda x: logger.debug)
+
     @classmethod
     def from_uri(cls, uri: str, data_folder: Path, **kwargs):
         parsed_uri = URL(uri)
@@ -188,6 +200,12 @@ def _clean(self):
         if self.data_dir.is_dir():
             shutil.rmtree(self.data_dir)
 
+    def wait(self, *args, **kwargs):
+        if self.proc is None:
+            return
+
+        self.proc.wait(*args, **kwargs)
+
 
 class GethNetworkConfig(PluginConfig):
     # Make sure you are running the right networks when you try for these
@@ -409,19 +427,23 @@ def _stream_request(self, method: str, params: List, iter_path="result.item"):
             del results[:]
 
 
-class GethDev(BaseGethProvider, TestProviderAPI):
+class GethDev(BaseGethProvider, TestProviderAPI, SubprocessProvider):
     _process: Optional[GethDevProcess] = None
     name: str = "geth"
     _can_use_parity_traces = False
 
+    @property
+    def process_name(self) -> str:
+        return self.name
+
     @property
     def chain_id(self) -> int:
         return GETH_DEV_CHAIN_ID
 
     @property
     def data_dir(self) -> Path:
-        # Overriden from BaseGeth class for placing debug logs in ape data folder.
-        return self.geth_config.data_dir or self.data_folder / "dev"
+        # Overridden from BaseGeth class for placing debug logs in ape data folder.
+        return self.geth_config.data_dir or self.data_folder / self.name
 
     def __repr__(self):
         if self._process is None:
@@ -455,12 +477,19 @@ def _start_geth(self):
 
         self._process = process
 
+        # For subprocess-provider
+        if self._process is not None and (process := self._process.proc):
+            self.process = process
+
     def disconnect(self):
         # Must disconnect process first.
         if self._process is not None:
             self._process.disconnect()
             self._process = None
 
+        # Also unset the subprocess-provider reference.
+        self.process = None
+
         super().disconnect()
 
     def snapshot(self) -> SnapshotID:
@@ -596,6 +625,9 @@ def _eth_call(self, arguments: List) -> bytes:
     def get_call_tree(self, txn_hash: str, **root_node_kwargs) -> CallTreeNode:
         return self._get_geth_call_tree(txn_hash, **root_node_kwargs)
 
+    def build_command(self) -> List[str]:
+        return self._process.command if self._process else []
+
 
 class Geth(BaseGethProvider, UpstreamProvider):
     @property