From 3a0bf34dedc1ae7c06b30363b5e6412edd6cdf1b Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 1 Aug 2023 13:02:39 -0500 Subject: [PATCH] feat: `networks run` CLI [APE-981] (#1576) --- .pre-commit-config.yaml | 2 +- docs/userguides/networks.md | 15 ++++ setup.py | 2 +- src/ape/api/providers.py | 5 +- src/ape/cli/options.py | 18 +++-- src/ape/logging.py | 83 +++++++++++++++++----- src/ape_geth/provider.py | 40 +++++++++-- src/ape_networks/_cli.py | 55 +++++++++++++- tests/functional/test_chain.py | 2 +- tests/functional/test_contract_instance.py | 6 +- tests/functional/test_logging.py | 14 ++++ tests/functional/test_project.py | 4 +- tests/functional/test_provider.py | 2 +- tests/integration/cli/test_compile.py | 2 +- tests/integration/cli/test_networks.py | 10 +++ 15 files changed, 217 insertions(+), 43 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec003a879d..fa218c2dc3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: name: black - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 diff --git a/docs/userguides/networks.md b/docs/userguides/networks.md index 364f9974ee..e685bbac44 100644 --- a/docs/userguides/networks.md +++ b/docs/userguides/networks.md @@ -108,3 +108,18 @@ Some reasons for this include: 3. Response differences in uncommon blocks, such as the `"pending"` block or the genesis block. 4. Revert messages and exception-handling differences. 5. You are limited to using `web3.py` and EVM-based chains. + +## Running a Network Process + +To run a network with a process, use the `ape networks run` command: + +```shell +ape networks run +``` + +By default, `ape networks run` runs a development Geth process. +To use a different network, such as `hardhat` or Anvil nodes, use the `--network` flag: + +```shell +ape networks run --network ethereum:local:foundry +``` diff --git a/setup.py b/setup.py index 71493092ab..e03cafe4de 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ "types-setuptools", # Needed due to mypy typeshed "pandas-stubs==1.2.0.62", # Needed due to mypy typeshed "types-SQLAlchemy>=1.4.49", # Needed due to mypy typeshed - "flake8>=6.0.0,<7", # Style linter + "flake8>=6.1.0,<7", # Style linter "flake8-breakpoint>=1.1.0,<2", # detect breakpoints left in code "flake8-print>=4.0.1,<5", # detect print statements left in code "isort>=5.10.1,<6", # Import sorting linter diff --git a/src/ape/api/providers.py b/src/ape/api/providers.py index d2818c9e1b..8d1fd95854 100644 --- a/src/ape/api/providers.py +++ b/src/ape/api/providers.py @@ -1623,9 +1623,8 @@ def start(self, timeout: int = 20): self.stderr_queue = JoinableQueue() self.stdout_queue = JoinableQueue() out_file = PIPE if logger.level <= LogLevel.DEBUG else DEVNULL - self.process = Popen( - self.build_command(), preexec_fn=pre_exec_fn, stdout=out_file, stderr=out_file - ) + cmd = self.build_command() + self.process = Popen(cmd, preexec_fn=pre_exec_fn, stdout=out_file, stderr=out_file) spawn(self.produce_stdout_queue) spawn(self.produce_stderr_queue) spawn(self.consume_stdout_queue) diff --git a/src/ape/cli/options.py b/src/ape/cli/options.py index c2fb1b8a8a..47804f2621 100644 --- a/src/ape/cli/options.py +++ b/src/ape/cli/options.py @@ -47,16 +47,18 @@ def abort(msg: str, base_error: Optional[Exception] = None) -> NoReturn: raise Abort(msg) -def verbosity_option(cli_logger: Optional[CliLogger] = None): +def verbosity_option(cli_logger: Optional[CliLogger] = None, default: str = DEFAULT_LOG_LEVEL): """A decorator that adds a `--verbosity, -v` option to the decorated command. """ _logger = cli_logger or logger - kwarguments = _create_verbosity_kwargs(_logger=_logger) + kwarguments = _create_verbosity_kwargs(_logger=_logger, default=default) return lambda f: click.option(*_VERBOSITY_VALUES, **kwarguments)(f) -def _create_verbosity_kwargs(_logger: Optional[CliLogger] = None) -> Dict: +def _create_verbosity_kwargs( + _logger: Optional[CliLogger] = None, default: str = DEFAULT_LOG_LEVEL +) -> Dict: cli_logger = _logger or logger def set_level(ctx, param, value): @@ -66,7 +68,7 @@ def set_level(ctx, param, value): names_str = f"{', '.join(level_names[:-1])}, or {level_names[-1]}" return { "callback": set_level, - "default": DEFAULT_LOG_LEVEL, + "default": default or DEFAULT_LOG_LEVEL, "metavar": "LVL", "expose_value": False, "help": f"One of {names_str}", @@ -74,15 +76,19 @@ def set_level(ctx, param, value): } -def ape_cli_context(): +def ape_cli_context(default_log_level: str = DEFAULT_LOG_LEVEL): """ A ``click`` context object with helpful utilities. Use in your commands to get access to common utility features, such as logging or accessing managers. + + Args: + default_log_level (str): The log-level value to pass to + :meth:`~ape.cli.options.verbosity_option`. """ def decorator(f): - f = verbosity_option(logger)(f) + f = verbosity_option(logger, default=default_log_level)(f) f = click.make_pass_decorator(ApeCliContextObject, ensure=True)(f) return f diff --git a/src/ape/logging.py b/src/ape/logging.py index 8eb67a288e..9fabfafeaf 100644 --- a/src/ape/logging.py +++ b/src/ape/logging.py @@ -19,6 +19,7 @@ class LogLevel(IntEnum): logging.addLevelName(LogLevel.SUCCESS.value, LogLevel.SUCCESS.name) logging.SUCCESS = LogLevel.SUCCESS.value # type: ignore DEFAULT_LOG_LEVEL = LogLevel.INFO.name +DEFAULT_LOG_FORMAT = "%(levelname)s: %(message)s" def success(self, message, *args, **kws): @@ -61,8 +62,9 @@ def _isatty(stream: IO) -> bool: class ApeColorFormatter(logging.Formatter): - def __init__(self): - super().__init__(fmt="%(levelname)s: %(message)s") + def __init__(self, fmt: Optional[str] = None): + fmt = fmt or DEFAULT_LOG_FORMAT + super().__init__(fmt=fmt) def format(self, record): if _isatty(sys.stdout) and _isatty(sys.stderr): @@ -95,17 +97,43 @@ def emit(self, record): class CliLogger: _mentioned_verbosity_option = False - def __init__(self): - _logger = _get_logger("ape") + def __init__( + self, + _logger: logging.Logger, + fmt: str, + web3_request_logger: Optional[logging.Logger] = None, + web3_http_logger: Optional[logging.Logger] = None, + ): self.error = _logger.error self.warning = _logger.warning self.success = getattr(_logger, "success", _logger.info) self.info = _logger.info self.debug = _logger.debug self._logger = _logger - self._web3_request_manager_logger = _get_logger("web3.RequestManager") - self._web3_http_provider_logger = _get_logger("web3.providers.HTTPProvider") + self._web3_request_manager_logger = web3_request_logger + self._web3_http_provider_logger = web3_http_logger self._load_from_sys_argv() + self.fmt = fmt + + @classmethod + def create(cls, fmt: Optional[str] = None, third_party: bool = True) -> "CliLogger": + fmt = fmt or DEFAULT_LOG_FORMAT + kwargs = {} + if third_party: + kwargs["web3_request_logger"] = _get_logger("web3.RequestManager", fmt=fmt) + kwargs["web3_http_logger"] = _get_logger("web3.providers.HTTPProvider", fmt=fmt) + + _logger = _get_logger("ape", fmt=fmt) + return cls(_logger, fmt, **kwargs) + + def format(self, fmt: Optional[str] = None): + self.fmt = fmt or DEFAULT_LOG_FORMAT + fmt = fmt or DEFAULT_LOG_FORMAT + _format_logger(self._logger, fmt) + if req_log := self._web3_request_manager_logger: + _format_logger(req_log, fmt) + if prov_log := self._web3_http_provider_logger: + _format_logger(prov_log, fmt) def _load_from_sys_argv(self, default: Optional[Union[str, int]] = None): """ @@ -126,9 +154,7 @@ def _load_from_sys_argv(self, default: Optional[Union[str, int]] = None): self._logger.error(f"Must be one of '{names_str}', not '{level}'.") sys.exit(2) - self._logger.setLevel(log_level) - self._web3_request_manager_logger.setLevel(log_level) - self._web3_http_provider_logger.setLevel(log_level) + self.set_level(log_level) @property def level(self) -> int: @@ -145,9 +171,13 @@ def set_level(self, level: Union[str, int]): if level == self._logger.level: return - self._logger.setLevel(level) - self._web3_request_manager_logger.setLevel(level) - self._web3_http_provider_logger.setLevel(level) + for log in ( + self._logger, + self._web3_http_provider_logger, + self._web3_request_manager_logger, + ): + if obj := log: + obj.setLevel(level) def log_error(self, err: Exception): """ @@ -190,14 +220,29 @@ def log_debug_stack_trace(self): stack_trace = traceback.format_exc() self._logger.debug(stack_trace) + def _clear_web3_loggers(self): + self._web3_request_manager_logger = None + self._web3_http_provider_logger = None -def _get_logger(name: str) -> logging.Logger: - """Get a logger with the given ``name`` and configure it for usage with Click.""" - cli_logger = logging.getLogger(name) + +def _format_logger(_logger: logging.Logger, fmt: str): handler = ClickHandler(echo_kwargs=CLICK_ECHO_KWARGS) - handler.setFormatter(ApeColorFormatter()) - cli_logger.handlers = [handler] - return cli_logger + formatter = ApeColorFormatter(fmt=fmt) + handler.setFormatter(formatter) + + # Remove existing handler(s) + for existing_handler in _logger.handlers[:]: + if isinstance(existing_handler, ClickHandler): + _logger.removeHandler(existing_handler) + + _logger.addHandler(handler) + + +def _get_logger(name: str, fmt: Optional[str] = None) -> logging.Logger: + """Get a logger with the given ``name`` and configure it for usage with Click.""" + obj = logging.getLogger(name) + _format_logger(obj, fmt=fmt or DEFAULT_LOG_FORMAT) + return obj def _get_level(level: Optional[Union[str, int]] = None) -> str: @@ -209,7 +254,7 @@ def _get_level(level: Optional[Union[str, int]] = None) -> str: return level -logger = CliLogger() +logger = CliLogger.create() __all__ = ["DEFAULT_LOG_LEVEL", "logger", "LogLevel"] diff --git a/src/ape_geth/provider.py b/src/ape_geth/provider.py index 6c308d254a..da8a6e5622 100644 --- a/src/ape_geth/provider.py +++ b/src/ape_geth/provider.py @@ -36,7 +36,14 @@ from web3.providers.auto import load_provider_from_environment from yarl import URL -from ape.api import PluginConfig, TestProviderAPI, TransactionAPI, UpstreamProvider, Web3Provider +from ape.api import ( + PluginConfig, + SubprocessProvider, + TestProviderAPI, + TransactionAPI, + UpstreamProvider, + Web3Provider, +) from ape.exceptions import APINotImplementedError, ProviderError from ape.logging import LogLevel, logger from ape.types import CallTreeNode, SnapshotID, SourceTraceback, TraceFrame @@ -135,6 +142,11 @@ def make_logs_paths(stream_name: str): stderr_logfile_path=make_logs_paths("stderr"), ) + if logger.level <= LogLevel.DEBUG: + # Show process output. + self.register_stdout_callback(lambda x: logger.debug) + self.register_stderr_callback(lambda x: logger.debug) + @classmethod def from_uri(cls, uri: str, data_folder: Path, **kwargs): parsed_uri = URL(uri) @@ -188,6 +200,12 @@ def _clean(self): if self.data_dir.is_dir(): shutil.rmtree(self.data_dir) + def wait(self, *args, **kwargs): + if self.proc is None: + return + + self.proc.wait(*args, **kwargs) + class GethNetworkConfig(PluginConfig): # Make sure you are running the right networks when you try for these @@ -409,19 +427,23 @@ def _stream_request(self, method: str, params: List, iter_path="result.item"): del results[:] -class GethDev(BaseGethProvider, TestProviderAPI): +class GethDev(BaseGethProvider, TestProviderAPI, SubprocessProvider): _process: Optional[GethDevProcess] = None name: str = "geth" _can_use_parity_traces = False + @property + def process_name(self) -> str: + return self.name + @property def chain_id(self) -> int: return GETH_DEV_CHAIN_ID @property def data_dir(self) -> Path: - # Overriden from BaseGeth class for placing debug logs in ape data folder. - return self.geth_config.data_dir or self.data_folder / "dev" + # Overridden from BaseGeth class for placing debug logs in ape data folder. + return self.geth_config.data_dir or self.data_folder / self.name def __repr__(self): if self._process is None: @@ -455,12 +477,19 @@ def _start_geth(self): self._process = process + # For subprocess-provider + if self._process is not None and (process := self._process.proc): + self.process = process + def disconnect(self): # Must disconnect process first. if self._process is not None: self._process.disconnect() self._process = None + # Also unset the subprocess-provider reference. + self.process = None + super().disconnect() def snapshot(self) -> SnapshotID: @@ -596,6 +625,9 @@ def _eth_call(self, arguments: List) -> bytes: def get_call_tree(self, txn_hash: str, **root_node_kwargs) -> CallTreeNode: return self._get_geth_call_tree(txn_hash, **root_node_kwargs) + def build_command(self) -> List[str]: + return self._process.command if self._process else [] + class Geth(BaseGethProvider, UpstreamProvider): @property diff --git a/src/ape_networks/_cli.py b/src/ape_networks/_cli.py index 2635bab1e2..c9ee7c951b 100644 --- a/src/ape_networks/_cli.py +++ b/src/ape_networks/_cli.py @@ -5,9 +5,11 @@ from rich.tree import Tree from ape import networks -from ape.cli import ape_cli_context +from ape.api import SubprocessProvider +from ape.cli import ape_cli_context, network_option from ape.cli.choices import OutputFormat from ape.cli.options import output_format_option +from ape.logging import LogLevel def _filter_option(name: str, options): @@ -73,3 +75,54 @@ def make_sub_tree(data: Dict, create_tree: Callable) -> Tree: elif output_format == OutputFormat.YAML: click.echo(cli_ctx.network_manager.networks_yaml.strip()) + + +@cli.command() +@ape_cli_context() +@network_option(default="ethereum:local:geth") +def run(cli_ctx, network): + """ + Start a node process + """ + + # Ignore web3 logs + cli_ctx.logger._clear_web3_loggers() + + network_ctx = cli_ctx.network_manager.parse_network_choice(network) + provider = network_ctx._provider + if not isinstance(provider, SubprocessProvider): + cli_ctx.abort( + f"`ape networks run` requires a provider that manages a process, not '{provider.name}'." + ) + elif provider.is_connected: + cli_ctx.abort("Process already running.") + + # Start showing process logs. + original_level = cli_ctx.logger.level + original_format = cli_ctx.logger.fmt + cli_ctx.logger.set_level(LogLevel.DEBUG) + + # Change format to exclude log level (since it is always just DEBUG) + cli_ctx.logger.format(fmt="%(message)s") + try: + _run(cli_ctx, provider) + finally: + cli_ctx.logger.set_level(original_level) + cli_ctx.logger.format(fmt=original_format) + + +def _run(cli_ctx, provider: SubprocessProvider): + provider.connect() + if process := provider.process: + try: + process.wait() + finally: + try: + provider.disconnect() + except Exception: + # Prevent not being able to CTRL-C. + cli_ctx.abort("Terminated") + + else: + provider.disconnect() + cli_ctx.abort("Process already running.") diff --git a/tests/functional/test_chain.py b/tests/functional/test_chain.py index 08d3dfecdb..fb820781b7 100644 --- a/tests/functional/test_chain.py +++ b/tests/functional/test_chain.py @@ -449,7 +449,7 @@ def test_get_deployments_local(chain, owner, contract_0, contract_1): # Assert for contract_list in (contracts_list_0, contracts_list_1): - assert type(contract_list[0]) == ContractInstance + assert type(contract_list[0]) is ContractInstance index_0 = len(contracts_list_0) - len(starting_contracts_list_0) - 1 index_1 = len(contracts_list_1) - len(starting_contracts_list_1) - 1 diff --git a/tests/functional/test_contract_instance.py b/tests/functional/test_contract_instance.py index fc86fc3401..360a41cc7f 100644 --- a/tests/functional/test_contract_instance.py +++ b/tests/functional/test_contract_instance.py @@ -217,7 +217,7 @@ def test_structs(contract_instance, sender, chain): # Expected: b == block.prevhash. assert actual.b == actual["b"] == actual[1] == actual_prev_block == chain.blocks[-2].hash - assert type(actual.b) == HexBytes + assert type(actual.b) is HexBytes def test_nested_structs(contract_instance, sender, chain): @@ -244,7 +244,7 @@ def test_nested_structs(contract_instance, sender, chain): == actual_prev_block_1 == chain.blocks[-2].hash ) - assert type(actual_1.t.b) == HexBytes + assert type(actual_1.t.b) is HexBytes assert ( actual_2.t.b == actual_2.t["b"] @@ -252,7 +252,7 @@ def test_nested_structs(contract_instance, sender, chain): == actual_prev_block_2 == chain.blocks[-2].hash ) - assert type(actual_2.t.b) == HexBytes + assert type(actual_2.t.b) is HexBytes def test_nested_structs_in_tuples(contract_instance, sender, chain): diff --git a/tests/functional/test_logging.py b/tests/functional/test_logging.py index 2fc0a35913..4d78fea63f 100644 --- a/tests/functional/test_logging.py +++ b/tests/functional/test_logging.py @@ -84,3 +84,17 @@ def cmd(cli_ctx): result = simple_runner.invoke(group_for_testing, ["cmd", "-v", "WARNING"]) assert "SUCCESS" not in result.output assert "this is a test" not in result.output + + +def test_format(simple_runner): + @group_for_testing.command() + @ape_cli_context() + def cmd(cli_ctx): + cli_ctx.logger.format(fmt="%(message)s") + try: + cli_ctx.logger.success("this is a test") + finally: + cli_ctx.logger.format() + + result = simple_runner.invoke(group_for_testing, ["cmd", "-v", "SUCCESS"]) + assert "SUCCESS" not in result.output diff --git a/tests/functional/test_project.py b/tests/functional/test_project.py index 2bd74ffe6e..7f7eda4263 100644 --- a/tests/functional/test_project.py +++ b/tests/functional/test_project.py @@ -109,8 +109,8 @@ def _make_new_contract(existing_contract: ContractType, name: str): def test_extract_manifest(project_with_dependency_config): # NOTE: Only setting dependency_config to ensure existence of project. manifest = project_with_dependency_config.extract_manifest() - assert type(manifest) == PackageManifest - assert type(manifest.compilers) == list + assert type(manifest) is PackageManifest + assert type(manifest.compilers) is list assert manifest.meta == project_with_dependency_config.meta assert manifest.compilers == project_with_dependency_config.compiler_data assert manifest.deployments == project_with_dependency_config.tracked_deployments diff --git a/tests/functional/test_provider.py b/tests/functional/test_provider.py index 58d59df02c..3ddc419971 100644 --- a/tests/functional/test_provider.py +++ b/tests/functional/test_provider.py @@ -173,7 +173,7 @@ def test_provider_get_balance(project, networks, accounts): """ balance = networks.provider.get_balance(accounts.test_accounts[0].address) - assert type(balance) == int + assert type(balance) is int assert balance == 1000000000000000000000000 diff --git a/tests/integration/cli/test_compile.py b/tests/integration/cli/test_compile.py index 19c71b0468..decf597c55 100644 --- a/tests/integration/cli/test_compile.py +++ b/tests/integration/cli/test_compile.py @@ -222,7 +222,7 @@ def test_compile_with_dependency(ape_cli, runner, project, contract_path): "renamed-contracts-folder-specified-in-config", ): assert name in list(project.dependencies.keys()) - assert type(project.dependencies[name]["local"]["name"]) == ContractContainer + assert type(project.dependencies[name]["local"]["name"]) is ContractContainer @skip_projects_except("with-dependencies") diff --git a/tests/integration/cli/test_networks.py b/tests/integration/cli/test_networks.py index 503c2e4b57..c367bfee8e 100644 --- a/tests/integration/cli/test_networks.py +++ b/tests/integration/cli/test_networks.py @@ -129,3 +129,13 @@ def test_filter_networks(ape_cli, runner, networks): def test_filter_providers(ape_cli, runner, networks): result = runner.invoke(ape_cli, ["networks", "list", "--provider", "test"]) assert_rich_text(result.output, _TEST_PROVIDER_TREE_OUTPUT) + + +@run_once +def test_node_not_subprocess_provider(ape_cli, runner): + result = runner.invoke(ape_cli, ["networks", "run", "--network", "ethereum:local:test"]) + assert result.exit_code != 0 + assert ( + result.output + == "ERROR: `ape networks run` requires a provider that manages a process, not 'test'.\n" + )