From 189e1a758f8f025d3567a4def889c81c63dfdf1f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 11 Aug 2024 12:21:49 +0800 Subject: [PATCH 01/11] feat[tool]: add all imported modules to `-f annotated_ast` output --- vyper/ast/nodes.py | 6 ++++++ vyper/ast/nodes.pyi | 1 + vyper/compiler/output.py | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/vyper/ast/nodes.py b/vyper/ast/nodes.py index b4042c75a7..03ab189fd2 100644 --- a/vyper/ast/nodes.py +++ b/vyper/ast/nodes.py @@ -944,6 +944,12 @@ def validate(self): class Ellipsis(Constant): __slots__ = () + def to_dict(self): + ast_dict = super().to_dict() + # python ast ellipsis() is not json serializable; use a string + ast_dict["value"] = self.node_source_code + return ast_dict + class Dict(ExprNode): __slots__ = ("keys", "values") diff --git a/vyper/ast/nodes.pyi b/vyper/ast/nodes.pyi index 58c7d0b2e4..3e3a9a62b2 100644 --- a/vyper/ast/nodes.pyi +++ b/vyper/ast/nodes.pyi @@ -70,6 +70,7 @@ class TopLevel(VyperNode): class Module(TopLevel): path: str = ... resolved_path: str = ... + source_id: int = ... def namespace(self) -> Any: ... # context manager class FunctionDef(TopLevel): diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 577afd3822..2c42e5836a 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -3,6 +3,7 @@ from collections import deque from pathlib import PurePath +import vyper.ast as vy_ast from vyper.ast import ast_to_dict from vyper.codegen.ir_node import IRnode from vyper.compiler.output_bundle import SolcJSONWriter, VyperArchiveWriter @@ -11,7 +12,9 @@ from vyper.evm import opcodes from vyper.exceptions import VyperException from vyper.ir import compile_ir +from vyper.semantics.analysis.base import ModuleInfo from vyper.semantics.types.function import FunctionVisibility, StateMutability +from vyper.semantics.types.module import InterfaceT from vyper.typing import StorageLayout from vyper.utils import vyper_warn from vyper.warnings import ContractSizeLimitWarning @@ -26,9 +29,27 @@ def build_ast_dict(compiler_data: CompilerData) -> dict: def build_annotated_ast_dict(compiler_data: CompilerData) -> dict: + imported_module_infos = compiler_data.global_ctx.reachable_imports + unique_modules: dict[int, vy_ast.Module] = {} + for info in imported_module_infos: + if isinstance(info.typ, InterfaceT): + ast = info.typ.decl_node + if ast is None: # json abi + continue + else: + assert isinstance(info.typ, ModuleInfo) + ast = info.typ.module_t._module + + assert isinstance(ast, vy_ast.Module) # help mypy + if ast.source_id in unique_modules: + # sanity check -- object equality + assert unique_modules[ast.source_id] is ast + unique_modules[ast.source_id] = ast + annotated_ast_dict = { "contract_name": str(compiler_data.contract_path), "ast": ast_to_dict(compiler_data.annotated_vyper_module), + "imports": [ast_to_dict(ast) for ast in unique_modules.values()], } return annotated_ast_dict From 33ecc011a3674b4df7579e92b1e3315841745f37 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 11 Aug 2024 12:26:43 +0800 Subject: [PATCH 02/11] add simple test --- tests/unit/ast/test_ast_dict.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 07da3c0ace..0ca93e82bc 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -1,3 +1,4 @@ +import copy import json from vyper import compiler @@ -216,24 +217,27 @@ def foo(): input_bundle = make_input_bundle({"lib1.vy": lib1, "main.vy": main}) lib1_file = input_bundle.load_file("lib1.vy") - out = compiler.compile_from_file_input( + lib1_out = compiler.compile_from_file_input( lib1_file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] ) - lib1_ast = out["annotated_ast_dict"]["ast"] + + lib1_ast = copy.deepcopy(lib1_out["annotated_ast_dict"]["ast"]) lib1_sha256sum = lib1_ast.pop("source_sha256sum") assert lib1_sha256sum == lib1_file.sha256sum to_strip = NODE_SRC_ATTRIBUTES + ("resolved_path", "variable_reads", "variable_writes") _strip_source_annotations(lib1_ast, to_strip=to_strip) main_file = input_bundle.load_file("main.vy") - out = compiler.compile_from_file_input( + main_out = compiler.compile_from_file_input( main_file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] ) - main_ast = out["annotated_ast_dict"]["ast"] + main_ast = main_out["annotated_ast_dict"]["ast"] main_sha256sum = main_ast.pop("source_sha256sum") assert main_sha256sum == main_file.sha256sum _strip_source_annotations(main_ast, to_strip=to_strip) + assert main_out["annotated_ast_dict"]["imports"][0] == lib1_out["annotated_ast_dict"]["ast"] + # TODO: would be nice to refactor this into bunch of small test cases assert main_ast == { "ast_type": "Module", From 6a588b6eec632a9e206c4a56b98dcd36a10831f0 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 11 Aug 2024 12:28:39 +0800 Subject: [PATCH 03/11] lint: fix an import --- vyper/compiler/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 2c42e5836a..fc3b807d40 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -4,7 +4,7 @@ from pathlib import PurePath import vyper.ast as vy_ast -from vyper.ast import ast_to_dict +from vyper.ast.utils import ast_to_dict from vyper.codegen.ir_node import IRnode from vyper.compiler.output_bundle import SolcJSONWriter, VyperArchiveWriter from vyper.compiler.phases import CompilerData From 6ca42a5a3e6a38fb644c214da03db73bb3f4fa11 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 11 Aug 2024 13:26:41 +0800 Subject: [PATCH 04/11] fix a bad assertion --- vyper/compiler/output.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index fc3b807d40..0e3e91b23a 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -30,7 +30,7 @@ def build_ast_dict(compiler_data: CompilerData) -> dict: def build_annotated_ast_dict(compiler_data: CompilerData) -> dict: imported_module_infos = compiler_data.global_ctx.reachable_imports - unique_modules: dict[int, vy_ast.Module] = {} + unique_modules: dict[str, vy_ast.Module] = {} for info in imported_module_infos: if isinstance(info.typ, InterfaceT): ast = info.typ.decl_node @@ -41,10 +41,13 @@ def build_annotated_ast_dict(compiler_data: CompilerData) -> dict: ast = info.typ.module_t._module assert isinstance(ast, vy_ast.Module) # help mypy - if ast.source_id in unique_modules: + # use resolved_path for uniqueness, since Module objects can actually + # come from multiple InputBundles (particularly builtin interfaces), + # so source_id is not guaranteed to be unique. + if ast.resolved_path in unique_modules: # sanity check -- object equality - assert unique_modules[ast.source_id] is ast - unique_modules[ast.source_id] = ast + assert unique_modules[ast.resolved_path] is ast + unique_modules[ast.resolved_path] = ast annotated_ast_dict = { "contract_name": str(compiler_data.contract_path), From 7064b54b492872550460e340cb6079825b68bc4d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sun, 11 Aug 2024 14:00:14 +0800 Subject: [PATCH 05/11] fix another test --- vyper/compiler/output.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 0e3e91b23a..65bed8b4f1 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -29,7 +29,8 @@ def build_ast_dict(compiler_data: CompilerData) -> dict: def build_annotated_ast_dict(compiler_data: CompilerData) -> dict: - imported_module_infos = compiler_data.global_ctx.reachable_imports + module_t = compiler_data.annotated_vyper_module._metadata["type"] + imported_module_infos = module_t.reachable_imports unique_modules: dict[str, vy_ast.Module] = {} for info in imported_module_infos: if isinstance(info.typ, InterfaceT): From 4d5e50d749a726fc03b87e7c74dce07e2d8376af Mon Sep 17 00:00:00 2001 From: cyberthirst Date: Thu, 10 Oct 2024 13:08:50 +0200 Subject: [PATCH 06/11] make comment more clear --- vyper/compiler/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 65bed8b4f1..ebb32959e6 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -46,7 +46,7 @@ def build_annotated_ast_dict(compiler_data: CompilerData) -> dict: # come from multiple InputBundles (particularly builtin interfaces), # so source_id is not guaranteed to be unique. if ast.resolved_path in unique_modules: - # sanity check -- object equality + # sanity check -- objects must be identical assert unique_modules[ast.resolved_path] is ast unique_modules[ast.resolved_path] = ast From 1c83b891b8e1daa89d79aedc5fdd21d27acdfc1e Mon Sep 17 00:00:00 2001 From: cyberthirst Date: Thu, 10 Oct 2024 15:06:21 +0200 Subject: [PATCH 07/11] add comment --- vyper/compiler/output.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index ebb32959e6..e1a4e86f68 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -30,6 +30,7 @@ def build_ast_dict(compiler_data: CompilerData) -> dict: def build_annotated_ast_dict(compiler_data: CompilerData) -> dict: module_t = compiler_data.annotated_vyper_module._metadata["type"] + # get all reachable imports including recursion imported_module_infos = module_t.reachable_imports unique_modules: dict[str, vy_ast.Module] = {} for info in imported_module_infos: From 4eff20a664ef5b6890561dbb4bc7548b06c4c7ac Mon Sep 17 00:00:00 2001 From: cyberthirst Date: Thu, 10 Oct 2024 15:07:05 +0200 Subject: [PATCH 08/11] add ast export test with recursion --- tests/unit/ast/test_ast_dict.py | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 0ca93e82bc..94586b762b 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -1780,3 +1780,48 @@ def qux2(): }, } ] + +def test_annotated_ast_export_recursion(make_input_bundle): + sources = { + "main.vy": """ +import lib1 + +@external +def foo(): + lib1.foo() + """, + "lib1.vy": """ +import lib2 + +def foo(): + lib2.foo() + """, + "lib2.vy": """ +def foo(): + pass + """ + } + + input_bundle = make_input_bundle(sources) + + def compile_and_get_ast(file_name): + file = input_bundle.load_file(file_name) + output = compiler.compile_from_file_input( + file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] + ) + return output["annotated_ast_dict"] + + lib1_ast = compile_and_get_ast("lib1.vy")["ast"] + lib2_ast = compile_and_get_ast("lib2.vy")["ast"] + main_out = compile_and_get_ast("main.vy") + + lib1_import_ast = main_out["imports"][1] + lib2_import_ast = main_out["imports"][0] + + # path is once virtual, once libX.vy + # type contains name which is based on path + kws = [s for s in lib1_import_ast.keys() if s not in {"path", "type"}] + + for kw in kws: + assert lib1_ast[kw] == lib1_import_ast[kw] + assert lib2_ast[kw] == lib2_import_ast[kw] \ No newline at end of file From e36a32bdec7d695d8ef348e42a292f9b21440f1e Mon Sep 17 00:00:00 2001 From: cyberthirst Date: Thu, 10 Oct 2024 17:18:24 +0200 Subject: [PATCH 09/11] improve variable name --- tests/unit/ast/test_ast_dict.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 94586b762b..0b4fc2c2a7 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -1820,8 +1820,8 @@ def compile_and_get_ast(file_name): # path is once virtual, once libX.vy # type contains name which is based on path - kws = [s for s in lib1_import_ast.keys() if s not in {"path", "type"}] + keys = [s for s in lib1_import_ast.keys() if s not in {"path", "type"}] - for kw in kws: + for kw in keys: assert lib1_ast[kw] == lib1_import_ast[kw] assert lib2_ast[kw] == lib2_import_ast[kw] \ No newline at end of file From e134a2687ca91d7f06ac70310ccdd0fce0e6d814 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 10 Oct 2024 12:28:40 -0700 Subject: [PATCH 10/11] Update tests/unit/ast/test_ast_dict.py --- tests/unit/ast/test_ast_dict.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 0b4fc2c2a7..924e08193e 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -1822,6 +1822,6 @@ def compile_and_get_ast(file_name): # type contains name which is based on path keys = [s for s in lib1_import_ast.keys() if s not in {"path", "type"}] - for kw in keys: - assert lib1_ast[kw] == lib1_import_ast[kw] - assert lib2_ast[kw] == lib2_import_ast[kw] \ No newline at end of file + for key in keys: + assert lib1_ast[key] == lib1_import_ast[key] + assert lib2_ast[key] == lib2_import_ast[key] \ No newline at end of file From 29946766d537994901f4b6e90087476d7b2a2189 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 10 Oct 2024 19:12:57 -0400 Subject: [PATCH 11/11] fix lint --- tests/unit/ast/test_ast_dict.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 924e08193e..c9d7248809 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -1781,6 +1781,7 @@ def qux2(): } ] + def test_annotated_ast_export_recursion(make_input_bundle): sources = { "main.vy": """ @@ -1799,7 +1800,7 @@ def foo(): "lib2.vy": """ def foo(): pass - """ + """, } input_bundle = make_input_bundle(sources) @@ -1824,4 +1825,4 @@ def compile_and_get_ast(file_name): for key in keys: assert lib1_ast[key] == lib1_import_ast[key] - assert lib2_ast[key] == lib2_import_ast[key] \ No newline at end of file + assert lib2_ast[key] == lib2_import_ast[key]