From f46fe5f04bd0340b0c670ee00f00d2da109bbb43 Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Thu, 28 Mar 2024 14:49:15 +0100 Subject: [PATCH 1/8] simplify tests --- tests/conftest.py | 37 ++++++++- tests/test_types_echidna.py | 147 +++--------------------------------- tests/test_types_medusa.py | 147 +++--------------------------------- 3 files changed, 59 insertions(+), 272 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fd6554e..0de60cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ """ Globally available fixtures""" import os -from typing import Any +import subprocess +import re +from typing import Any, Callable import pytest from slither import Slither @@ -99,3 +101,36 @@ def value_transfer() -> TestGenerator: corpus_dir = "corpus-value" return TestGenerator(target, target_path, corpus_dir) + + +def run_generation_command_test( + generate_tests: Callable, test_name: str, fuzzer: str, pattern: str +) -> None: + """Utility function to test unit test generation from a corpus and contract""" + generate_tests() + # Ensure the file was created + path = os.path.join(os.getcwd(), "test", f"{test_name}_{fuzzer}_Test.t.sol") + assert os.path.exists(path) + + # Ensure the file can be compiled + subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) + + # Ensure the file can be tested + result = subprocess.run( + ["forge", "test", "--match-contract", f"{test_name}_{fuzzer}_Test"], + capture_output=True, + text=True, + check=False, + ) + + # Remove ansi escape sequences + ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + output = ansi_escape.sub("", result.stdout) + + # Ensure all tests fail + match = re.search(pattern, output) + if match: + tests_passed = int(match.group(2)) + assert tests_passed == 0 + else: + assert False, "No tests were ran" diff --git a/tests/test_types_echidna.py b/tests/test_types_echidna.py index 1e46b35..955a13b 100644 --- a/tests/test_types_echidna.py +++ b/tests/test_types_echidna.py @@ -1,9 +1,6 @@ """ Tests for generating compilable test files from an Echidna corpus""" from pathlib import Path -import os -import re -import subprocess -from .conftest import TestGenerator +from .conftest import TestGenerator, run_generation_command_test TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data" PATTERN = r"(\d+)\s+failing tests,\s+(\d+)\s+tests succeeded" @@ -11,154 +8,34 @@ def test_echidna_basic_types(basic_types: TestGenerator) -> None: """Tests the BasicTypes contract with an Echidna corpus""" - basic_types.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "BasicTypes_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "BasicTypes_Echidna_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + basic_types.echidna_generate_tests, "BasicTypes", "Echidna", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_echidna_fixed_array_types(fixed_size_arrays: TestGenerator) -> None: """Tests the FixedArrays contract with an Echidna corpus""" - fixed_size_arrays.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "FixedArrays_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "FixedArrays_Echidna_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + fixed_size_arrays.echidna_generate_tests, "FixedArrays", "Echidna", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_echidna_dynamic_array_types(dynamic_arrays: TestGenerator) -> None: """Tests the DynamicArrays contract with an Echidna corpus""" - dynamic_arrays.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "DynamicArrays_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "DynamicArrays_Echidna_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + dynamic_arrays.echidna_generate_tests, "DynamicArrays", "Echidna", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_echidna_structs_and_enums(structs_and_enums: TestGenerator) -> None: """Tests the TupleTypes contract with an Echidna corpus""" - structs_and_enums.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "TupleTypes_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "TupleTypes_Echidna_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + structs_and_enums.echidna_generate_tests, "TupleTypes", "Echidna", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_echidna_value_transfer(value_transfer: TestGenerator) -> None: - """Tests the BasicTypes contract with an Echidna corpus""" - value_transfer.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "ValueTransfer_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "ValueTransfer_Echidna_Test"], - capture_output=True, - text=True, - check=False, + """Tests the ValueTransfer contract with an Echidna corpus""" + run_generation_command_test( + value_transfer.echidna_generate_tests, "ValueTransfer", "Echidna", PATTERN ) - - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" diff --git a/tests/test_types_medusa.py b/tests/test_types_medusa.py index 35ae84c..e3e9a71 100644 --- a/tests/test_types_medusa.py +++ b/tests/test_types_medusa.py @@ -1,10 +1,7 @@ """ Tests for generating compilable test files from an Medusa corpus""" from pathlib import Path -import os -import re -import subprocess import pytest -from .conftest import TestGenerator +from .conftest import TestGenerator, run_generation_command_test TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data" PATTERN = r"(\d+)\s+failing tests,\s+(\d+)\s+tests succeeded" @@ -12,155 +9,33 @@ def test_medusa_basic_types(basic_types: TestGenerator) -> None: """Tests the BasicTypes contract with a Medusa corpus""" - basic_types.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "BasicTypes_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "BasicTypes_Medusa_Test"], - capture_output=True, - text=True, - check=False, - ) - - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" + run_generation_command_test(basic_types.medusa_generate_tests, "BasicTypes", "Medusa", PATTERN) def test_medusa_fixed_array_types(fixed_size_arrays: TestGenerator) -> None: """Tests the FixedArrays contract with a Medusa corpus""" - fixed_size_arrays.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "FixedArrays_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "FixedArrays_Medusa_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + fixed_size_arrays.medusa_generate_tests, "FixedArrays", "Medusa", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_medusa_dynamic_array_types(dynamic_arrays: TestGenerator) -> None: """Tests the DynamicArrays contract with a Medusa corpus""" - dynamic_arrays.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "DynamicArrays_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "DynamicArrays_Medusa_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + dynamic_arrays.medusa_generate_tests, "DynamicArrays", "Medusa", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_medusa_structs_and_enums(structs_and_enums: TestGenerator) -> None: """Tests the TupleTypes contract with a Medusa corpus""" - structs_and_enums.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "TupleTypes_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "TupleTypes_Medusa_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + structs_and_enums.medusa_generate_tests, "TupleTypes", "Medusa", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - @pytest.mark.xfail(strict=True) # type: ignore[misc] def test_medusa_value_transfer(value_transfer: TestGenerator) -> None: - """Tests the BasicTypes contract with a Medusa corpus""" - value_transfer.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "ValueTransfer_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "ValueTransfer_Medusa_Test"], - capture_output=True, - text=True, - check=False, + """Tests the ValueTransfer contract with a Medusa corpus""" + run_generation_command_test( + value_transfer.medusa_generate_tests, "ValueTransfer", "Medusa", PATTERN ) - - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" From 68c4d5345cc2ee43694bb1e2381910fb4912952b Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Thu, 28 Mar 2024 14:55:30 +0100 Subject: [PATCH 2/8] run tests in temporary directory --- Makefile | 3 -- tests/conftest.py | 72 ++++++++++++++++++++++++++++++++++++- tests/test_harness.py | 29 +++++++++++---- tests/test_types_echidna.py | 27 +++++++++++--- tests/test_types_medusa.py | 27 +++++++++++--- 5 files changed, 137 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index fadabc9..3f13a80 100644 --- a/Makefile +++ b/Makefile @@ -68,9 +68,6 @@ reformat: test tests: $(VENV)/pyvenv.cfg . $(VENV_BIN)/activate && \ solc-select use 0.8.19 --always-install && \ - cd tests/test_data && \ - forge install && \ - cd ../.. && \ pytest --ignore tests/test_data/lib $(T) $(TEST_ARGS) .PHONY: package diff --git a/tests/conftest.py b/tests/conftest.py index 0de60cd..b8acaa1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import os import subprocess import re +import shutil from typing import Any, Callable import pytest @@ -40,7 +41,7 @@ def medusa_generate_tests(self) -> None: self.medusa_generator.create_poc() -@pytest.fixture(autouse=True) # type: ignore[misc] +@pytest.fixture() # type: ignore[misc] def change_test_dir(request: Any, monkeypatch: Any) -> None: """Helper fixture to change the working directory""" # Directory of the test file @@ -103,6 +104,75 @@ def value_transfer() -> TestGenerator: return TestGenerator(target, target_path, corpus_dir) +@pytest.fixture(scope="session") # type: ignore[misc] +def setup_foundry_temp_dir(tmp_path_factory): + """Sets up a temporary directory for the tests that contain all the necessary Foundry files""" + # Create a temporary directory valid for the session + temp_dir = tmp_path_factory.mktemp("foundry_session") + original_dir: str = os.getcwd() + + print("Installing Forge...") + subprocess.run(["forge", "init", "--no-git"], check=True, cwd=temp_dir) + subprocess.run(["forge", "install", "crytic/properties", "--no-git"], check=True, cwd=temp_dir) + subprocess.run( + ["forge", "install", "transmissions11/solmate", "--no-git"], check=True, cwd=temp_dir + ) + # Create remappings file + remappings = temp_dir / "remappings.txt" + with open(remappings, "w", encoding="utf-8") as outfile: + outfile.write( + "forge-std/=lib/forge-std/src/\nproperties/=lib/properties/contracts/\nsolmate/=lib/solmate/src/\nsrc/=src/" + ) + + # Delete unnecessary files + counter_path = temp_dir / "src" / "Counter.sol" + counter_path.unlink() + assert not counter_path.exists() + + counter_test_path = temp_dir / "test" / "Counter.t.sol" + counter_test_path.unlink() + assert not counter_test_path.exists() + + scripts_dir = temp_dir / "script" + shutil.rmtree(scripts_dir) + assert not scripts_dir.exists() + + # Create the corpora directories in the temporary dir + echidna_corpora_dir = temp_dir / "echidna-corpora" + medusa_corpora_dir = temp_dir / "medusa-corpora" + echidna_corpora_dir.mkdir(exist_ok=True) + medusa_corpora_dir.mkdir(exist_ok=True) + + # Copy all our contracts and corpora to the temporary directory + copy_directory_contents( + os.path.join(original_dir, "tests", "test_data", "echidna-corpora"), + temp_dir / "echidna-corpora", + ) + copy_directory_contents( + os.path.join(original_dir, "tests", "test_data", "medusa-corpora"), + temp_dir / "medusa-corpora", + ) + copy_directory_contents( + os.path.join(original_dir, "tests", "test_data", "src"), temp_dir / "src" + ) + + os.chdir(temp_dir) + + +def copy_directory_contents(src_dir, dest_dir) -> None: + """ + Copies the contents of src_dir into dest_dir. The dest_dir must already exist. + Directories under src_dir will be created under dest_dir and files will be copied. + """ + for item in os.listdir(src_dir): + s = os.path.join(src_dir, item) + d = os.path.join(dest_dir, item) + if os.path.isdir(s): + shutil.copytree(s, d, dirs_exist_ok=True) # For Python 3.8+, use dirs_exist_ok=True + else: + shutil.copy2(s, d) + + def run_generation_command_test( generate_tests: Callable, test_name: str, fuzzer: str, pattern: str ) -> None: diff --git a/tests/test_harness.py b/tests/test_harness.py index aeabab9..c75dc4e 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -2,6 +2,7 @@ from pathlib import Path import copy import subprocess +from pytest import TempPathFactory from slither import Slither from slither.core.declarations.contract import Contract from slither.core.declarations.function_contract import FunctionContract @@ -32,7 +33,9 @@ } -def test_modifier_filtering() -> None: +def test_modifier_filtering( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument +) -> None: """Test non-strict modifier filtering""" filters = { "strict": False, @@ -51,7 +54,9 @@ def test_modifier_filtering() -> None: ) -def test_external_call_filtering() -> None: +def test_external_call_filtering( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument +) -> None: """Test non-strict external call filtering""" filters = { "strict": False, @@ -70,7 +75,9 @@ def test_external_call_filtering() -> None: ) -def test_payable_filtering() -> None: +def test_payable_filtering( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument +) -> None: """Test non-strict payable call filtering""" filters = { "strict": False, @@ -89,7 +96,9 @@ def test_payable_filtering() -> None: ) -def test_modifier_and_external_call_filtering() -> None: +def test_modifier_and_external_call_filtering( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument +) -> None: """Test non-strict modifier and external call filtering""" filters = { "strict": False, @@ -108,7 +117,9 @@ def test_modifier_and_external_call_filtering() -> None: ) -def test_strict_modifier_and_external_call_filtering() -> None: +def test_strict_modifier_and_external_call_filtering( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument +) -> None: """Test strict modifier and external call filtering""" filters = { "strict": True, @@ -127,7 +138,9 @@ def test_strict_modifier_and_external_call_filtering() -> None: ) -def test_multiple_external_calls_filtering() -> None: +def test_multiple_external_calls_filtering( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument +) -> None: """Test multiple external calls filtering""" filters = { "strict": True, @@ -146,7 +159,9 @@ def test_multiple_external_calls_filtering() -> None: ) -def test_strict_multiple_external_calls_filtering() -> None: +def test_strict_multiple_external_calls_filtering( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument +) -> None: """Test strict multiple external calls filtering""" filters = { "strict": True, diff --git a/tests/test_types_echidna.py b/tests/test_types_echidna.py index 955a13b..962c3e9 100644 --- a/tests/test_types_echidna.py +++ b/tests/test_types_echidna.py @@ -1,40 +1,57 @@ """ Tests for generating compilable test files from an Echidna corpus""" from pathlib import Path +from pytest import TempPathFactory from .conftest import TestGenerator, run_generation_command_test + TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data" PATTERN = r"(\d+)\s+failing tests,\s+(\d+)\s+tests succeeded" -def test_echidna_basic_types(basic_types: TestGenerator) -> None: +def test_echidna_basic_types( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + basic_types: TestGenerator, +) -> None: """Tests the BasicTypes contract with an Echidna corpus""" run_generation_command_test( basic_types.echidna_generate_tests, "BasicTypes", "Echidna", PATTERN ) -def test_echidna_fixed_array_types(fixed_size_arrays: TestGenerator) -> None: +def test_echidna_fixed_array_types( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + fixed_size_arrays: TestGenerator, +) -> None: """Tests the FixedArrays contract with an Echidna corpus""" run_generation_command_test( fixed_size_arrays.echidna_generate_tests, "FixedArrays", "Echidna", PATTERN ) -def test_echidna_dynamic_array_types(dynamic_arrays: TestGenerator) -> None: +def test_echidna_dynamic_array_types( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + dynamic_arrays: TestGenerator, +) -> None: """Tests the DynamicArrays contract with an Echidna corpus""" run_generation_command_test( dynamic_arrays.echidna_generate_tests, "DynamicArrays", "Echidna", PATTERN ) -def test_echidna_structs_and_enums(structs_and_enums: TestGenerator) -> None: +def test_echidna_structs_and_enums( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + structs_and_enums: TestGenerator, +) -> None: """Tests the TupleTypes contract with an Echidna corpus""" run_generation_command_test( structs_and_enums.echidna_generate_tests, "TupleTypes", "Echidna", PATTERN ) -def test_echidna_value_transfer(value_transfer: TestGenerator) -> None: +def test_echidna_value_transfer( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + value_transfer: TestGenerator, +) -> None: """Tests the ValueTransfer contract with an Echidna corpus""" run_generation_command_test( value_transfer.echidna_generate_tests, "ValueTransfer", "Echidna", PATTERN diff --git a/tests/test_types_medusa.py b/tests/test_types_medusa.py index e3e9a71..9c21dc5 100644 --- a/tests/test_types_medusa.py +++ b/tests/test_types_medusa.py @@ -1,32 +1,46 @@ """ Tests for generating compilable test files from an Medusa corpus""" from pathlib import Path import pytest +from pytest import TempPathFactory + from .conftest import TestGenerator, run_generation_command_test TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data" PATTERN = r"(\d+)\s+failing tests,\s+(\d+)\s+tests succeeded" -def test_medusa_basic_types(basic_types: TestGenerator) -> None: +def test_medusa_basic_types( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + basic_types: TestGenerator, +) -> None: """Tests the BasicTypes contract with a Medusa corpus""" run_generation_command_test(basic_types.medusa_generate_tests, "BasicTypes", "Medusa", PATTERN) -def test_medusa_fixed_array_types(fixed_size_arrays: TestGenerator) -> None: +def test_medusa_fixed_array_types( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + fixed_size_arrays: TestGenerator, +) -> None: """Tests the FixedArrays contract with a Medusa corpus""" run_generation_command_test( fixed_size_arrays.medusa_generate_tests, "FixedArrays", "Medusa", PATTERN ) -def test_medusa_dynamic_array_types(dynamic_arrays: TestGenerator) -> None: +def test_medusa_dynamic_array_types( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + dynamic_arrays: TestGenerator, +) -> None: """Tests the DynamicArrays contract with a Medusa corpus""" run_generation_command_test( dynamic_arrays.medusa_generate_tests, "DynamicArrays", "Medusa", PATTERN ) -def test_medusa_structs_and_enums(structs_and_enums: TestGenerator) -> None: +def test_medusa_structs_and_enums( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + structs_and_enums: TestGenerator, +) -> None: """Tests the TupleTypes contract with a Medusa corpus""" run_generation_command_test( structs_and_enums.medusa_generate_tests, "TupleTypes", "Medusa", PATTERN @@ -34,7 +48,10 @@ def test_medusa_structs_and_enums(structs_and_enums: TestGenerator) -> None: @pytest.mark.xfail(strict=True) # type: ignore[misc] -def test_medusa_value_transfer(value_transfer: TestGenerator) -> None: +def test_medusa_value_transfer( + setup_foundry_temp_dir: TempPathFactory, # pylint: disable=unused-argument + value_transfer: TestGenerator, +) -> None: """Tests the ValueTransfer contract with a Medusa corpus""" run_generation_command_test( value_transfer.medusa_generate_tests, "ValueTransfer", "Medusa", PATTERN From 2c229ccb939abb85c39766f3ffa93862e641aa64 Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Thu, 28 Mar 2024 15:08:46 +0100 Subject: [PATCH 3/8] fix mypy errors --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b8acaa1..380709c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -105,7 +105,7 @@ def value_transfer() -> TestGenerator: @pytest.fixture(scope="session") # type: ignore[misc] -def setup_foundry_temp_dir(tmp_path_factory): +def setup_foundry_temp_dir(tmp_path_factory: Any) -> None: """Sets up a temporary directory for the tests that contain all the necessary Foundry files""" # Create a temporary directory valid for the session temp_dir = tmp_path_factory.mktemp("foundry_session") @@ -159,7 +159,7 @@ def setup_foundry_temp_dir(tmp_path_factory): os.chdir(temp_dir) -def copy_directory_contents(src_dir, dest_dir) -> None: +def copy_directory_contents(src_dir: str, dest_dir: str) -> None: """ Copies the contents of src_dir into dest_dir. The dest_dir must already exist. Directories under src_dir will be created under dest_dir and files will be copied. From aac2661b6d7fbd1cd8e27ae341ad30568169e6aa Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Thu, 28 Mar 2024 16:10:00 +0100 Subject: [PATCH 4/8] add call sequences to test string decoding --- .../reproducers/629498797734376802.txt | 1 - .../reproducers/test_specific_string.txt | 1 + .../test_results/test_specific_string.json | 25 +++++++++++++++++++ tests/test_data/src/BasicTypes.sol | 2 +- 4 files changed, 27 insertions(+), 2 deletions(-) delete mode 100644 tests/test_data/echidna-corpora/corpus-basic/reproducers/629498797734376802.txt create mode 100644 tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_string.txt create mode 100644 tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_string.json diff --git a/tests/test_data/echidna-corpora/corpus-basic/reproducers/629498797734376802.txt b/tests/test_data/echidna-corpora/corpus-basic/reproducers/629498797734376802.txt deleted file mode 100644 index f2837d3..0000000 --- a/tests/test_data/echidna-corpora/corpus-basic/reproducers/629498797734376802.txt +++ /dev/null @@ -1 +0,0 @@ -[{"call":{"contents":["check_specific_string",[{"contents":"\"\\NUL\"","tag":"AbiString"}]],"tag":"SolCall"},"delay":["0x0000000000000000000000000000000000000000000000000000000000000000","0x0000000000000000000000000000000000000000000000000000000000000000"],"dst":"0x00a329c0648769A73afAc7F9381E08FB43dBEA72","gas":12500000,"gasprice":"0x0000000000000000000000000000000000000000000000000000000000000000","src":"0x0000000000000000000000000000000000010000","value":"0x0000000000000000000000000000000000000000000000000000000000000000"}] \ No newline at end of file diff --git a/tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_string.txt b/tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_string.txt new file mode 100644 index 0000000..1d4655c --- /dev/null +++ b/tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_string.txt @@ -0,0 +1 @@ +[{"call":{"contents":["check_specific_string",[{"contents":"\"TEST_STRING\"","tag":"AbiString"}]],"tag":"SolCall"},"delay":["0x0000000000000000000000000000000000000000000000000000000000087adc","0x00000000000000000000000000000000000000000000000000000000000013bd"],"dst":"0x00a329c0648769A73afAc7F9381E08FB43dBEA72","gas":12500000,"gasprice":"0x0000000000000000000000000000000000000000000000000000000000000000","src":"0x0000000000000000000000000000000000020000","value":"0x0000000000000000000000000000000000000000000000000000000000000000"}] \ No newline at end of file diff --git a/tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_string.json b/tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_string.json new file mode 100644 index 0000000..ccf9dfb --- /dev/null +++ b/tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_string.json @@ -0,0 +1,25 @@ +[ + { + "call": { + "from": "0x0000000000000000000000000000000000020000", + "to": "0xa647ff3c36cfab592509e13860ab8c4f28781a66", + "nonce": 0, + "value": "0x0", + "gasLimit": 12500000, + "gasPrice": "0x1", + "gasFeeCap": "0x0", + "gasTipCap": "0x0", + "data": "0xe8f0d2db0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000b544553545f535452494e47000000000000000000000000000000000000000000", + "dataAbiValues": { + "methodSignature": "check_specific_string(string)", + "inputValues": [ + "TEST_STRING" + ] + }, + "AccessList": null, + "SkipAccountChecks": false + }, + "blockNumberDelay": 6, + "blockTimestampDelay": 360605 + } +] \ No newline at end of file diff --git a/tests/test_data/src/BasicTypes.sol b/tests/test_data/src/BasicTypes.sol index 0b51c3e..fa1d3b8 100644 --- a/tests/test_data/src/BasicTypes.sol +++ b/tests/test_data/src/BasicTypes.sol @@ -99,7 +99,7 @@ contract BasicTypes { function check_specific_string(string memory provided) public { require(bytes(provided).length > 0); - if (keccak256(bytes(provided)) == keccak256(bytes(hex"00"))) { + if (keccak256(bytes(provided)) == keccak256(bytes("TEST_STRING"))) { assert(false); } } From fbcd7b29547fe67f1cf4bc259ca24e2940a0e92a Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Thu, 28 Mar 2024 18:11:33 +0100 Subject: [PATCH 5/8] fix CLI argument in generate, update Echidna bytes/string decoding, add bytes test case --- fuzz_utils/generate/fuzzers/Echidna.py | 6 +- fuzz_utils/parsing/commands/generate.py | 2 +- fuzz_utils/utils/encoding.py | 125 +++++++++++------- .../reproducers/test_specific_bytes.txt | 1 + .../test_results/test_specific_bytes.json | 25 ++++ tests/test_data/src/BasicTypes.sol | 7 + 6 files changed, 112 insertions(+), 54 deletions(-) create mode 100644 tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_bytes.txt create mode 100644 tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_bytes.json diff --git a/fuzz_utils/generate/fuzzers/Echidna.py b/fuzz_utils/generate/fuzzers/Echidna.py index 05488a4..d561745 100644 --- a/fuzz_utils/generate/fuzzers/Echidna.py +++ b/fuzz_utils/generate/fuzzers/Echidna.py @@ -185,7 +185,7 @@ def _match_elementary_types(self, param: dict, recursive: bool) -> str | NoRetur contents = param["contents"][1] if is_fixed_size else param["contents"] # Haskell encoding needs to be stripped and then converted to a hex literal - hex_string = parse_echidna_byte_string(contents.strip('"')) + hex_string = parse_echidna_byte_string(contents.strip('"'), True) interpreted_string = f'hex"{hex_string}"' if not recursive: result = ( @@ -198,8 +198,8 @@ def _match_elementary_types(self, param: dict, recursive: bool) -> str | NoRetur casting = f"bytes{size}({interpreted_string})" return casting case "AbiString": - hex_string = parse_echidna_byte_string(param["contents"].strip('"')) - interpreted_string = f'string(hex"{hex_string}")' + hex_string = parse_echidna_byte_string(param["contents"].strip('"'), False) + interpreted_string = f'string(unicode"{hex_string}")' return interpreted_string case _: handle_exit( diff --git a/fuzz_utils/parsing/commands/generate.py b/fuzz_utils/parsing/commands/generate.py index 86a943f..e1ef8b8 100644 --- a/fuzz_utils/parsing/commands/generate.py +++ b/fuzz_utils/parsing/commands/generate.py @@ -91,7 +91,7 @@ def generate_command(args: Namespace) -> None: config["allSequences"] = False CryticPrint().print_information("Running Slither...") - slither = Slither(args.file_path) + slither = Slither(args.compilation_path) fuzzer: Echidna | Medusa match config["fuzzer"]: diff --git a/fuzz_utils/utils/encoding.py b/fuzz_utils/utils/encoding.py index 6b8652d..e05e808 100644 --- a/fuzz_utils/utils/encoding.py +++ b/fuzz_utils/utils/encoding.py @@ -2,63 +2,88 @@ import re ascii_escape_map = { - "\\NUL": "\x00", # Null character - "\\SOH": "\x01", - "\\STX": "\x02", - "\\ETX": "\x03", - "\\EOT": "\x04", - "\\ENQ": "\x05", - "\\ACK": "\x06", - "\\FF": "\x0c", - "\\CR": "\x0d", - "\\SO": "\x0e", - "\\SI": "\x0f", - "\\DLE": "\x10", - "\\DC1": "\x11", - "\\DC2": "\x12", - "\\DC3": "\x13", - "\\DC4": "\x14", - "\\NAK": "\x15", - "\\SYN": "\x16", - "\\ETB": "\x17", - "\\CAN": "\x18", - "\\EM": "\x19", - "\\SUB": "\x1a", - "\\ESC": "\x1b", - "\\FS": "\x1c", - "\\GS": "\x1d", - "\\RS": "\x1e", - "\\US": "\x1f", - "\\SP": "\x20", - "\\DEL": "\x7f", - "\\0": "\x00", - "\\a": "\x07", # Alert - "\\b": "\x08", # Backspace - "\\f": "\x0c", - "\\n": "\x0a", # New line - "\\r": "\x0d", - "\\t": "\x09", # Horizontal Tab - "\\v": "\x0b", # Vertical Tab + "\\NUL": b'\x00', # Null character + "\\SOH": b'\x01', + "\\STX": b'\x02', + "\\ETX": b'\x03', + "\\EOT": b'\x04', + "\\ENQ": b'\x05', + "\\ACK": b'\x06', + "\\FF": b'\x0c', + "\\CR": b'\x0d', + "\\SO": b'\x0e', + "\\SI": b'\x0f', + "\\DLE": b'\x10', + "\\DC1": b'\x11', + "\\DC2": b'\x12', + "\\DC3": b'\x13', + "\\DC4": b'\x14', + "\\NAK": b'\x15', + "\\SYN": b'\x16', + "\\ETB": b'\x17', + "\\CAN": b'\x18', + "\\EM": b'\x19', + "\\SUB": b'\x1a', + "\\ESC": b'\x1b', + "\\FS": b'\x1c', + "\\GS": b'\x1d', + "\\RS": b'\x1e', + "\\US": b'\x1f', + "\\SP": b'\x20', + "\\DEL": b'\x7f', + "\\0": b'\x00', + "\\a": b'\x07', # Alert + "\\b": b'\x08', # Backspace + "\\f": b'\x0c', + "\\n": b'\x0a', # New line + "\\r": b'\x0d', + "\\t": b'\x09', # Horizontal Tab + "\\v": b'\x0b', # Vertical Tab } -def parse_echidna_byte_string(s: str) -> str: - """Parses Haskell byte sequence into a Solidity hex literal""" +def parse_echidna_byte_string(s: str, isBytes: bool) -> str: + """Parses Haskell byte sequence into a Solidity hex literal or unicode literal""" # Replace Haskell-specific escapes with Python bytes - for key, value in ascii_escape_map.items(): - s = s.replace(key, value) + # Resultant bytes object + result_bytes = bytearray() - # Handle octal escapes (like \\135) - def octal_to_byte(match: re.Match) -> str: - octal_value = match.group(0)[1:] # Remove the backslash + # Regular expression to match decimal values like \160 + decimal_pattern = re.compile(r'\\(\d{1,3})') - return chr(int(octal_value, 8)) - - s = re.sub(r"\\[0-3]?[0-7][0-7]", octal_to_byte, s) - - # Convert to bytes and then to hexadecimal - return s.encode().hex() + # Iterator over the string + i = 0 + while i < len(s): + # Check for escape sequence + if s[i] == '\\': + matched = False + # Check for known escape sequences + for seq, byte in ascii_escape_map.items(): + if s.startswith(seq, i): + result_bytes.extend(byte) + i += len(seq) + matched = True + break + if not matched: + # Check for decimal escape + dec_match = decimal_pattern.match(s, i) + if dec_match: + # Convert the decimal escape to bytes + decimal_value = int(dec_match.group(1)) + result_bytes.append(decimal_value) + i += len(dec_match.group(0)) + else: + # Unknown escape, skip the backslash and process the next character normally + i += 1 + else: + # Normal character, encode and add + result_bytes.append(ord(s[i])) + i += 1 + # Convert the resultant bytes to hexadecimal string + if not isBytes: + return byte_to_escape_sequence(bytes(result_bytes)) + return result_bytes.hex() def byte_to_escape_sequence(byte_data: bytes) -> str: """Generates unicode escaped string from bytes""" diff --git a/tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_bytes.txt b/tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_bytes.txt new file mode 100644 index 0000000..e3fb34c --- /dev/null +++ b/tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_bytes.txt @@ -0,0 +1 @@ +[{"call":{"contents":["check_specific_bytes",[{"contents":"\"\\RS#9R\\168\\180\"","tag":"AbiBytesDynamic"}]],"tag":"SolCall"},"delay":["0x0000000000000000000000000000000000000000000000000000000000087adc","0x00000000000000000000000000000000000000000000000000000000000013bd"],"dst":"0x00a329c0648769A73afAc7F9381E08FB43dBEA72","gas":12500000,"gasprice":"0x0000000000000000000000000000000000000000000000000000000000000000","src":"0x0000000000000000000000000000000000020000","value":"0x0000000000000000000000000000000000000000000000000000000000000000"}] \ No newline at end of file diff --git a/tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_bytes.json b/tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_bytes.json new file mode 100644 index 0000000..cc97968 --- /dev/null +++ b/tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_bytes.json @@ -0,0 +1,25 @@ +[ + { + "call": { + "from": "0x0000000000000000000000000000000000020000", + "to": "0xa647ff3c36cfab592509e13860ab8c4f28781a66", + "nonce": 0, + "value": "0x0", + "gasLimit": 12500000, + "gasPrice": "0x1", + "gasFeeCap": "0x0", + "gasTipCap": "0x0", + "data": "0x0a45f4bb000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000061e233952a8b4000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "dataAbiValues": { + "methodSignature": "check_specific_bytes(bytes)", + "inputValues": [ + "1e233952a8b4" + ] + }, + "AccessList": null, + "SkipAccountChecks": false + }, + "blockNumberDelay": 6, + "blockTimestampDelay": 360605 + } +] \ No newline at end of file diff --git a/tests/test_data/src/BasicTypes.sol b/tests/test_data/src/BasicTypes.sol index fa1d3b8..cb142d7 100644 --- a/tests/test_data/src/BasicTypes.sol +++ b/tests/test_data/src/BasicTypes.sol @@ -122,6 +122,13 @@ contract BasicTypes { } } + function check_specific_bytes(bytes memory provided) public { + require(bytes(provided).length > 0); + if (keccak256(bytes(provided)) == keccak256(bytes(hex"1e233952a8b4"))) { + assert(false); + } + } + /// @notice bytes32 has decoding issues right now /* function setBytes32(bytes32 input) public { require(input != bytes32(0)); From 1d1db7b863cb0a683c2ce80402efc219ec7b1530 Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Thu, 28 Mar 2024 18:11:59 +0100 Subject: [PATCH 6/8] format --- fuzz_utils/utils/encoding.py | 79 ++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/fuzz_utils/utils/encoding.py b/fuzz_utils/utils/encoding.py index e05e808..8e6569c 100644 --- a/fuzz_utils/utils/encoding.py +++ b/fuzz_utils/utils/encoding.py @@ -2,43 +2,43 @@ import re ascii_escape_map = { - "\\NUL": b'\x00', # Null character - "\\SOH": b'\x01', - "\\STX": b'\x02', - "\\ETX": b'\x03', - "\\EOT": b'\x04', - "\\ENQ": b'\x05', - "\\ACK": b'\x06', - "\\FF": b'\x0c', - "\\CR": b'\x0d', - "\\SO": b'\x0e', - "\\SI": b'\x0f', - "\\DLE": b'\x10', - "\\DC1": b'\x11', - "\\DC2": b'\x12', - "\\DC3": b'\x13', - "\\DC4": b'\x14', - "\\NAK": b'\x15', - "\\SYN": b'\x16', - "\\ETB": b'\x17', - "\\CAN": b'\x18', - "\\EM": b'\x19', - "\\SUB": b'\x1a', - "\\ESC": b'\x1b', - "\\FS": b'\x1c', - "\\GS": b'\x1d', - "\\RS": b'\x1e', - "\\US": b'\x1f', - "\\SP": b'\x20', - "\\DEL": b'\x7f', - "\\0": b'\x00', - "\\a": b'\x07', # Alert - "\\b": b'\x08', # Backspace - "\\f": b'\x0c', - "\\n": b'\x0a', # New line - "\\r": b'\x0d', - "\\t": b'\x09', # Horizontal Tab - "\\v": b'\x0b', # Vertical Tab + "\\NUL": b"\x00", # Null character + "\\SOH": b"\x01", + "\\STX": b"\x02", + "\\ETX": b"\x03", + "\\EOT": b"\x04", + "\\ENQ": b"\x05", + "\\ACK": b"\x06", + "\\FF": b"\x0c", + "\\CR": b"\x0d", + "\\SO": b"\x0e", + "\\SI": b"\x0f", + "\\DLE": b"\x10", + "\\DC1": b"\x11", + "\\DC2": b"\x12", + "\\DC3": b"\x13", + "\\DC4": b"\x14", + "\\NAK": b"\x15", + "\\SYN": b"\x16", + "\\ETB": b"\x17", + "\\CAN": b"\x18", + "\\EM": b"\x19", + "\\SUB": b"\x1a", + "\\ESC": b"\x1b", + "\\FS": b"\x1c", + "\\GS": b"\x1d", + "\\RS": b"\x1e", + "\\US": b"\x1f", + "\\SP": b"\x20", + "\\DEL": b"\x7f", + "\\0": b"\x00", + "\\a": b"\x07", # Alert + "\\b": b"\x08", # Backspace + "\\f": b"\x0c", + "\\n": b"\x0a", # New line + "\\r": b"\x0d", + "\\t": b"\x09", # Horizontal Tab + "\\v": b"\x0b", # Vertical Tab } @@ -49,13 +49,13 @@ def parse_echidna_byte_string(s: str, isBytes: bool) -> str: result_bytes = bytearray() # Regular expression to match decimal values like \160 - decimal_pattern = re.compile(r'\\(\d{1,3})') + decimal_pattern = re.compile(r"\\(\d{1,3})") # Iterator over the string i = 0 while i < len(s): # Check for escape sequence - if s[i] == '\\': + if s[i] == "\\": matched = False # Check for known escape sequences for seq, byte in ascii_escape_map.items(): @@ -85,6 +85,7 @@ def parse_echidna_byte_string(s: str, isBytes: bool) -> str: return byte_to_escape_sequence(bytes(result_bytes)) return result_bytes.hex() + def byte_to_escape_sequence(byte_data: bytes) -> str: """Generates unicode escaped string from bytes""" arr = [] From 7d530ae4a890b83dc07b5124ae1cea4722f2c876 Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Fri, 29 Mar 2024 11:28:30 +0100 Subject: [PATCH 7/8] fix remapping detection, add tests for it --- fuzz_utils/utils/remappings.py | 2 +- tests/conftest.py | 15 +++++++---- tests/test_remapping_detection.py | 42 +++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 tests/test_remapping_detection.py diff --git a/fuzz_utils/utils/remappings.py b/fuzz_utils/utils/remappings.py index 4f3cb48..e12950a 100644 --- a/fuzz_utils/utils/remappings.py +++ b/fuzz_utils/utils/remappings.py @@ -19,7 +19,7 @@ def find_remappings(include_attacks: bool) -> dict: remappings = file.read() else: output = subprocess.run(["forge", "remappings"], capture_output=True, text=True, check=True) - remappings = str(output) + remappings = str(output.stdout) oz_matches = re.findall(openzeppelin, remappings) sol_matches = re.findall(solmate, remappings) diff --git a/tests/conftest.py b/tests/conftest.py index 380709c..dbc9064 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -118,11 +118,7 @@ def setup_foundry_temp_dir(tmp_path_factory: Any) -> None: ["forge", "install", "transmissions11/solmate", "--no-git"], check=True, cwd=temp_dir ) # Create remappings file - remappings = temp_dir / "remappings.txt" - with open(remappings, "w", encoding="utf-8") as outfile: - outfile.write( - "forge-std/=lib/forge-std/src/\nproperties/=lib/properties/contracts/\nsolmate/=lib/solmate/src/\nsrc/=src/" - ) + create_remappings_file(temp_dir) # Delete unnecessary files counter_path = temp_dir / "src" / "Counter.sol" @@ -159,6 +155,15 @@ def setup_foundry_temp_dir(tmp_path_factory: Any) -> None: os.chdir(temp_dir) +def create_remappings_file(temp_dir: Any) -> None: + """Creates a remappings file""" + remappings = os.path.join(temp_dir, "remappings.txt") + with open(remappings, "w", encoding="utf-8") as outfile: + outfile.write( + "forge-std/=lib/forge-std/src/\nproperties/=lib/properties/contracts/\nsolmate/=lib/solmate/src/\nsrc/=src/" + ) + + def copy_directory_contents(src_dir: str, dest_dir: str) -> None: """ Copies the contents of src_dir into dest_dir. The dest_dir must already exist. diff --git a/tests/test_remapping_detection.py b/tests/test_remapping_detection.py new file mode 100644 index 0000000..292ad6d --- /dev/null +++ b/tests/test_remapping_detection.py @@ -0,0 +1,42 @@ +"""Remapping detection unit tests""" +import os +from typing import Any +import pytest +from fuzz_utils.utils.remappings import find_remappings +from .conftest import create_remappings_file + + +def test_remappings_are_detected_when_no_file( + setup_foundry_temp_dir: Any, # pylint: disable=unused-argument +) -> None: + """Test if remappings are fetched when no remappings.txt file exists""" + temp_dir = os.getcwd() + + # Remove remappings file + remappings_path = os.path.join(temp_dir, "remappings.txt") + os.remove(remappings_path) + assert not os.path.exists(remappings_path) + + # Look for remappings, expecting not to fail + try: + find_remappings(True) + except SystemExit: + # Re-create remappings file and raise error + create_remappings_file(temp_dir) + pytest.fail("Finding remappings failed with SystemExit") + except Exception: # pylint: disable=broad-except + # Re-create remappings file and raise error + create_remappings_file(temp_dir) + pytest.fail("Finding remappings failed with an Exception") + + # If success + create_remappings_file(temp_dir) + + +def test_remappings_are_detected_when_file_exists( + setup_foundry_temp_dir: Any, # pylint: disable=unused-argument +) -> None: + """Test if remappings are fetched when a remappings.txt file exists""" + temp_dir = os.getcwd() + assert os.path.exists(os.path.join(temp_dir, "remappings.txt")) + find_remappings(True) From 0bad08223450b36bb9ce246783717f90c98ff61d Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Fri, 29 Mar 2024 12:01:55 +0100 Subject: [PATCH 8/8] simple testing guidance --- CONTRIBUTING.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 03d7626..6128c15 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,4 +79,20 @@ To automatically reformat the code: - `make reformat` -We use pylint `2.13.4`, black `22.3.0`. \ No newline at end of file +We use pylint `2.13.4`, black `22.3.0`. + +### Testing + +The `fuzz-utils` test suite contains basic unit tests for the available commands. All additions and modifications should be accompanied by at least one unit test. All existing tests should be run with `make test` to ensure they still pass. + +Testing is still a work-in-progress and specific guidance does not exists for all features, nor are all features currently tested. When in doubt, try creating a test yourself and request feedback on the PR. + +#### Testing Foundry unit test generation +When changes or additions are made to the `generate` command, including any changes to the template strings, corpus parsing and processing, supported Solidity types, etc., a unit test should be present. + +- When new type support is added: +1. Create or modify a contract in `tests/test_data/src` +2. Add functions that make use of the type, and a property that can be trivially caught by Echidna and Medusa. +3. Add a corpus for this contract by either running Echidna and Medusa on it, or by manually adding transaction sequences to the corpus. Ensure only failing sequences are kept and that the corpus follows the naming convention used. +4. (only required if a new contract is added) Add a fixture for this contract in `tests/conftest.py` +5. (only required if a new contract is added) Add a unit tests for this contract in `tests/test_types_echidna.py` and `tests/test_types_medusa.py` \ No newline at end of file