Merge pull request #34 from crytic/refactor-tests

Refactor tests

CONTRIBUTING.md

@@ -79,4 +79,20 @@ To automatically reformat the code:
 
 - `make reformat`
 
-We use pylint `2.13.4`, black `22.3.0`.
+We use pylint `2.13.4`, black `22.3.0`.
+
+### Testing
+
+The `fuzz-utils` test suite contains basic unit tests for the available commands. All additions and modifications should be accompanied by at least one unit test. All existing tests should be run with `make test` to ensure they still pass.
+
+Testing is still a work-in-progress and specific guidance does not exists for all features, nor are all features currently tested. When in doubt, try creating a test yourself and request feedback on the PR.
+
+#### Testing Foundry unit test generation
+When changes or additions are made to the `generate` command, including any changes to the template strings, corpus parsing and processing, supported Solidity types, etc., a unit test should be present.
+
+- When new type support is added:
+1. Create or modify a contract in `tests/test_data/src`
+2. Add functions that make use of the type, and a property that can be trivially caught by Echidna and Medusa.
+3. Add a corpus for this contract by either running Echidna and Medusa on it, or by manually adding transaction sequences to the corpus. Ensure only failing sequences are kept and that the corpus follows the naming convention used.
+4. (only required if a new contract is added) Add a fixture for this contract in `tests/conftest.py`
+5. (only required if a new contract is added) Add a unit tests for this contract in `tests/test_types_echidna.py` and `tests/test_types_medusa.py`

Makefile

@@ -68,9 +68,6 @@ reformat:
 test tests: $(VENV)/pyvenv.cfg
 	. $(VENV_BIN)/activate && \
 		solc-select use 0.8.19 --always-install && \
-		cd tests/test_data && \
-		forge install && \
-		cd ../.. && \
 		pytest --ignore tests/test_data/lib $(T) $(TEST_ARGS)
 
 .PHONY: package

fuzz_utils/generate/fuzzers/Echidna.py

@@ -185,7 +185,7 @@ def _match_elementary_types(self, param: dict, recursive: bool) -> str | NoRetur
                 contents = param["contents"][1] if is_fixed_size else param["contents"]
 
                 # Haskell encoding needs to be stripped and then converted to a hex literal
-                hex_string = parse_echidna_byte_string(contents.strip('"'))
+                hex_string = parse_echidna_byte_string(contents.strip('"'), True)
                 interpreted_string = f'hex"{hex_string}"'
                 if not recursive:
                     result = (
@@ -198,8 +198,8 @@ def _match_elementary_types(self, param: dict, recursive: bool) -> str | NoRetur
                 casting = f"bytes{size}({interpreted_string})"
                 return casting
             case "AbiString":
-                hex_string = parse_echidna_byte_string(param["contents"].strip('"'))
-                interpreted_string = f'string(hex"{hex_string}")'
+                hex_string = parse_echidna_byte_string(param["contents"].strip('"'), False)
+                interpreted_string = f'string(unicode"{hex_string}")'
                 return interpreted_string
             case _:
                 handle_exit(

fuzz_utils/parsing/commands/generate.py

@@ -91,7 +91,7 @@ def generate_command(args: Namespace) -> None:
             config["allSequences"] = False
 
     CryticPrint().print_information("Running Slither...")
-    slither = Slither(args.file_path)
+    slither = Slither(args.compilation_path)
     fuzzer: Echidna | Medusa
 
     match config["fuzzer"]:

fuzz_utils/utils/encoding.py

@@ -2,62 +2,88 @@
 import re
 
 ascii_escape_map = {
-    "\\NUL": "\x00",  # Null character
-    "\\SOH": "\x01",
-    "\\STX": "\x02",
-    "\\ETX": "\x03",
-    "\\EOT": "\x04",
-    "\\ENQ": "\x05",
-    "\\ACK": "\x06",
-    "\\FF": "\x0c",
-    "\\CR": "\x0d",
-    "\\SO": "\x0e",
-    "\\SI": "\x0f",
-    "\\DLE": "\x10",
-    "\\DC1": "\x11",
-    "\\DC2": "\x12",
-    "\\DC3": "\x13",
-    "\\DC4": "\x14",
-    "\\NAK": "\x15",
-    "\\SYN": "\x16",
-    "\\ETB": "\x17",
-    "\\CAN": "\x18",
-    "\\EM": "\x19",
-    "\\SUB": "\x1a",
-    "\\ESC": "\x1b",
-    "\\FS": "\x1c",
-    "\\GS": "\x1d",
-    "\\RS": "\x1e",
-    "\\US": "\x1f",
-    "\\SP": "\x20",
-    "\\DEL": "\x7f",
-    "\\0": "\x00",
-    "\\a": "\x07",  # Alert
-    "\\b": "\x08",  # Backspace
-    "\\f": "\x0c",
-    "\\n": "\x0a",  # New line
-    "\\r": "\x0d",
-    "\\t": "\x09",  # Horizontal Tab
-    "\\v": "\x0b",  # Vertical Tab
+    "\\NUL": b"\x00",  # Null character
+    "\\SOH": b"\x01",
+    "\\STX": b"\x02",
+    "\\ETX": b"\x03",
+    "\\EOT": b"\x04",
+    "\\ENQ": b"\x05",
+    "\\ACK": b"\x06",
+    "\\FF": b"\x0c",
+    "\\CR": b"\x0d",
+    "\\SO": b"\x0e",
+    "\\SI": b"\x0f",
+    "\\DLE": b"\x10",
+    "\\DC1": b"\x11",
+    "\\DC2": b"\x12",
+    "\\DC3": b"\x13",
+    "\\DC4": b"\x14",
+    "\\NAK": b"\x15",
+    "\\SYN": b"\x16",
+    "\\ETB": b"\x17",
+    "\\CAN": b"\x18",
+    "\\EM": b"\x19",
+    "\\SUB": b"\x1a",
+    "\\ESC": b"\x1b",
+    "\\FS": b"\x1c",
+    "\\GS": b"\x1d",
+    "\\RS": b"\x1e",
+    "\\US": b"\x1f",
+    "\\SP": b"\x20",
+    "\\DEL": b"\x7f",
+    "\\0": b"\x00",
+    "\\a": b"\x07",  # Alert
+    "\\b": b"\x08",  # Backspace
+    "\\f": b"\x0c",
+    "\\n": b"\x0a",  # New line
+    "\\r": b"\x0d",
+    "\\t": b"\x09",  # Horizontal Tab
+    "\\v": b"\x0b",  # Vertical Tab
 }
 
 
-def parse_echidna_byte_string(s: str) -> str:
-    """Parses Haskell byte sequence into a Solidity hex literal"""
+def parse_echidna_byte_string(s: str, isBytes: bool) -> str:
+    """Parses Haskell byte sequence into a Solidity hex literal or unicode literal"""
     # Replace Haskell-specific escapes with Python bytes
-    for key, value in ascii_escape_map.items():
-        s = s.replace(key, value)
+    # Resultant bytes object
+    result_bytes = bytearray()
 
-    # Handle octal escapes (like \\135)
-    def octal_to_byte(match: re.Match) -> str:
-        octal_value = match.group(0)[1:]  # Remove the backslash
+    # Regular expression to match decimal values like \160
+    decimal_pattern = re.compile(r"\\(\d{1,3})")
 
-        return chr(int(octal_value, 8))
+    # Iterator over the string
+    i = 0
+    while i < len(s):
+        # Check for escape sequence
+        if s[i] == "\\":
+            matched = False
+            # Check for known escape sequences
+            for seq, byte in ascii_escape_map.items():
+                if s.startswith(seq, i):
+                    result_bytes.extend(byte)
+                    i += len(seq)
+                    matched = True
+                    break
+            if not matched:
+                # Check for decimal escape
+                dec_match = decimal_pattern.match(s, i)
+                if dec_match:
+                    # Convert the decimal escape to bytes
+                    decimal_value = int(dec_match.group(1))
+                    result_bytes.append(decimal_value)
+                    i += len(dec_match.group(0))
+                else:
+                    # Unknown escape, skip the backslash and process the next character normally
+                    i += 1
+        else:
+            # Normal character, encode and add
+            result_bytes.append(ord(s[i]))
+            i += 1
 
-    s = re.sub(r"\\[0-3]?[0-7][0-7]", octal_to_byte, s)
-
-    # Convert to bytes and then to hexadecimal
-    return s.encode().hex()
+    # Convert the resultant bytes to hexadecimal string
+    if not isBytes:
+        return byte_to_escape_sequence(bytes(result_bytes))
+    return result_bytes.hex()
 
 
 def byte_to_escape_sequence(byte_data: bytes) -> str:

fuzz_utils/utils/remappings.py

@@ -19,7 +19,7 @@ def find_remappings(include_attacks: bool) -> dict:
             remappings = file.read()
     else:
         output = subprocess.run(["forge", "remappings"], capture_output=True, text=True, check=True)
-        remappings = str(output)
+        remappings = str(output.stdout)
 
     oz_matches = re.findall(openzeppelin, remappings)
     sol_matches = re.findall(solmate, remappings)

tests/conftest.py

@@ -1,6 +1,9 @@
 """ Globally available fixtures"""
 import os
-from typing import Any
+import subprocess
+import re
+import shutil
+from typing import Any, Callable
 import pytest
 
 from slither import Slither
@@ -38,7 +41,7 @@ def medusa_generate_tests(self) -> None:
         self.medusa_generator.create_poc()
 
 
-@pytest.fixture(autouse=True)  # type: ignore[misc]
+@pytest.fixture()  # type: ignore[misc]
 def change_test_dir(request: Any, monkeypatch: Any) -> None:
     """Helper fixture to change the working directory"""
     # Directory of the test file
@@ -99,3 +102,110 @@ def value_transfer() -> TestGenerator:
     corpus_dir = "corpus-value"
 
     return TestGenerator(target, target_path, corpus_dir)
+
+
+@pytest.fixture(scope="session")  # type: ignore[misc]
+def setup_foundry_temp_dir(tmp_path_factory: Any) -> None:
+    """Sets up a temporary directory for the tests that contain all the necessary Foundry files"""
+    # Create a temporary directory valid for the session
+    temp_dir = tmp_path_factory.mktemp("foundry_session")
+    original_dir: str = os.getcwd()
+
+    print("Installing Forge...")
+    subprocess.run(["forge", "init", "--no-git"], check=True, cwd=temp_dir)
+    subprocess.run(["forge", "install", "crytic/properties", "--no-git"], check=True, cwd=temp_dir)
+    subprocess.run(
+        ["forge", "install", "transmissions11/solmate", "--no-git"], check=True, cwd=temp_dir
+    )
+    # Create remappings file
+    create_remappings_file(temp_dir)
+
+    # Delete unnecessary files
+    counter_path = temp_dir / "src" / "Counter.sol"
+    counter_path.unlink()
+    assert not counter_path.exists()
+
+    counter_test_path = temp_dir / "test" / "Counter.t.sol"
+    counter_test_path.unlink()
+    assert not counter_test_path.exists()
+
+    scripts_dir = temp_dir / "script"
+    shutil.rmtree(scripts_dir)
+    assert not scripts_dir.exists()
+
+    # Create the corpora directories in the temporary dir
+    echidna_corpora_dir = temp_dir / "echidna-corpora"
+    medusa_corpora_dir = temp_dir / "medusa-corpora"
+    echidna_corpora_dir.mkdir(exist_ok=True)
+    medusa_corpora_dir.mkdir(exist_ok=True)
+
+    # Copy all our contracts and corpora to the temporary directory
+    copy_directory_contents(
+        os.path.join(original_dir, "tests", "test_data", "echidna-corpora"),
+        temp_dir / "echidna-corpora",
+    )
+    copy_directory_contents(
+        os.path.join(original_dir, "tests", "test_data", "medusa-corpora"),
+        temp_dir / "medusa-corpora",
+    )
+    copy_directory_contents(
+        os.path.join(original_dir, "tests", "test_data", "src"), temp_dir / "src"
+    )
+
+    os.chdir(temp_dir)
+
+
+def create_remappings_file(temp_dir: Any) -> None:
+    """Creates a remappings file"""
+    remappings = os.path.join(temp_dir, "remappings.txt")
+    with open(remappings, "w", encoding="utf-8") as outfile:
+        outfile.write(
+            "forge-std/=lib/forge-std/src/\nproperties/=lib/properties/contracts/\nsolmate/=lib/solmate/src/\nsrc/=src/"
+        )
+
+
+def copy_directory_contents(src_dir: str, dest_dir: str) -> None:
+    """
+    Copies the contents of src_dir into dest_dir. The dest_dir must already exist.
+    Directories under src_dir will be created under dest_dir and files will be copied.
+    """
+    for item in os.listdir(src_dir):
+        s = os.path.join(src_dir, item)
+        d = os.path.join(dest_dir, item)
+        if os.path.isdir(s):
+            shutil.copytree(s, d, dirs_exist_ok=True)  # For Python 3.8+, use dirs_exist_ok=True
+        else:
+            shutil.copy2(s, d)
+
+
+def run_generation_command_test(
+    generate_tests: Callable, test_name: str, fuzzer: str, pattern: str
+) -> None:
+    """Utility function to test unit test generation from a corpus and contract"""
+    generate_tests()
+    # Ensure the file was created
+    path = os.path.join(os.getcwd(), "test", f"{test_name}_{fuzzer}_Test.t.sol")
+    assert os.path.exists(path)
+
+    # Ensure the file can be compiled
+    subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True)
+
+    # Ensure the file can be tested
+    result = subprocess.run(
+        ["forge", "test", "--match-contract", f"{test_name}_{fuzzer}_Test"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    # Remove ansi escape sequences
+    ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+    output = ansi_escape.sub("", result.stdout)
+
+    # Ensure all tests fail
+    match = re.search(pattern, output)
+    if match:
+        tests_passed = int(match.group(2))
+        assert tests_passed == 0
+    else:
+        assert False, "No tests were ran"

tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_bytes.txt

@@ -0,0 +1 @@
+[{"call":{"contents":["check_specific_bytes",[{"contents":"\"\\RS#9R\\168\\180\"","tag":"AbiBytesDynamic"}]],"tag":"SolCall"},"delay":["0x0000000000000000000000000000000000000000000000000000000000087adc","0x00000000000000000000000000000000000000000000000000000000000013bd"],"dst":"0x00a329c0648769A73afAc7F9381E08FB43dBEA72","gas":12500000,"gasprice":"0x0000000000000000000000000000000000000000000000000000000000000000","src":"0x0000000000000000000000000000000000020000","value":"0x0000000000000000000000000000000000000000000000000000000000000000"}]

tests/test_data/echidna-corpora/corpus-basic/reproducers/test_specific_string.txt

@@ -0,0 +1 @@
+[{"call":{"contents":["check_specific_string",[{"contents":"\"TEST_STRING\"","tag":"AbiString"}]],"tag":"SolCall"},"delay":["0x0000000000000000000000000000000000000000000000000000000000087adc","0x00000000000000000000000000000000000000000000000000000000000013bd"],"dst":"0x00a329c0648769A73afAc7F9381E08FB43dBEA72","gas":12500000,"gasprice":"0x0000000000000000000000000000000000000000000000000000000000000000","src":"0x0000000000000000000000000000000000020000","value":"0x0000000000000000000000000000000000000000000000000000000000000000"}]

tests/test_data/medusa-corpora/corpus-basic/test_results/test_specific_bytes.json

@@ -0,0 +1,25 @@
+[
+    {
+        "call": {
+            "from": "0x0000000000000000000000000000000000020000",
+            "to": "0xa647ff3c36cfab592509e13860ab8c4f28781a66",
+            "nonce": 0,
+            "value": "0x0",
+            "gasLimit": 12500000,
+            "gasPrice": "0x1",
+            "gasFeeCap": "0x0",
+            "gasTipCap": "0x0",
+            "data": "0x0a45f4bb000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000061e233952a8b4000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+            "dataAbiValues": {
+                "methodSignature": "check_specific_bytes(bytes)",
+                "inputValues": [
+                    "1e233952a8b4"
+                ]
+            },
+            "AccessList": null,
+            "SkipAccountChecks": false
+        },
+        "blockNumberDelay": 6,
+        "blockTimestampDelay": 360605
+    }
+]