SymQEMU tests and Dockefile (#41)

This commit adds:
* testing script and simple test binary
* SymCC as submodule, SymQEMU Dockerfile relies on SymCC Dockerfile
* Dockerfile for building and running e2e tests
* github action that runs on PR

---------

Co-authored-by: Aurelien Francillon <[email protected]>
Co-authored-by: aurelf <[email protected]>

.dockerignore

@@ -0,0 +1 @@
+build

.github/workflows/main.yml

@@ -0,0 +1,13 @@
+on:
+  workflow_dispatch:
+  push:
+  pull_request:
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - run: git submodule update --init --recursive symcc
+    - run: docker build -t symcc symcc
+    - run: docker build -t symqemu .

.gitmodules

@@ -43,3 +43,6 @@
 [submodule "tests/lcitool/libvirt-ci"]
 	path = tests/lcitool/libvirt-ci
 	url = https://gitlab.com/libvirt/libvirt-ci.git
+[submodule "symcc"]
+	path = symcc
+	url = https://github.com/eurecom-s3/symcc.git

Dockerfile

@@ -0,0 +1,39 @@
+FROM ubuntu:22.04
+
+RUN apt update
+RUN apt install -y \
+    ninja-build \
+    libglib2.0-dev \
+    llvm \
+    git \
+    python3 \
+    python3-pip
+
+COPY . /symqemu_source
+WORKDIR /symqemu_source
+
+# Meson gives an error if symcc is in a subdirectory of symqemu
+RUN mv /symqemu_source/symcc /symcc
+
+# The only symcc artifact needed by symqemu is libSymRuntime.so
+# Instead of compiling symcc in this image, we rely on the existing symcc docker image and
+# we just copy libSymRuntime.so at the location where symqemu expects it
+COPY --from=symcc /symcc_build/SymRuntime-prefix/src/SymRuntime-build/libSymRuntime.so /symcc/build/SymRuntime-prefix/src/SymRuntime-build/libSymRuntime.so
+
+RUN ./configure                                                       \
+          --audio-drv-list=                                           \
+          --disable-sdl                                               \
+          --disable-gtk                                               \
+          --disable-vte                                               \
+          --disable-opengl                                            \
+          --disable-virglrenderer                                     \
+          --disable-werror                                            \
+          --target-list=x86_64-linux-user                             \
+          --enable-debug                                              \
+          --symcc-source=/symcc                                       \
+          --symcc-build=/symcc/build
+
+RUN make -j
+
+WORKDIR /symqemu_source/tests/symqemu
+RUN python3 -m unittest test.py

README.md

@@ -89,6 +89,27 @@ prefix the target command with `x86_64-linux-user/symqemu-x86_64`. (Note that
 you'll have to run AFL in QEMU mode by adding `-Q` to its command line; the
 fuzzing helper will automatically pick up the setting and use QEMU mode too.)
 
+## Build with Docker
+First, make sure to have an up-to-date Docker image of SymCC. If you
+don't have one, either in the submodule, run:
+
+``` shell
+git submodule update --init --recursive symcc
+cd symcc
+docker build -t symcc .
+cd ..
+```
+
+Then build the SymQEMU image with (this will also run the tests):
+```shell
+docker build -t symqemu  .
+```
+
+You can use the docker with:
+```shell
+docker run -it --rm symqemu
+```
+
 ## Documentation
 
 The [paper](http://www.s3.eurecom.fr/tools/symbolic_execution/symqemu.html)

tests/symqemu/.gitignore

@@ -0,0 +1,4 @@
+.idea
+__pycache__
+_trial_temp
+.gdb_history

tests/symqemu/README.md

@@ -0,0 +1,35 @@
+# SymQEMU tests
+
+The purpose of those tests is to automatically run SymQEMU on some known
+binaries and check that it gives the expected outputs (i.e. check that it
+generates the expected test cases).
+
+To run the tests, cd into this directory and run:
+
+```
+python3 -m unittest test.py
+```
+
+The directory `binaries` contain a directory for each test binary. A test
+binary directory contains the following:
+
+- An executable file `binary`
+- A file `input` whose path will be given as an argument to `binary` and whose
+  content will be symbolic
+- A text file `args` that contains the arguments `binary` will be called
+  with. One of the arguments must be `@@` and it will be replaced with the path
+  of the `input` file.
+- A directory `expected_outputs`: the test cases that SymQEMU should generate
+  when called like this: `<symqemu> <path/to/binary> <args>`, with the content
+  of `input` being symbolic.
+
+## Adding a test
+
+- Create a new directory in `binaries` and put the files `binary` and `input`
+  inside it
+- Run `python3 init-new <name of your new binary directory>`. This will run
+  SymQEMU on your new binary, create a new directory `binaries/<name of your new
+  directory>/expected_outputs` and store the test cases generated by SymQEMU in
+  it.
+- Edit `test.py` to add your binary
+

tests/symqemu/binaries/simple/Makefile

@@ -0,0 +1,4 @@
+binary:
+
+clean:
+	rm binary

tests/symqemu/binaries/simple/args

@@ -0,0 +1 @@
+@@

tests/symqemu/binaries/simple/binary.c

@@ -0,0 +1,33 @@
+#include <stdio.h>
+
+int main(int argc, char *argv[]) {
+
+    if (argc != 2) {
+        puts("ERROR: You need one argument.");
+        return 1;
+    }
+
+    FILE* file_stream = fopen(argv[1], "r");
+
+    if (!file_stream) {
+        puts("ERROR: Could not open file.");
+        return 1;
+    }
+
+    char input1 = getc(file_stream);
+    char input2 = getc(file_stream);
+    char input3 = getc(file_stream);
+
+    if (input1 == 'a') {
+        puts("foo1");
+    }
+
+    if (input2 == 'b') {
+        puts("foo2");
+    }
+
+    if (input3 == 'c') {
+        puts("foo3");
+    }
+
+}

tests/symqemu/binaries/simple/expected_outputs/000000

@@ -0,0 +1 @@
+azzzzzzzz

tests/symqemu/binaries/simple/expected_outputs/000001

@@ -0,0 +1 @@
+zbzzzzzzz

tests/symqemu/binaries/simple/expected_outputs/000002

@@ -0,0 +1 @@
+zzczzzzzz

tests/symqemu/binaries/simple/input

@@ -0,0 +1 @@
+zzzzzzzzz

tests/symqemu/init-new.py

@@ -0,0 +1,28 @@
+import sys
+
+import util
+
+if __name__ == '__main__':
+
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <binary name>")
+        sys.exit(1)
+
+    binary_name = sys.argv[1]
+    binary_dir = util.BINARIES_DIR / binary_name
+
+    if not binary_dir.exists():
+        print(f"Error: {binary_dir} does not exist")
+        sys.exit(1)
+
+    output_dir = binary_dir / 'expected_outputs'
+
+    if output_dir.exists():
+        print(f"Error: {output_dir} already exists")
+        sys.exit(1)
+
+    output_dir.mkdir()
+
+    util.run_symqemu_on_test_binary(binary_name=binary_name, generated_test_cases_output_dir=output_dir)
+
+    print(f"Expected outputs for {binary_name} generated in {output_dir}")

tests/symqemu/test.py

@@ -0,0 +1,32 @@
+import filecmp
+import pathlib
+import shutil
+import unittest
+
+import util
+
+
+class SymQemuTests(unittest.TestCase):
+    SYMQEMU_OUTPUT_DIR = pathlib.Path(__file__).parent / "symqemu_output"
+
+    def setUp(self):
+        self.SYMQEMU_OUTPUT_DIR.mkdir()
+
+    def tearDown(self):
+        shutil.rmtree(self.SYMQEMU_OUTPUT_DIR)
+
+    def run_symqemu_and_assert_correct_result(self, binary_name):
+
+        util.run_symqemu_on_test_binary(binary_name=binary_name, generated_test_cases_output_dir=self.SYMQEMU_OUTPUT_DIR)
+
+        # `filecmp.dircmp` does a "shallow" comparison, but this is not a problem here because
+        # the timestamps should always be different, so the actual content of the files will be compared.
+        # See https://docs.python.org/3/library/filecmp.html#filecmp.dircmp
+        expected_vs_actual_output_comparison = filecmp.dircmp(self.SYMQEMU_OUTPUT_DIR, util.BINARIES_DIR / binary_name / 'expected_outputs')
+        self.assertEqual(expected_vs_actual_output_comparison.diff_files, [])
+        self.assertEqual(expected_vs_actual_output_comparison.left_only, [])
+        self.assertEqual(expected_vs_actual_output_comparison.right_only, [])
+        self.assertEqual(expected_vs_actual_output_comparison.funny_files, [])
+
+    def test_simple(self):
+        self.run_symqemu_and_assert_correct_result('simple')

tests/symqemu/util.py

@@ -0,0 +1,60 @@
+import pathlib
+import subprocess
+
+SYMQEMU_EXECUTABLE = pathlib.Path(__file__).parent.parent.parent / "build" / "x86_64-linux-user" / "qemu-x86_64"
+BINARIES_DIR = pathlib.Path(__file__).parent / "binaries"
+
+
+class SymqemuRunFailed(Exception):
+    pass
+
+
+def run_symqemu(
+        binary: pathlib.Path,
+        binary_arguments: list[str],
+        generated_test_cases_output_dir: pathlib.Path,
+        symbolized_input_file: pathlib.Path,
+) -> None:
+    command = str(SYMQEMU_EXECUTABLE), str(binary), *binary_arguments
+
+    environment_variables = {
+        'SYMCC_OUTPUT_DIR': str(generated_test_cases_output_dir),
+        'SYMCC_INPUT_FILE': str(symbolized_input_file)
+    }
+
+    print(f'about to run command: {" ".join(command)}')
+    print(f'with environment variables: {environment_variables}')
+
+    try:
+        subprocess.run(
+            command,
+            env=environment_variables,
+            capture_output=True,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        raise SymqemuRunFailed(
+            f'command {e.cmd} failed with exit code {e.returncode} and output: {e.stderr.decode()}'
+        )
+
+
+def run_symqemu_on_test_binary(
+        binary_name: str,
+        generated_test_cases_output_dir: pathlib.Path
+) -> None:
+    binary_dir = BINARIES_DIR / binary_name
+
+    with open(binary_dir / 'args', 'r') as f:
+        binary_args = f.read().strip().split(' ')
+
+    def replace_placeholder_with_input(arg: str) -> str:
+        return str(binary_dir / 'input') if arg == '@@' else arg
+
+    binary_args = list(map(replace_placeholder_with_input, binary_args))
+
+    run_symqemu(
+        binary=binary_dir / 'binary',
+        binary_arguments=binary_args,
+        generated_test_cases_output_dir=generated_test_cases_output_dir,
+        symbolized_input_file=binary_dir / 'input',
+    )