diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 0000000..b1dbdbc --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,21 @@ +name: doc +on: + push: + branches: + - main + +jobs: + deploy-doc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install Dependencies + run: | + pip install '.[doc]' + - name: Deploy doc + run: mkdocs gh-deploy --force diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ff9d7d..3ae0b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,67 +1,75 @@ -# v0.4.1 +## v0.4.1 This version enforces the usage of a more efficient version of numbat and fixes some little bugs. -## Features +### Features All: + - enforce numbat >= 0.2 to increase analysis speed -## Fixes +### Fixes File system parser: + - check db existence with the appropriate numbat method - remove error-prone path modifications in symlink resolution -**Full Changelog**: https://github.com/quarkslab/pyrrha/compare/v0.4.0...v0.4.1 +**Full Changelog**: [https://github.com/quarkslab/pyrrha/compare/v0.4.0...v0.4.1](https://github.com/quarkslab/pyrrha/compare/v0.4.0...v0.4.1) -# v0.4.0—Numbat version +## v0.4.0—Numbat version This version introduces the usage of Numbat, our home-made Sourcetrail SDK fully Pythonic. Thanks to it, `pyrrha` is much easier to install. -## Features +### Features All: + * Remove SourcetrailDB dependency to use `numbat` library File system parser: + * multiprocess binary parsing (lief export) Docker/CI: + * Adapt to numbat dependency. Remove all the useless installations. * Add package publication on pypi. -## :warning: Important Changes +### :warning: Important Changes - Package name was changes into `pyrrha-mapper` as the `pyrrha` package already exists on Pypi. -## Fixes +### Fixes - Symlink resolution was partially broken due to not extensive checks on the path. It was trying to parse directory for example. -## Associated Python package +### Associated Python package This release contains a CI that automatically upload the package on Pypi. You can now install Pyrrha by doing ```python pip install pyrrha-mapper ``` -**Full Changelog**: https://github.com/quarkslab/pyrrha/compare/v0.3.0...v0.4.0 +**Full Changelog**: [https://github.com/quarkslab/pyrrha/compare/v0.3.0...v0.4.0](https://github.com/quarkslab/pyrrha/compare/v0.3.0...v0.4.0) -# v0.3.0—Hack.lu edition +## v0.3.0—Hack.lu edition Version release at the occasion of the talk [Pyrrha: navigate easily into your system binaries](https://pretalx.com/hack-lu-2023/talk/WVFPNK/) given at the CTI-summit of Hack.lu. **Full Changelog**: https://github.com/quarkslab/pyrrha/compare/v0.2.0...v0.3.0 -## Features +### Features File system parser: + * change JSON export structure Documentation: + * add example of diffing using JSON export * extend README to include new features Docker/CI: + * Change base Docker image to a lighter one (`python` to `python-slim`) * Add automatic build and upload of Docker image on Quarkslab's Github registry -## Fixes +### Fixes None -## Associated Docker Image +### Associated Docker Image Install from command line: ``` commandline docker pull ghcr.io/quarkslab/pyrrha:v0.3.0 @@ -71,26 +79,29 @@ Use as base image in Dockerfile: FROM ghcr.io/quarkslab/pyrrha:v0.3.0 ``` -# v0.2.0 +## v0.2.0 For more details, check [associated package page](https://github.com/quarkslab/pyrrha/pkgs/container/pyrrha/138112209?tag=v0.3.0). -## Features +### Features CLI: + * setup logging and add debug option * add `-h` option to show the usage (equivalent of `--help`) -* + File system parser: + * add PE support (:warning: it is case sensitive for *all* imports (functions and libraries) * add progress bar to show in real time percentage of wiles which have been indexed * unresolved imports (lib and/or symbols) point now on non-indexed symbols to keep information in the database * the mapping done by Pyrrha can be exported as a JSON file Doc: + * add options to have real time Docker output in the terminal (for logs and progress bars) -## Fixes +### Fixes * Dockerfile was copying non existing directory, this action has been removed. -# v0.1 +## v0.1 First public release of Pyrrha \ No newline at end of file diff --git a/README.md b/README.md index 6534df0..b974028 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,15 @@ -# Pyrrha - -* [Introduction](#introduction) -* [Installation](#installation) -* [Usage](#usage) -* [Docker](#docker) -* [Usage Example](#quick-start----usage-example) -* [Authors](#authors) - -## Introduction +# Pyrrha: A mapper collection for firmware analysis Pyrrha is a filesystem cartography and correlation software focusing on visualization. It currently focuses on the relationship between executable files but aims at enabling anyone to map and visualize any relationship types. It uses the open-source code source explorer [Sourcetrail](https://github.com/CoatiSoftware/Sourcetrail) to provide users with an easy way to navigate through and search for path to function. -![](docs/img/imports.png) +![](img/imports.png)

An example of the symbols and libraries imported by libgcc_s.so.1 and of the symbols which reference this library.

-![](docs/img/symlinks.png) +![](img/symlinks.png)

An example of the symlinks which point on busybox.

@@ -26,102 +17,44 @@ path to function. ## Installation The installation is done in two parts: -- installing `Pyrrha` (as a Python module); -- installing `Sourcetrail` to be able to visualize Pyrrha's results. - -### Sourcetrail installation -Sourcetrail can be installed using [its last release](https://github.com/CoatiSoftware/Sourcetrail/releases/tag/2021.4.19) and its [documentation](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#installation). - -### Pyrrha installation -Pyrrha requires a Python version >= 3.10. -It is recommended to install the Python package inside a virtualenv. You can use `pip` to install it. -```python -pip install pyrrha-mapper -``` -If you prefer using sources to install Pyrrha, do the following: -```commandline -# Do not forget to activate your virtualenv -$ pip install 'pyrrha @ git+https://github.com/quarkslab/pyrrha' - -# If you prefer, you can manually clone the repository and then install the package -$ git clone https://github.com/quarkslab/pyrrha -$ cd pyrrha -$ pip install '.' -``` - -*Tested for Linux and Windows.* For Windows systems, installation should be run from MSVC Developer shell to build the wheel. -## Docker -`pyrrha` can be used with a docker. It provides Pyrrha but you still need to install Sourcetrail on your system as described in the [Sourcetrail Installation](#sourcetrail-installation) section. - -```commandline -$ cd ROOT_DIRECTORY/.. -$ docker run --rm -t -v $PWD:/tmp/pyrrha ghcr.io/quarkslab/pyrrha:latest fs [OPTIONS] ROOT_DIRECTORY -``` - -A docker image is directly available from our [Github registry](https://github.com/orgs/quarkslab/packages/container/package/pyrrha), but you can also build it from the sources. - -```commandline -$ git clone PYRRHA_URL && cd pyrrha -$ docker build -t pyrrha . -``` +- Installing `Pyrrha` as a Python module (`pip install pyrrha-mapper`) or using its Docker image. +- Installing `Sourcetrail` to be able to visualize Pyrrha's results. You can use [its last release](https://github.com/CoatiSoftware/Sourcetrail/releases/tag/2021.4.19) and its [documentation](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#installation). ## Usage -### Mapping with Pyrrha -First, create your db with `pyrrha`. The `ROOT_DIRECTORY` should contain the whole filesystem you want to map, it should be already extracted or mounted. `ROOT_DIRECTORY` will be considered by Pyrrha as the filesystem root for all the symlink resolutions. - -```commandline -Usage: pyrrha fs [OPTIONS] ROOT_DIRECTORY - - Map a filesystem into a sourcetrail-compatible db. - -Options: - --db PATH Sourcetrail DB file path (.srctrldb). [default: pyrrha.srctrldb] - -e, --json Create a JSON export of the resulting mapping. - -h, --help Show this message and exit. -``` - -You can also export your Pyrrha results as a JSON file (option `-j`) to be able to postprocess them. For example, you can diff the results between two versions of the same system and list the binaries added/removed and which symbols has been added/removed (*cf* example script in `example`). - -## Visualization with Sourcetrail -Open the resulting project with `sourcetrail`. You can now navigate on the resulting cartography. The user interface is described in depth in the [Sourcetrail documentation](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#user-interface). - -To match the Sourcetrail language, the binaries, the exported functions and symbols, and the symlinks are represented as follows in Sourcetrail. - -Binaries | Exported functions | Exported symbols | Symlinks -:---:|:----------------------------:|:--------------------------:| :---: -![](docs/img/classes.png) | ![](docs/img/function.png) | ![](docs/img/variable.png) | ![](docs/img/typedefs.png) - -Do not hesitate to take a look at [Sourcetrail documentation](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#graph-view-1) to explore all the possibilities offered by Sourcetrail. [Custom Trails](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#custom-trail-dialog) could be really useful in a lot of cases. - -## Quick Start—Usage Example -Let's take the example of an OpenWRT firmware which is a common Linux distribution for embedded targets like routers. - -First, download the firmware and extract its root-fs into a directory. Here we download the last OpenWRT version for generic x86_64 systems. -```commandline -$ wget https://downloads.openwrt.org/releases/22.03.5/targets/x86/64/openwrt-22.03.5-x86-64-rootfs.tar.gz -O openwrt_rootfs.tar.gz -$ mkdir openwrt_root_fs && cd openwrt_root_fs -$ tar -xf ../openwrt_rootfs.tar.gz -$ cd .. && rm openwrt_rootfs.tar.gz -``` - -Then we can run Pyrrha on it. It will produce some logs indicating which symlinks or imports cannot be solved directly by the tool. -*(Do not forget to activate your virtualenv if you have created one for Pyrrha installation.)* -```commandline -$ pyrrha fs --db openwrt_db openwrt_root_fs -$ ls -openwrt_root_fs openwrt_db.srctrldb openwrt_db.srctrlprj -``` - -You can now navigate into the resulting cartography with Sourcetrail. -```commandline -$ sourcetrail openwrt_db.srctrlprj -``` - -![](docs/img/example_sourcetrail.png) -

-Pyrrha result opened with Sourcetrail. -

+The usage workflow is composed of two steps which allow you to separate DB creation and result visualization. + +1. Run Pyrrha to obtain Sourcetrail compatible files (`*.srctrlprj` for the project file and `*.srctrldb` for the DB file). With the python package, you can just launch the command: + ``` + $ pyrrha + Usage: pyrrha [OPTIONS] COMMAND [ARGS]... + + Mapper collection for firmware analysis. + + Options: + -h, --help Show this message and exit. + + Commands: + fs Map PE and ELF files of a filesystem into a sourcetrail-compatible db. + + ``` + or with the Docker + ``` + $ docker run --rm -t -v $PWD:/tmp/pyrrha ghcr.io/quarkslab/pyrrha:latest [OPTIONS] COMMAND [ARGS]... + ``` +2. Visualize your results with Sourcetrail + ``` + $ sourcetrail PROJECT_NAME.srctrlprj + ``` + +The detailed documentation of each mapper is available in the [documentation](https://quarkslab.github.io/pyrrha/mappers/mappers/). + +## Publications + +Pyrrha has been presented at two conferences listed below. These talks include live demo of the `fs` parser which map links between libraries and executables files. + +- [Pyrrha: navigate easily into your system binaries 🖥️](https://github.com/quarkslab/conf-presentations/blob/master/Confs/HackLu23/pyrrha.pdf) [📽️](https://www.youtube.com/watch?v=-dMl-SvQl4k) *Hack.lu'23* +- [Map your Firmware! 🖥️](https://github.com/quarkslab/conf-presentations/blob/master/Confs/PTS23/PTS2023-Talk-14-Pyrrha-map-your-firmware.pdf) [📽️](https://passthesalt.ubicast.tv/videos/2023-map-your-firmware/) *PTS'23* ## Authors - Eloïse Brocas (@ebrocas), Quarkslab diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..90cb31c --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1 @@ +--8<-- "CHANGELOG.md" \ No newline at end of file diff --git a/docs/contributing/dev_mapper.md b/docs/contributing/dev_mapper.md new file mode 100644 index 0000000..7eb932e --- /dev/null +++ b/docs/contributing/dev_mapper.md @@ -0,0 +1,116 @@ +# Add a new mapper + +## Mapper Development +First develop your mapper. We are using `numbat` to manipulate the db used by sourcetrail to store the pieces of information to show in Sourcetrail. Everything is explained in Numbat's Getting Started Tutorial. (Numbat will be open-sourced before the end of March 2024) + +Then, add the required dependencies into `pyproject.toml`. + +## Integration into the main program +Once the mapper is ready, it should be integrated into `pyrrha` CLI by adding the corresponding subcommand in the `src/pyrrha_mapper/__main__.py`. The CLI system is handled with [click](https://click.palletsprojects.com) + +The subcommand corresponds to a function implementing the main of your mapper and some decorators to declare the subcommand name, its options and its arguments. + +The command name is declared with the following decorator. It automatically adds two options: `--db` to indicate the path of the db and `-d` to set the log level at `DEBUG` instead of `INFO`. + +```python linenums="121" +@pyrrha.command( + 'my_mapper', # the command name + cls=MapperCommand, # it will add default options + short_help='A quick help.', + help='A longer help, display only for this command help, no the general one.' +) +``` +You can now add options and arguments if needed. Below you can found some examples but as `click` is a powerful tool, check the documentation about [`click.option`](https://click.palletsprojects.com/options/) and [`click.argument`](https://click.palletsprojects.com/arguments/) for more details. +```python linenums="127" +# a flag option (if activated = True, else False) +@click.option('-o', '--myoption', # short and long option name + help='An help message', + is_flag=True, + default=False, + show_default=False) +# an option to precise the number of threads +@click.option('-j', '--jobs', + help='Number of parallel jobs created (threads).', + type=click.IntRange(1, multiprocessing.cpu_count(), clamp=True), + metavar='INT', + default=1, + show_default=True) +# an argument +@click.argument('target_directory', + type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path)) +``` +Then, you can implement the function that will run your mapper. It will have as parameters all +the options and arguments declared before. We also provide two utilities function which sets up the logs and create/open a db given a path. + +!!! note + + Do not forget that by default, the first two parameters will be `debug: bool, db: Path`. + +```python linenums="143" +def my_mapper(debug: bool, db: Path, myoption, jobs, target_directory): + setup_logs(debug) + db_instance = setup_db(db) + + # main work + + db_instance.close() # do not forget to close your db connection +``` + +???+ abstract "Final ` __main__.py`" + ``` py linenums="121" + @pyrrha.command( + 'my_mapper', # the command name + cls=MapperCommand, # it will add default options + short_help='A quick help.', + help='A longer help, display only for this command help, no the general one.' + ) + # a flag option (if activated = True, else False) + @click.option('-o', '--myoption', # short and long option name + help='An help message', + is_flag=True, + default=False, + show_default=False) + # an option to precise the number of threads + @click.option('-j', '--jobs', + help='Number of parallel jobs created (threads).', + type=click.IntRange(1, multiprocessing.cpu_count(), clamp=True), + metavar='INT', + default=1, + show_default=True) + # an argument + @click.argument('target_directory', + type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path)) + def my_mapper(debug: bool, db: Path, myoption, jobs, target_directory): + setup_logs(debug) + db_instance = setup_db(db) + + # main work + + db_instance.close() # do not forget to close your db connection + + if __name__ == '__main__': + pyrrha() + + ``` + + +## Documentation +Finally, you should add a page relative to your mapper inside the documentation. The list below resume the steps to add your mapper in every required places of the documentation: + +1. Write your documentation in a markdown file that should be place into the `docs/mappers` folder. + + !!! tip + We are using `material` theme of the `mkdocs` doc system. It provides a lot of nice features to improve your documentation like this note block. Do not hesitate to take a look at their [documentation](https://squidfunk.github.io/mkdocs-material/reference/)! + +2. Add your mapper in mapper lists (in `README.md` and in `docs/mappers/mappers.md`). +3. Complete the `nav` section in the `mkdocs.yml` file to add your file in the site navigation system. + + ```yaml linenums="33" hl_lines="7" + nav: + - Home: index.md + - Installation: installation.md + - Mappers: + - mappers/mappers.md + - Filesystem: mappers/fs.md + - My Mapper: mappers/my_mapper.md + ``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..4ab0748 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +--8<-- "README.md" \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..2fa5ab1 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,52 @@ +# Installation +The installation is done in two parts: + +- installing `Pyrrha` (as a Python module); +- installing `Sourcetrail` to be able to visualize Pyrrha's results. + +## Sourcetrail installation +Sourcetrail can be installed using [its last release](https://github.com/CoatiSoftware/Sourcetrail/releases/tag/2021.4.19) and its [documentation](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#installation). + +## Pyrrha installation +### Python package +Pyrrha requires a Python version >= 3.10. +It is recommended to install the Python package inside a virtualenv. You can use `pip` to install it. +```python +pip install pyrrha-mapper +``` +If you prefer using sources to install Pyrrha, do the following: +```commandline +# Do not forget to activate your virtualenv +$ pip install 'pyrrha @ git+https://github.com/quarkslab/pyrrha' + +# If you prefer, you can manually clone the repository and then install the package +$ git clone https://github.com/quarkslab/pyrrha +$ cd pyrrha +$ pip install '.' +``` + +### Docker +`pyrrha` can be used with a docker. It provides Pyrrha, but you still need to install Sourcetrail on your system as described in the [Sourcetrail Installation](#sourcetrail-installation) section. + +```commandline +$ cd ROOT_DIRECTORY/.. +$ docker run --rm -t -v $PWD:/tmp/pyrrha ghcr.io/quarkslab/pyrrha:latest fs [OPTIONS] ROOT_DIRECTORY +``` + +A docker image is directly available from our [Github registry](https://github.com/orgs/quarkslab/packages/container/package/pyrrha), but you can also build it from the sources. + +```commandline +$ git clone PYRRHA_URL && cd pyrrha +$ docker build -t pyrrha . +``` + +## Documentation + +If you want to build the documentation, you need to install Pyrrha with the `[doc]` dependencies and then serve the documentation on a local server with `mkdocs`. + +```bash +pip install 'pyrrha-mapper[doc]' + +# serve doc locally +mkdocs serve +``` \ No newline at end of file diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..64ad9b4 --- /dev/null +++ b/docs/license.md @@ -0,0 +1 @@ +--8<-- "LICENSE" \ No newline at end of file diff --git a/docs/mappers/fs.md b/docs/mappers/fs.md new file mode 100644 index 0000000..1bf1b66 --- /dev/null +++ b/docs/mappers/fs.md @@ -0,0 +1,91 @@ +# `fs`: ELF/PE imports/exports and the associated symlinks + + +## Usage +### Mapping with Pyrrha +First, create your db with `pyrrha`. The `ROOT_DIRECTORY` should contain the whole filesystem you want to map, it should be already extracted or mounted. `ROOT_DIRECTORY` will be considered by Pyrrha as the filesystem root for all the symlink resolutions. + +```commandline +Usage: pyrrha fs [OPTIONS] ROOT_DIRECTORY + + Map a filesystem into a sourcetrail-compatible db. It maps ELF and PE files, their imports and their + exports plus the symlinks that points on these executable files. + +Options: + -d, --debug Set log level to DEBUG + --db PATH Sourcetrail DB file path (.srctrldb). [default: pyrrha.srctrldb] + -e, --json Create a JSON export of the resulting mapping. + -j, --jobs INT Number of parallel jobs created (threads). [default: 1; 1<=x<=16] + -h, --help Show this message and exit. + +``` + +You can also export your Pyrrha results as a JSON file (option `-j`) to be able to postprocess them. For example, you can diff the results between two versions of the same system and list the binaries added/removed and which symbols has been added/removed (*cf* example script in `example`). + +### Visualization with Sourcetrail +Open the resulting project with `sourcetrail`. You can now navigate on the resulting cartography. The user interface is described in depth in the [Sourcetrail documentation](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#user-interface). + +To match the Sourcetrail language, the binaries, the exported functions and symbols, and the symlinks are represented as follows in Sourcetrail. + +Binaries | Exported functions | Exported symbols | Symlinks +:---:|:------------------------:|:------------------------:| :---: +![](../img/classes.png) | ![](../img/function.png) | ![](../img/variable.png) | ![](../img/typedefs.png) + +
+ ![](../img/imports.png) +
An example of the symbols and libraries imported by libgcc_s.so.1 and of the symbols which reference this library.
+
+ +
+ ![](../img/symlinks.png) +
An example of the symlinks which point on busybox.
+
+ +Do not hesitate to take a look at [Sourcetrail documentation](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#graph-view-1) to explore all the possibilities offered by Sourcetrail. [Custom Trails](https://github.com/CoatiSoftware/Sourcetrail/blob/master/DOCUMENTATION.md#custom-trail-dialog) could be really useful in a lot of cases. + +!!! example "Demo" + An live demo of how we can use Sourcetrail to visualize this mapper results is available [here](https://www.youtube.com/watch?v=-dMl-SvQl4k&t=12m33s). + +## Quick Start—Usage Example +Let's take the example of an OpenWRT firmware which is a common Linux distribution for embedded targets like routers. + +First, download the firmware and extract its root-fs into a directory. Here we download the last OpenWRT version for generic x86_64 systems. +```commandline +$ wget https://downloads.openwrt.org/releases/22.03.5/targets/x86/64/openwrt-22.03.5-x86-64-rootfs.tar.gz -O openwrt_rootfs.tar.gz +$ mkdir openwrt_root_fs && cd openwrt_root_fs +$ tar -xf ../openwrt_rootfs.tar.gz +$ cd .. && rm openwrt_rootfs.tar.gz +``` + +Then we can run Pyrrha on it. It will produce some logs indicating which symlinks or imports cannot be solved directly by the tool. +*(Do not forget to activate your virtualenv if you have created one for Pyrrha installation.)* +```commandline +$ pyrrha fs --db openwrt_db openwrt_root_fs +$ ls +openwrt_root_fs openwrt_db.srctrldb openwrt_db.srctrlprj +``` + +You can now navigate into the resulting cartography with Sourcetrail. +```commandline +$ sourcetrail openwrt_db.srctrlprj +``` + +
+ ![](../img/example_sourcetrail.png) +
Pyrrha result opened with Sourcetrail.
+
+ + +## Postprocessing `fs` result: the diffing example + +When you have to compare two bunch of executable files, for example two versions of the same firmware, it could be quickly difficult to determine where to start and have results in a short time. + +Diffing could be a solution. However, as binary diffing can be quite time-consuming, a first approach could be to diff the symbols contained in the binary files to determine which ones were added/removed. For example, using this technics can help you to determines quickly the files that have changed their internal structures versus the files that only contained little update of their dependency. To do that, you can use the JSON export of `fs` parser results. + +The following script prints on the standard output the list of files that has been added/removed and then the symbol changes file by file. + +???+ abstract "`examples/diffing_pyrrha_export.py`" + + ``` py + --8<-- "examples/diffing_pyrrha_exports.py" + ``` \ No newline at end of file diff --git a/docs/mappers/mappers.md b/docs/mappers/mappers.md new file mode 100644 index 0000000..61a5b1f --- /dev/null +++ b/docs/mappers/mappers.md @@ -0,0 +1,6 @@ +# Pyrrha Mappers + +Pyrrha provides the following mappers: + +- [`fs`](fs.md): a filesystem mapper. It maps ELF/PE files, their imports and their exports. + Also map symlinks which target ELF files. \ No newline at end of file diff --git a/img b/img new file mode 120000 index 0000000..412f2b2 --- /dev/null +++ b/img @@ -0,0 +1 @@ +docs/img \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..97d8fba --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,78 @@ +site_name: Pyrrha Documentation +site_description: "A mapper collection for firmware analysis." +repo_url: "https://github.com/quarkslab/pyrrha" +repo_name: "quarkslab/pyrrha" +watch: [ mkdocs.yml, README.md, CHANGELOG.md, src/pyrrha_mapper ] +copyright: Copyright © 2023-2024 Quarkslab + +theme: + name: "material" + palette: + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: red + accent: red + toggle: + icon: material/weather-sunny + name: Switch to dark mode + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: red + accent: red + toggle: + icon: material/weather-night + name: Switch to light mode + + features: + - content.code.annotate + - content.code.copy + - footer + +nav: + - Home: index.md + - Installation: installation.md + - Mappers: + - mappers/mappers.md + - Filesystem: mappers/fs.md + - Contributing: + - Mapper Development: contributing/dev_mapper.md + - Changelog: changelog.md + - License: license.md + + +plugins: + - autorefs + - glightbox # picture zoom + - search + - section-index + + + +markdown_extensions: + - admonition + - attr_list + - md_in_html + - pymdownx.details + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets: + base_path: [ ".", ".." ] + check_paths: true + - pymdownx.superfences + +extra: + social: + - icon: fontawesome/solid/globe + link: https:www.quarkslab.com + - icon: fontawesome/brands/github + link: https://github.com/quarkslab + - icon: fontawesome/brands/x-twitter + link: https://twitter.com/quarkslab \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 634f622..107a84f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2023 Quarkslab +# Copyright 2023-2024 Quarkslab # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ authors = [ { name = 'Eloïse Brocas', email = 'ebrocas@quarkslab.com' }, ] readme = "README.md" -description = "Filsesystem binaries mapper tool" +description = "A mapper collection for firmware analysis" requires-python = ">=3.10" license = {text = "Apache License 2.0"} classifiers = [ @@ -52,4 +52,14 @@ Tracker = "https://github.com/quarkslab/pyrrha/issues" Changelog = "https://github.com/quarkslab/pyrrha/changleog.md" [project.scripts] -pyrrha = 'pyrrha_mapper:pyrrha' \ No newline at end of file +pyrrha = 'pyrrha_mapper:pyrrha' + +[project.optional-dependencies] +doc = [ + 'black', + 'mkdocs', + 'mkdocs-autorefs', + 'mkdocs-glightbox', + 'mkdocs-material', + 'mkdocs-section-index' +] \ No newline at end of file diff --git a/src/pyrrha_mapper/__init__.py b/src/pyrrha_mapper/__init__.py index 5c42f2d..c62d002 100644 --- a/src/pyrrha_mapper/__init__.py +++ b/src/pyrrha_mapper/__init__.py @@ -1 +1,17 @@ +# -*- coding: utf-8 -*- + +# Copyright 2023-2024 Quarkslab +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .__main__ import pyrrha diff --git a/src/pyrrha_mapper/__main__.py b/src/pyrrha_mapper/__main__.py index 1f2e318..85508ab 100644 --- a/src/pyrrha_mapper/__main__.py +++ b/src/pyrrha_mapper/__main__.py @@ -1,3 +1,19 @@ +# -*- coding: utf-8 -*- + +# Copyright 2023-2024 Quarkslab +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import coloredlogs import logging import multiprocessing @@ -8,35 +24,92 @@ from .filesystem import FileSystemMapper -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +# ------------------------------------------------------------------------------- +# Common stuff for mappers +# ------------------------------------------------------------------------------- + + +class MapperCommand(click.Command): + """ + Common class to add shared options for mapper + + Code from: https://stackoverflow.com/a/53875557 + """ -@click.group(context_settings=CONTEXT_SETTINGS) -@click.option('-d', '--debug', is_flag=True, help='set log level to DEBUG') -def pyrrha(debug): - # define log style and level + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.params.insert( + 0, + click.core.Option(('--db',), + help='Sourcetrail DB file path (.srctrldb).', + type=click.Path(file_okay=True, dir_okay=True, path_type=Path), + default=Path() / 'pyrrha.srctrldb', + show_default=True) + ) + self.params.insert( + 0, + click.core.Option(('-d', '--debug'), + is_flag=True, help='Set log level to DEBUG') + ) + self.no_args_is_help = True + + +def setup_logs(is_debug_level: bool) -> None: + """ + Setup logs. + :param is_debug_level: if True set the log level as DEBUG else INFO + """ log_format = dict(fmt='[%(asctime)s][%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - coloredlogs.install(level=logging.DEBUG if debug else logging.INFO, + coloredlogs.install(level=logging.DEBUG if is_debug_level else logging.INFO, level_styles={'debug' : {'color': 'magenta'}, 'info': {'color': 'cyan'}, 'warning' : {'color': 'yellow'}, 'error': {'color': 'red'}, 'critical': {'bold': True, 'color': 'red'}}, field_styles={'asctime': {'color': 'green'}, 'levelname': {'bold': True}}, **log_format) +def setup_db(db_path, overwrite_db: bool = True) -> SourcetrailDB: + """ + Create and/or open the corresponding Sourcetrail DB. + :param db_path: path of the db to open/create + :param overwrite_db: if the path corresponds to an existing db, if True, it will be cleared else not + :return: the created or opened Sourcetrail DB + """ + # db creation/and or opening + if SourcetrailDB.exists(db_path): + db = SourcetrailDB.open(db_path, clear=overwrite_db) + else: + db = SourcetrailDB.create(db_path) + return db + + +# ------------------------------------------------------------------------------- +# CLI +# ------------------------------------------------------------------------------- + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'], + max_content_width=120) + + +@click.group(context_settings=CONTEXT_SETTINGS, + help='Mapper collection for firmware analysis.', + no_args_is_help=True) +def pyrrha(): + pass + + """ Filesystem mapper. - Map ELF files, their imports and their exports. - Also map symlinks which target ELF files. + Map ELF/PE files, their imports and their exports. + Also map symlinks which target ELF/PE files. """ @pyrrha.command('fs', - help='Map a filesystem into a sourcetrail-compatible db.') -@click.option('--db', - help='Sourcetrail DB file path (.srctrldb).', - type=click.Path(file_okay=True, dir_okay=True, path_type=Path), - default=Path() / 'pyrrha.srctrldb', - show_default=True) + cls=MapperCommand, + short_help='Map PE and ELF files of a filesystem into a sourcetrail-compatible db.', + help='Map a filesystem into a sourcetrail-compatible db. It maps ELF and PE files, \ +their imports and their exports plus the symlinks that points on these executable files.') @click.option('-e', '--json', help='Create a JSON export of the resulting mapping.', is_flag=True, @@ -45,22 +118,21 @@ def pyrrha(debug): @click.option('-j', '--jobs', help='Number of parallel jobs created (threads).', type=click.IntRange(1, multiprocessing.cpu_count(), clamp=True), + metavar='INT', default=1, show_default=True) @click.argument('root_directory', # help='Path of the directory containing the filesystem to map.', type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path)) -def fs(db: Path, json, jobs, root_directory): - if SourcetrailDB.exists(db): - db_interface = SourcetrailDB.open(db, clear=True) - else: - db_interface = SourcetrailDB.create(db) - root_directory = root_directory.absolute() +def fs(debug: bool, db: Path, json, jobs, root_directory): + setup_logs(debug) + db_instance = setup_db(db) - mapper = FileSystemMapper(root_directory, db_interface) - mapper.map(jobs, json) + root_directory = root_directory.absolute() + fs_mapper = FileSystemMapper(root_directory, db_instance) + fs_mapper.map(jobs, json) - db_interface.close() + db_instance.close() if __name__ == '__main__': diff --git a/src/pyrrha_mapper/filesystem.py b/src/pyrrha_mapper/filesystem.py index 048141b..1bd84da 100755 --- a/src/pyrrha_mapper/filesystem.py +++ b/src/pyrrha_mapper/filesystem.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2023 Quarkslab +# Copyright 2023-2024 Quarkslab # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ from numbat import SourcetrailDB from rich.progress import Progress +lief.logging.disable() @dataclass class Binary: