Merge branch 'master' of https://github.com/Arker123/flare-floss into…

… HEAD

.github/workflows/build.yml

@@ -26,19 +26,21 @@ jobs:
     # Pin action version by commit hash to maximize trust, ref: https://securitylab.github.com/research/github-actions-building-blocks/
     steps:
       - name: Checkout floss
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           submodules: false
       # using Python 3.8 to support running across multiple operating systems including Windows 7
       - name: Set up Python 3.8
-        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
           python-version: '3.8'
       - name: Install floss [build]
-        run: pip install -e .[build]
+        run: |
+          pip install -r requirements.txt
+          pip install -e .[build]
       - name: Build standalone executable
         run: pyinstaller .github/pyinstaller/floss.spec
-      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: ${{ matrix.asset_name }}
           path: dist/${{ matrix.artifact_name }}
@@ -67,11 +69,11 @@ jobs:
             asset_name: macos
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
         with:
           name: ${{ matrix.asset_name }}
       - name: Checkout testfiles
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           repository: mandiant/flare-floss-testfiles
           path: tests/data
@@ -100,7 +102,7 @@ jobs:
             artifact_name: floss
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag
@@ -110,7 +112,7 @@ jobs:
       - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
         run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
       - name: Upload ${{ env.zip_name }} to GH Release
-        uses: svenstaro/upload-release-action@v2
+        uses: svenstaro/upload-release-action@04733e069f2d7f7f0b4aebc4fbdbce8613b03ccd # v2.9.0
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           file: ${{ env.zip_name }}

.github/workflows/publish.yml

@@ -1,31 +1,42 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
+# use PyPI trusted publishing, as described here: 
+# https://blog.trailofbits.com/2023/05/23/trusted-publishing-a-new-benchmark-for-packaging-security/
 name: publish to pypi
 
 on:
   release:
     types: [published]
-    branches: [master]
+
+permissions:
+  contents: write
 
 jobs:
-  deploy:
-    runs-on: ubuntu-20.04
-    # Pin action version by commit hash to maximize trust, ref: https://securitylab.github.com/research/github-actions-building-blocks/
+  pypi-publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+    permissions:
+      id-token: write
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Set up Python
-        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
           python-version: '3.8'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+          pip install -r requirements.txt
+          pip install -e .[build]
+      - name: build package
         run: |
-          python setup.py sdist bdist_wheel
-          twine upload --skip-existing dist/*
+          python -m build
+      - name: upload package artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          path: dist/*
+      - name: publish package
+        uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450  # v1.8.14
+        with:
+          skip-existing: true
+          verbose: true
+          print-hash: true

.github/workflows/tests.yml

@@ -16,13 +16,15 @@ jobs:
     # Pin action version by commit hash to maximize trust, ref: https://securitylab.github.com/research/github-actions-building-blocks/
     steps:
     - name: Checkout FLOSS
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - name: Set up Python 3.8
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: '3.8'
     - name: Install dependencies
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev]
     - name: Lint with isort
       run: pre-commit run isort
     - name: Lint with black
@@ -48,17 +50,19 @@ jobs:
             python-version: '3.10'
     steps:
     - name: Checkout FLOSS with submodule
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         submodules: true
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pyyaml
       if: matrix.os == 'ubuntu-20.04'
       run: sudo apt-get install -y libyaml-dev
     - name: Install FLOSS
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev]
     - name: Run tests
       run: pytest tests/

.gitignore

@@ -29,3 +29,6 @@ flare_floss.egg-info
 
 # vscode
 .vscode
+.direnv/
+.env/
+.envrc

README.md

@@ -39,7 +39,7 @@ Not all compilers use string formats that the classic `strings.exe` algorithm su
  1. Go
  2. Rust
 
-The strings FLOSS extracts specific to a compiler are must easier to inspect by humans. 
+The strings FLOSS extracts specific to a compiler are much easier to inspect by humans. 
 
 Please consult the documentation to learn more about the [language-specific string extraction](doc/language_specific_strings.md).
 

doc/installation.md

@@ -77,6 +77,9 @@ This means that Python will load the FLOSS module from this local
 This is good, because it is easy for us to modify files and see the
  effects reflected immediately.
 But be careful not to remove this directory unless uninstalling FLOSS!
+If you encounter the error `ERROR: Project has a 'pyproject.toml' and its build backend is missing the 'build_editable' hook.`,
+ please ensure that you have upgraded to the latest versions of pip and setuptools.
+
 
 - Install FLOSS:
 
@@ -87,6 +90,12 @@ You'll find that the `floss.exe` (Windows) or `floss` (Linux, macOS) executables
 
 ### Step 3: Install development and testing dependencies
 
+When developing FLOSS, please use the pinned dependencies found in `requirements.txt`.
+This ensures that everyone has the exact same, reproducible environment.
+Please install these dependencies before install FLOSS (from source or from PyPI):
+
+`$ pip install -r requirements.txt`
+
 To install all testing and development dependencies, run:
 
 `$ pip install -e /local/path/to/src[dev]`
@@ -112,22 +121,27 @@ We use the following tools to ensure consistent code style and formatting:
 We use [pre-commit](https://pre-commit.com/) so that its trivial to run the same linters & configuration locally as in CI.
 
 Run all linters liks:
-    ❯ pre-commit run --all-files
-    isort....................................................................Passed
-    black....................................................................Passed
-    mypy.....................................................................Passed
-
+```
+❯ pre-commit run --all-files
+isort....................................................................Passed
+black....................................................................Passed
+mypy.....................................................................Passed
+```
+
 Or run a single linter like:
-    ❯ pre-commit run --all-files isort
-    isort....................................................................Passed
+```
+❯ pre-commit run --all-files isort
+isort....................................................................Passed
+```
 
 Importantly, you can configure pre-commit to run automatically before every commit by running:
+```
+❯ pre-commit install --hook-type pre-commit
+pre-commit installed at .git/hooks/pre-commit
 
-    ❯ pre-commit install --hook-type pre-commit
-    pre-commit installed at .git/hooks/pre-commit
-
-    ❯ pre-commit install --hook-type pre-push
-    pre-commit installed at .git/hooks/pre-push
+❯ pre-commit install --hook-type pre-push
+pre-commit installed at .git/hooks/pre-push
+```
 
 This way you can ensure that you don't commit code style or formatting offenses.
 You can always temporarily skip the checks by using the `-n`/`--no-verify` git option.

doc/usage.md

@@ -140,6 +140,16 @@ Specify functions by using their hex-encoded virtual address.
     floss.exe --functions 0x401000 0x402000 malware.exe
 
 
+### Install/Uninstall right click menu option for Windows (`--install-right-click-menu/--uninstall-right-click-menu`)
+
+You can use the `--install-right-click-menu` and `--uninstall-right-click-menu` 
+ options to install/remove the `Open with FLOSS` option from the right-click menu 
+ of the Windows file explorer.
+
+After this option is installed, you can right-click on any file and select `Open with FLOSS`
+ to quickly open the target file with FLOSS for analysis.
+
+
 ## <a name="shellcode"></a>Shellcode analysis options
 
 Malicious shellcode often times contains obfuscated strings or stackstrings.

floss/identify.py

@@ -151,7 +151,11 @@ def find_decoding_function_features(vw, functions, disable_progress=False) -> Tu
 
             f = viv_utils.Function(vw, function_address)
 
-            function_data = {"meta": get_function_meta(f), "features": list()}
+            function_data = {
+                "meta": get_function_meta(f),
+                "features": [],
+                "xrefs_to": len(list(vw.getXrefsTo(function_address))),
+            }
 
             # meta data features
             function_data["features"].append(BlockCount(function_data["meta"].get("block_count")))

floss/language/__init__.py

@@ -0,0 +1 @@
+

floss/language/go/__init__.py

@@ -0,0 +1 @@
+

floss/language/go/extract.py

@@ -37,6 +37,9 @@ def find_stack_strings_with_regex(
                 if not binary_string:
                     continue
 
+                if binary_string.endswith(b"\x00"):
+                    binary_string = binary_string[:-1]
+
                 addr = m.start()
                 # need to subtract opcode bytes offset
                 off_regex = len(m.group(0)) - len(binary_string)
@@ -98,6 +101,9 @@ def find_i386_stackstrings(section_data, offset, min_length):
 def get_stackstrings(pe: pefile.PE, min_length: int) -> Iterable[StaticString]:
     """
     Find stackstrings in the given PE file.
+
+    TODO(mr-tz): algorithms need improvements / rethinking of approach
+     https://github.com/mandiant/flare-floss/issues/828
     """
 
     for section in pe.sections:
@@ -269,7 +275,9 @@ def get_string_blob_strings(pe: pefile.PE, min_length) -> Iterable[StaticString]
     with floss.utils.timing("find struct string candidates"):
         struct_strings = list(sorted(set(get_struct_string_candidates(pe)), key=lambda s: s.address))
         if not struct_strings:
-            logger.warning("Failed to find struct string candidates: Is this a Go binary?")
+            logger.warning(
+                "Failed to find struct string candidates: Is this a Go binary? If so, the Go version may be unsupported."
+            )
             return
 
     with floss.utils.timing("find string blob"):
@@ -354,12 +362,14 @@ def get_string_blob_strings(pe: pefile.PE, min_length) -> Iterable[StaticString]
         last_buf = string_blob_buf[last_pointer_offset:]
         for size in range(len(last_buf), 0, -1):
             try:
-                s = last_buf[:size].decode("utf-8")
+                _ = last_buf[:size].decode("utf-8")
             except UnicodeDecodeError:
                 continue
             else:
                 try:
-                    string = StaticString.from_utf8(last_buf[:size], last_pointer, min_length)
+                    string = StaticString.from_utf8(
+                        last_buf[:size], pe.get_offset_from_rva(last_pointer - image_base), min_length
+                    )
                     yield string
                 except ValueError:
                     pass
@@ -382,6 +392,25 @@ def extract_go_strings(sample, min_length) -> List[StaticString]:
     return go_strings
 
 
+def get_static_strings_from_blob_range(sample: pathlib.Path, static_strings: List[StaticString]) -> List[StaticString]:
+    pe = pefile.PE(data=pathlib.Path(sample).read_bytes(), fast_load=True)
+
+    struct_strings = list(sorted(set(get_struct_string_candidates(pe)), key=lambda s: s.address))
+    if not struct_strings:
+        return []
+
+    try:
+        string_blob_start, string_blob_end = find_string_blob_range(pe, struct_strings)
+    except ValueError:
+        return []
+
+    image_base = pe.OPTIONAL_HEADER.ImageBase
+    string_blob_start = pe.get_offset_from_rva(string_blob_start - image_base)
+    string_blob_end = pe.get_offset_from_rva(string_blob_end - image_base)
+
+    return list(filter(lambda s: string_blob_start <= s.offset < string_blob_end, static_strings))
+
+
 def main(argv=None):
     parser = argparse.ArgumentParser(description="Get Go strings")
     parser.add_argument("path", help="file or path to analyze")