Rebase pyelftools and reapply S2E-specific commits as needed

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: pyelftools-tests
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [2.7, 3.6, 3.7, 3.8]
+        os: [ubuntu-latest]
+
+    steps:
+
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Test
+      run: |
+        python test/all_tests.py
+

.gitignore

@@ -7,5 +7,8 @@ build
 dist
 MANIFEST
 *.sublime-workspace
+*.egg-info
+
+
 
 

CHANGES

@@ -1,19 +1,68 @@
 Changelog
 =========
 
-+ Version 0.25 (??)
++ Version 0.28 (2022.02.03)
+
+  - Added a method for returning the index of a section by name (#331)
+  - Allow filtering by section types in iter_sections (#345)
+  - Support Android compressed rel/rela sections (#357)
+  - Initial support for PPC64LE (#360)
+  - Initial DWARF v5 support (#363 with several follow-ups)
+  - Fixed parsing for structures containing uids or gids in core
+    dumps (#354)
+  - Allow filtering by segment types in iter_segments (#375)
+  - Add support for .note.gnu.property (#386)
+  - Update readelf tests to work with more recent version of
+    readelf (#387)
+  - Add support for note GNU_PROPERTY_X86_FEATURE_1_AND (#388)
+
++ Version 0.27 (2020.10.27)
+
+  - Print addend wfor RELA relocations without symbol (#292)
+  - Implement symbol lookup for {GNU,}HashSection (#290)
+  - Major rewrite of expression parsing
+  - Cashed random access to CUs and DIEs (#264)
+  - GNU expressions (#303)
+  - Support parsing LSDA pointers from FDEs (#308)
+  - Add support for DWA_OP_GNU_push_tls_address in expressions (#315)
+  - Some initial support for AArch64 little-endian (#318)
+  - Support for ELF files with a large number of sections (#333)
+  - Some minimal support for DWARFv1 (#335)
+  - Many small bug fixes; see git log.
+
++ Version 0.26 (2019.12.05)
+
+  - Call relocation for ARM v3 (#194)
+  - More complete architecture coverage for ENUM_E_MACHINE (#206)
+  - Support for .debug_pubtypes and .debug_pubnames sections (#208)
+  - Support for DWARF v4 location lists (#214)
+  - Decode strings in dynamic string tables (#217)
+  - Improve symbol table handling in dynamic segments (#219)
+  - Improved handling of location information (#225)
+  - Avoid deprecation warnings in Python 3.7+
+  - Add DWARF v5 OPs (#240)
+  - Handle many new translation forms and constants
+  - Lazy DIE parsing to speed up partial parsing of DWARF info (#249)
+
++ Version 0.25 (2018.09.01)
 
   - Make parsing of SH_TYPE and PT_TYPE fields dependent on the machine
     (e_machine header field), making it possible to support conflicting type
     enums between different machines (#71 and #121).
+  - Add parsing and readelf dumping for .eh_frame (#155)
+  - Support compressed sections (#152)
+  - Better support for parsing core dumps (#147)
+  - More comprehensive handling of ARM relocations (#121)
+  - Convert all ascii encoding to utf-8 encoding (#182)
   - Don't attempt to hex/string dump SHT_NOBITS sections in readelf (#119).
-  - Add Python 3.5 testing to the tox file.
+  - Test with Python 3.6
   - Minor bugfixes (#118)
   - Cleanup: Use argparse instead of optparse
   - Make readelf comparison tests run in parallel using multiprocessing; cuts
     testing time 3-5x
+  - Improvements in MIPS flags handling (#165)
 
-+ Version 0.24 (04.08.2016)
++ Version 0.24 (2016.08.04)
 
   - Retrieve symbols by name - get_symbol_by_name (#58).
   - Symbol/section names are strings internally now, not bytestrings (this may
@@ -29,15 +78,15 @@ Changelog
   - Support for zlib-compressed debug sections (#102)
   - Support for DWARF v4 line programs (#82)
 
-+ Version 0.23 (08.11.2014)
++ Version 0.23 (2014.11.08)
 
   - Minimal Python 2.x version raised to 2.7
   - Basic support for MIPS (contributed by Karl Vogel).
   - Support for PT_NOTE segment parsing (contributed by Alex Deymo).
   - Support for parsing symbol table in dynamic segment
     (contributed by Nam T. Nguyen).
 
-+ Version 0.22 (30.03.2014)
++ Version 0.22 (2014.03.30)
 
   - pyelftools repository moved to https://github.com/eliben/pyelftools
   - Support for version sections - contributed by Yann Rouillard.
@@ -50,7 +99,7 @@ Changelog
     file name / path (based on pull request #16 by Shaheed Haque).
   - Set up Travis CI integration.
 
-+ Version 0.21 (17.04.2013)
++ Version 0.21 (2013.04.17)
 
   - Added new example: dwarf_decode_address - decode function name and
     file & line information from an address.
@@ -63,11 +112,11 @@ Changelog
   - Output of scripts/readelf.py now matches that of binutils 2.23.52.
   - Added more machine EM_ values to ENUM_E_TYPE.
 
-+ Version 0.20 (27.01.2012)
++ Version 0.20 (2012.01.27)
 
   - Python 3 support
   - Fixed some problems with running tests
   - Issue #2: made all examples run (and test/run_examples_test.py pass)
     on Windows.
 
-+ Version 0.10 - Initial public release (06.01.2012)
++ Version 0.10 - Initial public release (2012.01.06)

MANIFEST.in

@@ -1,8 +1,8 @@
 recursive-include elftools *.py
 recursive-include scripts *.py
 recursive-include examples *.py *.elf *.out
-recursive-include test *.py *.elf *.arm *.mips
-include README
+recursive-include test *.py *.elf *.arm *.mips *.o
+include README.rst
 include LICENSE
 include CHANGES
 include tox.ini

README.rst

@@ -1,5 +1,10 @@
-Introduction: what is pyelftools?
----------------------------------
+==========
+pyelftools
+==========
+
+.. image:: https://github.com/eliben/pyelftools/workflows/pyelftools-tests/badge.svg
+  :align: center
+  :target: https://github.com/eliben/pyelftools/actions
 
 **pyelftools** is a pure-Python library for parsing and analyzing ELF files
 and DWARF debugging information. See the
@@ -10,7 +15,7 @@ Pre-requisites
 --------------
 
 As a user of **pyelftools**, one only needs Python to run. It works with
-Python versions 2.7 and 3.x (x >= 2). For hacking on **pyelftools** the
+Python versions 2.7 and 3.x (x >= 5). For hacking on **pyelftools** the
 requirements are a bit more strict, please see the
 `hacking guide <https://github.com/eliben/pyelftools/wiki/Hacking-guide>`_.
 
@@ -33,6 +38,9 @@ recent version of the code. This can be done by downloading the `master zip
 file <https://github.com/eliben/pyelftools/archive/master.zip>`_ or just
 cloning the Git repository.
 
+Since **pyelftools** has no external dependencies, it's also easy to use it
+without installing, by locally adjusting ``PYTHONPATH``.
+
 How to use it?
 --------------
 
@@ -45,13 +53,3 @@ License
 
 **pyelftools** is open source software. Its code is in the public domain. See
 the ``LICENSE`` file for more details.
-
-CI Status
----------
-
-**pyelftools** has automatic testing enabled through the convenient
-`Travis CI project <https://travis-ci.org>`_. Here is the latest build status:
-
-.. image:: https://travis-ci.org/eliben/pyelftools.png?branch=master
-  :align: center
-  :target: https://travis-ci.org/eliben/pyelftools

TODO

@@ -9,15 +9,10 @@ New version
 construct
 ---------
 
-The construct seems to be maintained again - they also backported my Python 3
-fixes. Theoretically, I can remove construct from pyelftools and use it as a
-dependency instead. I don't really have time to play with this now, but may
-do so in the future.
-
-Distribution
-------------
-
-python setup.py build sdist bdist_wheel upload
+construct seems to be maintained again - they also backported my Python 3 fixes.
+Theoretically, I can remove construct from pyelftools and use it as a dependency
+instead. I don't really have time to play with this now, but may do so in the
+future.
 
 Preparing a new release
 -----------------------
@@ -28,4 +23,14 @@ Preparing a new release
 * Untar the created ``dist/pyelftools-x.y.tar.gz`` and make sure
   everything looks ok
 * Now build with upload to send it to PyPi
+* Tag new version in git
 * Test with pip install from some new virtualenv
+
+Distribution
+------------
+
+1. First install Twine (https://packaging.python.org/tutorials/packaging-projects/)
+2. python3 -m twine upload dist/*, but make sure ``setup.py`` was already run
+   and the updated whl and tarbal are in dist/.
+
+Credentials for PyPI are stored in ~/.pypirc

elftools/__init__.py

@@ -4,4 +4,4 @@
 # Eli Bendersky ([email protected])
 # This code is in the public domain
 #-------------------------------------------------------------------------------
-__version__ = '0.24'
+__version__ = '0.28'

elftools/common/construct_utils.py

@@ -8,7 +8,7 @@
 #-------------------------------------------------------------------------------
 from ..construct import (
     Subconstruct, ConstructError, ArrayError, Adapter, Field, RepeatUntil,
-    Rename
+    Rename, SizeofError
     )
 
 

elftools/common/exceptions.py

@@ -6,12 +6,12 @@
 # Eli Bendersky ([email protected])
 # This code is in the public domain
 #-------------------------------------------------------------------------------
-class ELFError(Exception): 
+class ELFError(Exception):
     pass
 
 class ELFRelocationError(ELFError):
     pass
-        
+
 class ELFParseError(ELFError):
     pass
 

elftools/common/py3compat.py

@@ -21,6 +21,11 @@
     # and strings are different types and bytes hold numeric values when
     # iterated over.
 
+    def bytes2hex(b, sep=''):
+        if not sep:
+            return b.hex()
+        return sep.join(map('{:02x}'.format, b))
+
     def bytes2str(b): return b.decode('latin-1')
     def str2bytes(s): return s.encode('latin-1')
     def int2byte(i): return bytes((i,))
@@ -41,6 +46,12 @@ def iterbytes(b):
     import cStringIO
     StringIO = BytesIO = cStringIO.StringIO
 
+    def bytes2hex(b, sep=''):
+        res = b.encode('hex')
+        if not sep:
+            return res
+        return sep.join(res[i:i+2] for i in range(0, len(res), 2))
+
     def bytes2str(b): return b
     def str2bytes(s): return s
     int2byte = chr
@@ -64,3 +75,8 @@ def itervalues(d):
 def iteritems(d):
     """Return an iterator over the items of a dictionary."""
     return getattr(d, 'items' if PY3 else 'iteritems')()
+
+try:
+    from collections.abc import Mapping  # python >= 3.3
+except ImportError:
+    from collections import Mapping  # python < 3.3

elftools/common/utils.py

@@ -9,7 +9,7 @@
 from contextlib import contextmanager
 from .exceptions import ELFParseError, ELFError, DWARFError
 from .py3compat import int2byte
-from ..construct import ConstructError
+from ..construct import ConstructError, ULInt8
 
 
 def merge_dicts(*dicts):
@@ -102,6 +102,11 @@ def roundup(num, bits):
     """
     return (num - 1 | (1 << bits) - 1) + 1
 
+def read_blob(stream, length):
+    """Read length bytes from stream, return a list of ints
+    """
+    return [struct_parse(ULInt8(''), stream) for i in range(length)]
+
 #------------------------- PRIVATE -------------------------
 
 def _assert_with_exception(cond, msg, exception_type):

elftools/construct/adapters.py

@@ -279,7 +279,7 @@ class ExprAdapter(Adapter):
     * subcon - the subcon to adapt
     * encoder - a function that takes (obj, context) and returns an encoded
       version of obj
-    * decoder - a function that takes (obj, context) and returns an decoded
+    * decoder - a function that takes (obj, context) and returns a decoded
       version of obj
 
     Example:

elftools/construct/core.py

@@ -345,7 +345,7 @@ def __getstate__(self):
         return attrs
     def __setstate__(self, attrs):
         attrs["packer"] = Packer(attrs["packer"])
-        return StaticField.__setstate__(attrs)
+        return StaticField.__setstate__(self, attrs)
     def _parse(self, stream, context):
         try:
             return self.packer.unpack(_read_stream(stream, self.length))[0]
@@ -1297,6 +1297,8 @@ def _build(self, obj, stream, context):
         assert obj is None
     def _sizeof(self, context):
         return 0
+    def __reduce__(self):
+        return self.__class__.__name__
 Pass = Pass(None)
 
 class Terminator(Construct):