diff --git a/config/dpkg/control b/config/dpkg/control index 51ba198744..5c5e5bd5f6 100644 --- a/config/dpkg/control +++ b/config/dpkg/control @@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline) Package: python3-plaso Architecture: all -Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200414), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200501), python3-dfvfs (>= 20200604), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20181128), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends} +Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200414), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20150315), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-biplist (>= 1.0.3), python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200501), python3-dfvfs (>= 20200604), python3-dfwinreg (>= 20180712), python3-dtfabric (>= 20200621), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends} Description: Python 3 module of plaso (log2timeline) Plaso (log2timeline) is a framework to create super timelines. Its purpose is to extract timestamps from various files found on typical diff --git a/dependencies.ini b/dependencies.ini index 56b9083420..ecbd7c3b45 100644 --- a/dependencies.ini +++ b/dependencies.ini @@ -73,7 +73,7 @@ version_property: __version__ [dtfabric] dpkg_name: python3-dtfabric -minimum_version: 20181128 +minimum_version: 20200621 rpm_name: python3-dtfabric version_property: __version__ diff --git a/plaso/dependencies.py b/plaso/dependencies.py index 4a569d9ee7..0e1db54bf9 100644 --- a/plaso/dependencies.py +++ b/plaso/dependencies.py @@ -30,7 +30,7 @@ 'dfdatetime': ('__version__', '20200501', None, True), 'dfvfs': ('__version__', '20200604', None, True), 'dfwinreg': ('__version__', '20180712', None, True), - 'dtfabric': ('__version__', '20181128', None, True), + 'dtfabric': ('__version__', '20200621', None, True), 'elasticsearch': ('__versionstr__', '6.0', None, False), 'future': ('__version__', '0.16.0', None, True), 'idna': ('__version__', '2.5', None, True), diff --git a/plaso/parsers/asl.py b/plaso/parsers/asl.py index e66524c4f2..657d064596 100644 --- a/plaso/parsers/asl.py +++ b/plaso/parsers/asl.py @@ -63,8 +63,6 @@ class ASLParser(dtfabric_parser.DtFabricBaseParser): _DEFINITION_FILE = 'asl.yaml' - _FILE_SIGNATURE = b'ASL DB\x00\x00\x00\x00\x00\x00' - # Most significant bit of a 64-bit string offset. _STRING_OFFSET_MSB = 1 << 63 @@ -262,7 +260,8 @@ def GetFormatSpecification(cls): FormatSpecification: format specification. """ format_specification = specification.FormatSpecification(cls.NAME) - format_specification.AddNewSignature(cls._FILE_SIGNATURE, offset=0) + format_specification.AddNewSignature( + b'ASL DB\x00\x00\x00\x00\x00\x00', offset=0) return format_specification def ParseFileObject(self, parser_mediator, file_object): @@ -286,9 +285,6 @@ def ParseFileObject(self, parser_mediator, file_object): 'Unable to parse file header with error: {0!s}'.format( exception)) - if file_header.signature != self._FILE_SIGNATURE: - raise errors.UnableToParseFile('Invalid file signature.') - # TODO: generate event for creation time. file_size = file_object.get_size() diff --git a/plaso/parsers/asl.yaml b/plaso/parsers/asl.yaml index ee1e4a46fa..056fe26f07 100644 --- a/plaso/parsers/asl.yaml +++ b/plaso/parsers/asl.yaml @@ -47,6 +47,7 @@ members: type: stream element_data_type: byte elements_data_size: 12 + value: "ASL DB\x00\x00\x00\x00\x00\x00" - name: format_version data_type: uint32 - name: first_log_entry_offset diff --git a/plaso/parsers/bsm.py b/plaso/parsers/bsm.py index 33716d9a80..a1d90fa963 100644 --- a/plaso/parsers/bsm.py +++ b/plaso/parsers/bsm.py @@ -175,8 +175,6 @@ class BSMParser(dtfabric_parser.DtFabricBaseParser): 0x82: 'bsm_token_data_sockunix', } - _TRAILER_TOKEN_SIGNATURE = 0xb105 - _TOKEN_DATA_FORMAT_FUNCTIONS = { 0x11: '_FormatOtherFileToken', 0x21: '_FormatDataToken', @@ -684,9 +682,6 @@ def _ParseRecord(self, parser_mediator, file_object): token_values['error'], token_values['token_status'], token_values['call_status']) - if token_data.signature != self._TRAILER_TOKEN_SIGNATURE: - raise errors.ParseError('Unsupported signature in trailer token.') - if token_data.record_size != header_record_size: raise errors.ParseError( 'Mismatch of event record size between header and trailer token.') diff --git a/plaso/parsers/bsm.yaml b/plaso/parsers/bsm.yaml index 97a01c8047..523cd5968e 100644 --- a/plaso/parsers/bsm.yaml +++ b/plaso/parsers/bsm.yaml @@ -611,6 +611,7 @@ attributes: members: - name: signature data_type: uint16 + value: 0xb105 - name: record_size data_type: uint32 --- diff --git a/plaso/parsers/chrome_cache.py b/plaso/parsers/chrome_cache.py index 6b18abf16c..78f4969c07 100644 --- a/plaso/parsers/chrome_cache.py +++ b/plaso/parsers/chrome_cache.py @@ -113,8 +113,6 @@ class ChromeCacheIndexFileParser(dtfabric_parser.DtFabricBaseParser): _DEFINITION_FILE = 'chrome_cache.yaml' - _FILE_SIGNATURE = 0xc103cac3 - def __init__(self): """Initializes an index file.""" super(ChromeCacheIndexFileParser, self).__init__() @@ -140,9 +138,6 @@ def _ParseFileHeader(self, file_object): 'Unable to parse index file header with error: {0!s}'.format( exception)) - if file_header.signature != self._FILE_SIGNATURE: - raise errors.ParseError('Unsupported index file signature') - format_version = '{0:d}.{1:d}'.format( file_header.major_version, file_header.minor_version) if format_version not in ('2.0', '2.1'): @@ -208,8 +203,6 @@ class ChromeCacheDataBlockFileParser(dtfabric_parser.DtFabricBaseParser): _DEFINITION_FILE = 'chrome_cache.yaml' - _FILE_SIGNATURE = 0xc104cac3 - def _ParseFileHeader(self, file_object): """Parses the file header. @@ -230,9 +223,6 @@ def _ParseFileHeader(self, file_object): 'Unable to parse data block file header with error: {0!s}'.format( exception)) - if file_header.signature != self._FILE_SIGNATURE: - raise errors.ParseError('Unsupported data block file signature') - format_version = '{0:d}.{1:d}'.format( file_header.major_version, file_header.minor_version) if format_version not in ('2.0', '2.1'): diff --git a/plaso/parsers/chrome_cache.yaml b/plaso/parsers/chrome_cache.yaml index 6207f25586..382fe86243 100644 --- a/plaso/parsers/chrome_cache.yaml +++ b/plaso/parsers/chrome_cache.yaml @@ -46,6 +46,7 @@ attributes: members: - name: signature data_type: uint32 + value: 0xc104cac3 - name: minor_version data_type: uint16 - name: major_version @@ -130,6 +131,7 @@ attributes: members: - name: signature data_type: uint32 + value: 0xc103cac3 - name: minor_version data_type: uint16 - name: major_version diff --git a/plaso/parsers/custom_destinations.py b/plaso/parsers/custom_destinations.py index e2804f012d..1569494192 100644 --- a/plaso/parsers/custom_destinations.py +++ b/plaso/parsers/custom_destinations.py @@ -34,8 +34,6 @@ class CustomDestinationsParser(dtfabric_parser.DtFabricBaseParser): _LNK_GUID = ( b'\x01\x14\x02\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00\x46') - _FILE_FOOTER_SIGNATURE = 0xbabffbab - def _ParseLNKFile( self, parser_mediator, file_entry, file_offset, remaining_file_size): """Parses a LNK file stored within the .customDestinations-ms file. @@ -184,11 +182,6 @@ def ParseFileObject(self, parser_mediator, file_object): file_footer, _ = self._ReadStructureFromFileObject( file_object, file_offset, file_footer_map) - if file_footer.signature != self._FILE_FOOTER_SIGNATURE: - parser_mediator.ProduceExtractionWarning( - 'invalid entry header signature at offset: 0x{0:08x}'.format( - file_offset)) - except (ValueError, errors.ParseError) as exception: parser_mediator.ProduceExtractionWarning(( 'unable to parse footer at offset: 0x{0:08x} with error: ' @@ -212,10 +205,6 @@ def ParseFileObject(self, parser_mediator, file_object): file_footer, _ = self._ReadStructureFromFileObject( file_object, file_offset, file_footer_map) - if file_footer.signature != self._FILE_FOOTER_SIGNATURE: - parser_mediator.ProduceExtractionWarning( - 'invalid footer signature at offset: 0x{0:08x}'.format(file_offset)) - except (ValueError, errors.ParseError) as exception: parser_mediator.ProduceExtractionWarning(( 'unable to parse footer at offset: 0x{0:08x} with error: ' diff --git a/plaso/parsers/custom_destinations.yaml b/plaso/parsers/custom_destinations.yaml index 03b40b674f..4ca25d24ad 100644 --- a/plaso/parsers/custom_destinations.yaml +++ b/plaso/parsers/custom_destinations.yaml @@ -73,6 +73,7 @@ attributes: members: - name: signature data_type: uint32 + value: 0xbabffbab --- name: custom_entry_header type: structure diff --git a/plaso/parsers/fseventsd.py b/plaso/parsers/fseventsd.py index f9c55672ba..f3cfac43e1 100644 --- a/plaso/parsers/fseventsd.py +++ b/plaso/parsers/fseventsd.py @@ -56,8 +56,6 @@ class FseventsdParser(dtfabric_parser.DtFabricBaseParser): # The version 2 format was introduced in MacOS High Sierra (10.13). _DLS_V2_SIGNATURE = b'2SLD' - _DLS_SIGNATURES = [_DLS_V1_SIGNATURE, _DLS_V2_SIGNATURE] - _DEFINITION_FILE = 'fseventsd.yaml' @classmethod @@ -99,11 +97,6 @@ def _ParseDLSPageHeader(self, file_object, page_offset): 'Unable to parse page header at offset: 0x{0:08x} ' 'with error: {1!s}'.format(page_offset, exception)) - if page_header.signature not in self._DLS_SIGNATURES: - raise errors.ParseError( - 'Unsupported page header signature at offset: 0x{0:08x}'.format( - page_offset)) - return page_header, page_size def _BuildEventData(self, record): @@ -170,9 +163,6 @@ def ParseFileObject(self, parser_mediator, file_object): 'Unable to parse page header with error: {0!s}'.format( exception)) - if page_header.signature not in self._DLS_SIGNATURES: - raise errors.UnableToParseFile('Invalid file signature') - current_page_end = page_header.page_size file_entry = parser_mediator.GetFileEntry() diff --git a/plaso/parsers/fseventsd.yaml b/plaso/parsers/fseventsd.yaml index 16881e9802..0b81987432 100644 --- a/plaso/parsers/fseventsd.yaml +++ b/plaso/parsers/fseventsd.yaml @@ -32,6 +32,7 @@ members: type: stream element_data_type: byte number_of_elements: 4 + values: ["1SLD", "2SLD"] - name: padding type: stream element_data_type: byte diff --git a/plaso/parsers/mac_keychain.py b/plaso/parsers/mac_keychain.py index 3fe66983f8..91aa1be846 100644 --- a/plaso/parsers/mac_keychain.py +++ b/plaso/parsers/mac_keychain.py @@ -120,8 +120,6 @@ class KeychainParser(dtfabric_parser.DtFabricBaseParser): _DEFINITION_FILE = 'mac_keychain.yaml' - _FILE_SIGNATURE = b'kych' - _MAJOR_VERSION = 1 _MINOR_VERSION = 0 @@ -324,9 +322,6 @@ def _ReadFileHeader(self, file_object): file_header, _ = self._ReadStructureFromFileObject( file_object, 0, data_type_map) - if file_header.signature != self._FILE_SIGNATURE: - raise errors.ParseError('Unsupported file signature.') - if (file_header.major_format_version != self._MAJOR_VERSION or file_header.minor_format_version != self._MINOR_VERSION): raise errors.ParseError('Unsupported format version: {0:s}.{1:s}'.format( @@ -855,7 +850,7 @@ def GetFormatSpecification(cls): FormatSpecification: format specification. """ format_specification = specification.FormatSpecification(cls.NAME) - format_specification.AddNewSignature(cls._FILE_SIGNATURE, offset=0) + format_specification.AddNewSignature(b'kych', offset=0) return format_specification def ParseFileObject(self, parser_mediator, file_object): diff --git a/plaso/parsers/mac_keychain.yaml b/plaso/parsers/mac_keychain.yaml index 5f229cfd77..b76983d934 100644 --- a/plaso/parsers/mac_keychain.yaml +++ b/plaso/parsers/mac_keychain.yaml @@ -84,6 +84,7 @@ members: type: stream element_data_type: byte elements_data_size: 4 + value: "kych" - name: major_format_version data_type: uint16 - name: minor_format_version diff --git a/plaso/parsers/recycler.py b/plaso/parsers/recycler.py index bff896d5ca..dc7bc6214d 100644 --- a/plaso/parsers/recycler.py +++ b/plaso/parsers/recycler.py @@ -202,13 +202,10 @@ def _ParseInfo2Record( 'Unable to map record data at offset: 0x{0:08x} with error: ' '{1!s}').format(record_offset, exception)) - unicode_filename = unicode_filename.rstrip('\x00') - if record.deletion_time == 0: date_time = dfdatetime_semantic_time.NotSet() else: - date_time = dfdatetime_filetime.Filetime( - timestamp=record.deletion_time) + date_time = dfdatetime_filetime.Filetime(timestamp=record.deletion_time) event_data = WinRecycleBinEventData() event_data.drive_number = record.drive_number diff --git a/plaso/parsers/recycler.yaml b/plaso/parsers/recycler.yaml index 490b270ec0..198fa81d33 100644 --- a/plaso/parsers/recycler.yaml +++ b/plaso/parsers/recycler.yaml @@ -54,6 +54,7 @@ description: Windows Recycler INFO2 file entry Unicode original filename string type: string encoding: utf-16-le element_data_type: wchar16 +elements_data_size: 520 elements_terminator: "\x00\x00" --- name: recycler_info2_file_entry diff --git a/plaso/parsers/safari_cookies.py b/plaso/parsers/safari_cookies.py index b34f9df542..ec97048744 100644 --- a/plaso/parsers/safari_cookies.py +++ b/plaso/parsers/safari_cookies.py @@ -51,8 +51,6 @@ class BinaryCookieParser(dtfabric_parser.DtFabricBaseParser): _DEFINITION_FILE = 'safari_cookies.yaml' - _SIGNATURE = b'cook' - def __init__(self): """Initializes a parser object.""" super(BinaryCookieParser, self).__init__() @@ -220,9 +218,6 @@ def ParseFileObject(self, parser_mediator, file_object): raise errors.UnableToParseFile( 'Unable to read file header with error: {0!s}.'.format(exception)) - if file_header.signature != self._SIGNATURE: - raise errors.UnableToParseFile('Unsupported file signature.') - file_offset = file_header_data_size # TODO: move page sizes array into file header, this will require dtFabric diff --git a/plaso/parsers/safari_cookies.yaml b/plaso/parsers/safari_cookies.yaml index 0ce960f6e2..4903fd6e9a 100644 --- a/plaso/parsers/safari_cookies.yaml +++ b/plaso/parsers/safari_cookies.yaml @@ -60,6 +60,7 @@ members: type: stream element_data_type: byte number_of_elements: 4 + value: "cook" - name: number_of_pages data_type: uint32 --- diff --git a/plaso/parsers/systemd_journal.py b/plaso/parsers/systemd_journal.py index 686f5a340d..04d84afe5d 100644 --- a/plaso/parsers/systemd_journal.py +++ b/plaso/parsers/systemd_journal.py @@ -48,8 +48,6 @@ class SystemdJournalParser(dtfabric_parser.DtFabricBaseParser): _DEFINITION_FILE = 'systemd_journal.yaml' - _FILE_SIGNATURE = b'LPKSHHRH' - _OBJECT_COMPRESSED_FLAG_XZ = 1 _OBJECT_COMPRESSED_FLAG_LZ4 = 2 @@ -271,7 +269,7 @@ def GetFormatSpecification(cls): FormatSpecification: format specification. """ format_specification = specification.FormatSpecification(cls.NAME) - format_specification.AddNewSignature(cls._FILE_SIGNATURE, offset=0) + format_specification.AddNewSignature(b'LPKSHHRH', offset=0) return format_specification def ParseFileObject(self, parser_mediator, file_object): @@ -294,9 +292,6 @@ def ParseFileObject(self, parser_mediator, file_object): 'Unable to parse file header with error: {0!s}'.format( exception)) - if file_header.signature != self._FILE_SIGNATURE: - raise errors.UnableToParseFile('Invalid file signature.') - if file_header.header_size not in self._SUPPORTED_FILE_HEADER_SIZES: raise errors.UnableToParseFile( 'Unsupported file header size: {0:d}.'.format( diff --git a/plaso/parsers/systemd_journal.yaml b/plaso/parsers/systemd_journal.yaml index 6b6a67320e..7e204256bb 100644 --- a/plaso/parsers/systemd_journal.yaml +++ b/plaso/parsers/systemd_journal.yaml @@ -48,6 +48,7 @@ members: type: stream element_data_type: byte elements_data_size: 8 + value: "LPKSHHRH" - name: compatible_flags data_type: uint32 - name: incompatible_flags diff --git a/requirements.txt b/requirements.txt index d7b47b6c1f..80e8f817eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,10 +9,10 @@ cffi >= 1.9.1 chardet >= 2.0.1 cryptography >= 2.0.2 defusedxml >= 0.5.0 -dfdatetime >= 20180704 +dfdatetime >= 20200501 dfvfs >= 20200604 dfwinreg >= 20180712 -dtfabric >= 20181128 +dtfabric >= 20200621 elasticsearch >= 6.0 future >= 0.16.0 idna >= 2.5 diff --git a/setup.cfg b/setup.cfg index e716639bb9..fb545cac57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,10 +51,10 @@ requires = libbde-python3 >= 20140531 python3-cryptography >= 2.0.2 python3-dateutil >= 1.5 python3-defusedxml >= 0.5.0 - python3-dfdatetime >= 20180704 + python3-dfdatetime >= 20200501 python3-dfvfs >= 20200604 python3-dfwinreg >= 20180712 - python3-dtfabric >= 20181128 + python3-dtfabric >= 20200621 python3-elasticsearch >= 6.0 python3-future >= 0.16.0 python3-idna >= 2.5