diff --git a/plaso/parsers/dtfabric_parser.py b/plaso/parsers/dtfabric_parser.py index 783ecb985f..3cc48e0672 100644 --- a/plaso/parsers/dtfabric_parser.py +++ b/plaso/parsers/dtfabric_parser.py @@ -247,6 +247,7 @@ def _ReadStructureFromFileObject( ValueError: if file-like object or data type map is missing. """ context = None + data = b'' last_data_size = 0 data_size = data_type_map.GetByteSize() @@ -254,7 +255,11 @@ def _ReadStructureFromFileObject( data_size = data_type_map.GetSizeHint() while data_size != last_data_size: - data = self._ReadData(file_object, file_offset, data_size) + read_offset = file_offset + last_data_size + read_size = data_size - last_data_size + data_segment = self._ReadData(file_object, read_offset, read_size) + + data = b''.join([data, data_segment]) try: context = dtfabric_data_maps.DataTypeMapContext() diff --git a/plaso/parsers/systemd_journal.py b/plaso/parsers/systemd_journal.py index a78ed9657e..b40bc70656 100644 --- a/plaso/parsers/systemd_journal.py +++ b/plaso/parsers/systemd_journal.py @@ -81,7 +81,7 @@ def _ParseDataObject(self, file_object, file_offset): of the file-like object. Returns: - systemd_journal_data_object: data object. + bytes: data. Raises: ParseError: if the data object cannot be parsed. @@ -105,7 +105,28 @@ def _ParseDataObject(self, file_object, file_offset): raise errors.ParseError('Unsupported object flags: 0x{0:02x}.'.format( data_object.object_flags)) - return data_object + # The data is read seperately for performance reasons. + data_size = data_object.data_size - 64 + data = file_object.read(data_size) + + if data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_XZ: + data = lzma.decompress(data) + + elif data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_LZ4: + uncompressed_size_map = self._GetDataTypeMap('uint32le') + + try: + uncompressed_size = self._ReadStructureFromByteStream( + data, file_offset + 64, uncompressed_size_map) + except (ValueError, errors.ParseError) as exception: + raise errors.ParseError(( + 'Unable to parse LZ4 uncompressed size at offset: 0x{0:08x} with ' + 'error: {1!s}').format(file_offset + 64, exception)) + + data = lz4.block.decompress( + data[8:], uncompressed_size=uncompressed_size) + + return data def _ParseEntryArrayObject(self, file_object, file_offset): """Parses an entry array object. @@ -245,26 +266,7 @@ def _ParseJournalEntryItem(self, file_object, file_offset): Raises: ParseError: if the LZ4 uncompressed size cannot be parsed. """ - data_object = self._ParseDataObject(file_object, file_offset) - - event_data = data_object.data - if data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_XZ: - event_data = lzma.decompress(event_data) - - elif data_object.object_flags & self._OBJECT_COMPRESSED_FLAG_LZ4: - uncompressed_size_map = self._GetDataTypeMap('uint32le') - - try: - uncompressed_size = self._ReadStructureFromByteStream( - event_data, file_offset + 64, uncompressed_size_map) - except (ValueError, errors.ParseError) as exception: - raise errors.ParseError(( - 'Unable to parse LZ4 uncompressed size at offset: 0x{0:08x} with ' - 'error: {1!s}').format(file_offset + 64, exception)) - - event_data = lz4.block.decompress( - event_data[8:], uncompressed_size=uncompressed_size) - + event_data = self._ParseDataObject(file_object, file_offset) event_string = event_data.decode('utf-8') return event_string.split('=', 1) diff --git a/plaso/parsers/systemd_journal.yaml b/plaso/parsers/systemd_journal.yaml index 9c1a108181..59d6ae5fcf 100644 --- a/plaso/parsers/systemd_journal.yaml +++ b/plaso/parsers/systemd_journal.yaml @@ -105,22 +105,6 @@ members: - name: tail_entry_monotonic data_type: uint64 --- -name: systemd_journal_object_header -type: structure -attributes: - byte_order: little-endian -members: -- name: object_type - data_type: uint8 -- name: object_flags - data_type: uint8 -- name: reserved1 - type: stream - element_data_type: byte - elements_data_size: 6 -- name: data_size - data_type: uint64 ---- name: systemd_journal_data_object type: structure attributes: @@ -146,12 +130,6 @@ members: data_type: uint64 - name: entry_array_offset data_type: uint64 -- name: number_of_entries - data_type: uint64 -- name: data - type: stream - element_data_type: byte - elements_data_size: systemd_journal_data_object.data_size - 64 --- name: systemd_journal_entry_item type: structure diff --git a/tests/parsers/systemd_journal.py b/tests/parsers/systemd_journal.py index b1281646d7..5efa6bec72 100644 --- a/tests/parsers/systemd_journal.py +++ b/tests/parsers/systemd_journal.py @@ -11,6 +11,7 @@ except ImportError: systemd_journal = None +from tests import test_lib as shared_test_lib from tests.parsers import test_lib @@ -18,6 +19,8 @@ class SystemdJournalParserTest(test_lib.ParserTestCase): """Tests for the Systemd Journal parser.""" + @shared_test_lib.skipUnlessHasTestFile([ + 'systemd', 'journal', 'system.journal']) def testParse(self): """Tests the Parse function.""" parser = systemd_journal.SystemdJournalParser() @@ -47,6 +50,8 @@ def testParse(self): expected_short_message = '{0:s}...'.format(expected_message[:77]) self._TestGetMessageStrings(event, expected_message, expected_short_message) + @shared_test_lib.skipUnlessHasTestFile([ + 'systemd', 'journal', 'system.journal.lz4']) def testParseLZ4(self): """Tests the Parse function on a journal with LZ4 compressed events.""" parser = systemd_journal.SystemdJournalParser() @@ -81,6 +86,9 @@ def testParseLZ4(self): expected_short_message = '{0:s}...'.format(expected_message[:77]) self._TestGetMessageStrings(event, expected_message, expected_short_message) + @shared_test_lib.skipUnlessHasTestFile([ + 'systemd', 'journal', + 'system@00053f9c9a4c1e0e-2e18a70e8b327fed.journalTILDE']) def testParseDirty(self): """Tests the Parse function on a 'dirty' journal file.""" storage_writer = self._CreateStorageWriter()