Migrated construct-based parsers to use dtfabric log2timeline#1893

dependencies.ini

@@ -75,7 +75,7 @@ version_property: __version__
 
 [dtfabric]
 dpkg_name: python-dtfabric
-minimum_version: 20170524
+minimum_version: 20180521
 rpm_name: python-dtfabric
 version_property: __version__
 

plaso/parsers/data_formats.py

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+"""Shared functionality for dtFabric-based data format parsers."""
+
+from __future__ import unicode_literals
+
+import os
+
+from dtfabric import errors as dtfabric_errors
+from dtfabric.runtime import data_maps as dtfabric_data_maps
+
+from plaso.lib import errors
+from plaso.parsers import interface
+
+
+class DataFormatParser(interface.FileObjectParser):
+  """Shared functionality for dtFabric-based data format parsers."""
+
+  def _ReadData(self, file_object, file_offset, data_size, description):
+    """Reads data.
+
+    Args:
+      file_object (file): a file-like object.
+      file_offset (int): offset of the data relative from the start of
+          the file-like object.
+      data_size (int): size of the data.
+      description (str): description of the data.
+
+    Returns:
+      bytes: byte stream containing the data.
+
+    Raises:
+      ParseError: if the structure cannot be read.
+      ValueError: if file-like object or data type map are invalid.
+    """
+    if not file_object:
+      raise ValueError('Invalid file-like object.')
+
+    file_object.seek(file_offset, os.SEEK_SET)
+
+    read_error = ''
+
+    try:
+      data = file_object.read(data_size)
+
+      if len(data) != data_size:
+        read_error = 'missing data'
+
+    except IOError as exception:
+      read_error = '{0!s}'.format(exception)
+
+    if read_error:
+      raise errors.ParseError((
+          'Unable to read {0:s} data at offset: 0x{1:08x} with error: '
+          '{2:s}').format(description, file_offset, read_error))
+
+    return data
+
+  def _ReadStructure(
+      self, file_object, file_offset, data_size, data_type_map, description):
+    """Reads a structure.
+
+    Args:
+      file_object (file): a file-like object.
+      file_offset (int): offset of the data relative from the start of
+          the file-like object.
+      data_size (int): data size of the structure.
+      data_type_map (dtfabric.DataTypeMap): data type map of the structure.
+      description (str): description of the structure.
+
+    Returns:
+      object: structure values object.
+
+    Raises:
+      ParseError: if the structure cannot be read.
+      ValueError: if file-like object or data type map are invalid.
+    """
+    data = self._ReadData(file_object, file_offset, data_size, description)
+
+    return self._ReadStructureFromByteStream(
+        data, file_offset, data_type_map, description)
+
+  def _ReadStructureWithSizeHint(
+      self, file_object, file_offset, data_type_map, description):
+    """Reads a structure using a size hint.
+
+    Args:
+      file_object (file): a file-like object.
+      file_offset (int): offset of the data relative from the start of
+          the file-like object.
+      data_type_map (dtfabric.DataTypeMap): data type map of the structure.
+      description (str): description of the structure.
+
+    Returns:
+      tuple[object, int]: structure values object and data size of
+          the structure.
+
+    Raises:
+      ParseError: if the structure cannot be read.
+      ValueError: if file-like object or data type map are invalid.
+    """
+    context = None
+    last_size_hint = 0
+    size_hint = data_type_map.GetSizeHint()
+
+    while size_hint != last_size_hint:
+      data = self._ReadData(file_object, file_offset, size_hint, description)
+
+      try:
+        context = dtfabric_data_maps.DataTypeMapContext()
+        structure_values_object = self._ReadStructureFromByteStream(
+            data, file_offset, data_type_map, description, context=context)
+        return structure_values_object, size_hint
+
+      except dtfabric_errors.ByteStreamTooSmallError:
+        pass
+
+      last_size_hint = size_hint
+      size_hint = data_type_map.GetSizeHint(context=context)
+
+    raise errors.ParseError('Unable to read {0:s}'.format(description))
+
+  def _ReadStructureFromByteStream(
+      self, byte_stream, file_offset, data_type_map, description, context=None):
+    """Reads a structure from a byte stream.
+
+    Args:
+      byte_stream (bytes): byte stream.
+      file_offset (int): offset of the data relative from the start of
+          the file-like object.
+      data_type_map (dtfabric.DataTypeMap): data type map of the structure.
+      description (str): description of the structure.
+      context (Optional[dtfabric.DataTypeMapContext]): data type map context.
+
+    Returns:
+      object: structure values object.
+
+    Raises:
+      ParseError: if the structure cannot be read.
+      ValueError: if file-like object or data type map are invalid.
+    """
+    if not byte_stream:
+      raise ValueError('Invalid byte stream.')
+
+    if not data_type_map:
+      raise ValueError('Invalid data type map.')
+
+    try:
+      return data_type_map.MapByteStream(byte_stream, context=context)
+    except dtfabric_errors.MappingError as exception:
+      raise errors.ParseError((
+          'Unable to map {0:s} data at offset: 0x{1:08x} with error: '
+          '{2!s}').format(description, file_offset, exception))

plaso/parsers/winrestore.py

@@ -5,14 +5,17 @@
 
 import os
 
-import construct
-
 from dfdatetime import filetime as dfdatetime_filetime
 from dfdatetime import semantic_time as dfdatetime_semantic_time
 
+from dtfabric import errors as dtfabric_errors
+from dtfabric.runtime import fabric as dtfabric_fabric
+
 from plaso.containers import events
 from plaso.containers import time_events
 from plaso.lib import definitions
+from plaso.lib import errors
+from plaso.parsers import data_formats
 from plaso.parsers import interface
 from plaso.parsers import manager
 
@@ -38,7 +41,7 @@ def __init__(self):
     self.sequence_number = None
 
 
-class RestorePointLogParser(interface.FileObjectParser):
+class RestorePointLogParser(data_formats.DataFormatParser):
   """A parser for Windows Restore Point (rp.log) files."""
 
   NAME = 'rplog'
@@ -47,18 +50,53 @@ class RestorePointLogParser(interface.FileObjectParser):
   FILTERS = frozenset([
       interface.FileNameFileEntryFilter('rp.log')])
 
-  _FILE_HEADER_STRUCT = construct.Struct(
-      'file_header',
-      construct.ULInt32('event_type'),
-      construct.ULInt32('restore_point_type'),
-      construct.ULInt64('sequence_number'),
-      construct.RepeatUntil(
-          lambda character, ctx: character == b'\x00\x00',
-          construct.Field('description', 2)))
+  _DATA_TYPE_FABRIC_DEFINITION_FILE = os.path.join(
+      os.path.dirname(__file__), 'winrestore.yaml')
+
+  with open(_DATA_TYPE_FABRIC_DEFINITION_FILE, 'rb') as file_object:
+    _DATA_TYPE_FABRIC_DEFINITION = file_object.read()
+
+  _DATA_TYPE_FABRIC = dtfabric_fabric.DataTypeFabric(
+      yaml_definition=_DATA_TYPE_FABRIC_DEFINITION)
+
+  _FILE_HEADER = _DATA_TYPE_FABRIC.CreateDataTypeMap(
+      'rp_log_file_header')
+
+  _FILE_FOOTER = _DATA_TYPE_FABRIC.CreateDataTypeMap(
+      'rp_log_file_footer')
+
+  _FILE_FOOTER_SIZE = _FILE_FOOTER.GetByteSize()
+
+  def _ReadFileFooter(self, file_object):
+    """Reads the file footer.
+
+    Args:
+      file_object (dfvfs.FileIO): file-like object.
+
+    Raises:
+      ParseError: if the file footer cannot be read.
+    """
+    file_offset = self._file_size - 8
+    file_footer = self._ReadStructure(
+        file_object, file_offset, self._FILE_FOOTER_SIZE, self._FILE_FOOTER,
+        'file footer')
+
+  def ReadFileObject(self, file_object):
+    """Reads a Windows Restore Point rp.log file-like object.
+
+    Args:
+      file_object (file): file-like object.
 
-  _FILE_FOOTER_STRUCT = construct.Struct(
-      'file_footer',
-      construct.ULInt64('creation_time'))
+    Raises:
+      ParseError: if the file cannot be read.
+    """
+    self._ReadFileHeader(file_object)
+
+    data_size = (self._file_size - 8) - file_object.tell()
+    data = file_object.read(data_size)
+    self._DebugPrintData('Unknown1', data)
+
+    self._ReadFileFooter(file_object)
 
   def ParseFileObject(self, parser_mediator, file_object, **unused_kwargs):
     """Parses a Windows Restore Point (rp.log) log file-like object.
@@ -67,45 +105,45 @@ def ParseFileObject(self, parser_mediator, file_object, **unused_kwargs):
       parser_mediator (ParserMediator): mediates interactions between parsers
           and other components, such as storage and dfvfs.
       file_object (dfvfs.FileIO): file-like object.
+
+    Raises:
+      UnableToParseFile: when the file cannot be parsed.
     """
-    try:
-      file_header_struct = self._FILE_HEADER_STRUCT.parse_stream(file_object)
-    except (IOError, construct.FieldError) as exception:
-      parser_mediator.ProduceExtractionError(
-          'unable to parse file header with error: {0!s}'.format(exception))
-      return
+    file_size = file_object.get_size()
 
-    file_object.seek(-8, os.SEEK_END)
+    try:
+      file_header, _ = self._ReadStructureWithSizeHint(
+          file_object, 0, self._FILE_HEADER, 'file header')
+    except (ValueError, errors.ParseError) as exception:
+      raise errors.UnableToParseFile(
+          'Unable to parse file header with error: {0!s}'.format(
+              exception))
 
     try:
-      file_footer_struct = self._FILE_FOOTER_STRUCT.parse_stream(file_object)
-    except (IOError, construct.FieldError) as exception:
+      file_footer_offset = file_size - self._FILE_FOOTER_SIZE
+      file_footer = self._ReadStructure(
+          file_object, file_footer_offset, self._FILE_FOOTER_SIZE,
+          self._FILE_FOOTER, 'file footer')
+    except (ValueError, errors.ParseError) as exception:
       parser_mediator.ProduceExtractionError(
           'unable to parse file footer with error: {0!s}'.format(exception))
       return
 
-    try:
-      description = b''.join(file_header_struct.description)
-      # The struct includes the end-of-string character that we need
-      # to strip off.
-      description = description.decode('utf16')[:-1]
-    except UnicodeDecodeError as exception:
-      description = ''
-      parser_mediator.ProduceExtractionError((
-          'unable to decode description UTF-16 stream with error: '
-          '{0:s}').format(exception))
-
-    if file_footer_struct.creation_time == 0:
+    # The description in the file header includes the end-of-string character
+    # that we need to strip off.
+    description = file_header.description.rstrip('\0')
+
+    if file_footer.creation_time == 0:
       date_time = dfdatetime_semantic_time.SemanticTime('Not set')
     else:
       date_time = dfdatetime_filetime.Filetime(
-          timestamp=file_footer_struct.creation_time)
+          timestamp=file_footer.creation_time)
 
     event_data = RestorePointEventData()
     event_data.description = description
-    event_data.restore_point_event_type = file_header_struct.event_type
-    event_data.restore_point_type = file_header_struct.restore_point_type
-    event_data.sequence_number = file_header_struct.sequence_number
+    event_data.restore_point_event_type = file_header.event_type
+    event_data.restore_point_type = file_header.restore_point_type
+    event_data.sequence_number = file_header.sequence_number
 
     event = time_events.DateTimeValuesEvent(
         date_time, definitions.TIME_DESCRIPTION_CREATION)

plaso/parsers/winrestore.yaml

@@ -0,0 +1,49 @@
+name: rp_log
+type: format
+description: Windows restore point log (rp.log) format
+urls: ["https://github.com/libyal/dtformats/blob/master/documentation/Restore%20point%20formats.asciidoc"]
+---
+name: uint32
+type: integer
+attributes:
+  format: unsigned
+  size: 4
+  units: bytes
+---
+name: uint64
+type: integer
+attributes:
+  format: unsigned
+  size: 8
+  units: bytes
+---
+name: wchar16
+type: character
+attributes:
+  size: 2
+  units: bytes
+---
+name: rp_log_file_header
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: event_type
+  data_type: uint32
+- name: restore_point_type
+  data_type: uint32
+- name: sequence_number
+  data_type: uint64
+- name: description
+  type: string
+  encoding: utf-16-le
+  element_data_type: wchar16
+  elements_terminator: "\x00\x00"
+---
+name: rp_log_file_footer
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: creation_time
+  data_type: uint64