Refactor: move basic parsing into BaseParser class

Large refactoring of the basic parsing features to remove code duplication and
rely on regular expressions via `re` instead of iterations over lines. The main
changes are:

* The `Parser` class is renamed to `BaseParser` class to avoid confusion with
  the `aiida.core` `Parser` class.
* The `BaseParser.parse_stdout_from_retrieved` method can be used by all
  subclasses to obtain the `stdout`, basic parsed data and logs from retrieving
  and parsing the `stdout`.
* The error and warning messages are defined as class attributes. Base error
  messages are defined on the `BaseParser` class, which can be extended on the
  subclasses via the `class_[error,warning]_map` attributes.
* A `BaseParser.check_base_errors()` method is added, which checks for basic
  errors that indicate the `stdout` cannot be parsed for more data.
* Shared exit codes are moved to the `NamelistsCalculation` spec from that of
  the subclasses.
* Several changes in the logic of returning exit codes and emitting the logs.

Some of the discussions on these topics have not been fully concluded, and will
be revisited when integrating the `BaseParser` into the `PwParser`, see:

  #953

docs/source/conf.py

@@ -84,7 +84,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-# language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -151,6 +151,10 @@
     (r'.*', r'Tuple.*'),
 ]
 nitpick_ignore = [
+    ('py:class', 'AttributeDict'),
+    ('py:class', 'ExitCode'),
+    ('py:class', 'StructureData'),
+    ('py:class', 'PseudoPotentialFamily'),
     ('py:exc', 'ArithmeticError'),
     ('py:exc', 'AssertionError'),
     ('py:exc', 'AttributeError'),
@@ -209,6 +213,4 @@
     ('py:obj', 'Mapping'),
     ('py:obj', 'qe_tools.parsers.CpInputFile'),
     ('py:obj', 'qe_tools.parsers.PwInputFile'),
-    ('py:class', 'StructureData'),
-    ('py:class', 'PseudoPotentialFamily'),
 ]

src/aiida_quantumespresso/calculations/__init__.py

@@ -143,6 +143,22 @@ def define(cls, spec):
         )
         spec.inputs.validator = cls.validate_inputs
 
+        spec.exit_code(
+            302,
+            'ERROR_OUTPUT_STDOUT_MISSING',
+            message='The retrieved folder did not contain the required stdout output file.'
+        )
+        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ', message='The stdout output file could not be read.')
+        spec.exit_code(311, 'ERROR_OUTPUT_STDOUT_PARSE', message='The stdout output file could not be parsed.')
+        spec.exit_code(
+            312,
+            'ERROR_OUTPUT_STDOUT_INCOMPLETE',
+            message='The stdout output file was incomplete probably because the calculation got interrupted.'
+        )
+        spec.exit_code(
+            400, 'ERROR_OUT_OF_WALLTIME', message='The calculation stopped prematurely because it ran out of walltime.'
+        )
+
     @classmethod
     def validate_inputs(cls, value, port_namespace):
         """Validate the entire inputs namespace."""

src/aiida_quantumespresso/calculations/cp.py

@@ -131,12 +131,6 @@ def define(cls, spec):
             message='The required XML file is not present in the retrieved folder.')
         spec.exit_code(304, 'ERROR_OUTPUT_XML_MULTIPLE',
             message='The retrieved folder contains multiple XML files.')
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(311, 'ERROR_OUTPUT_STDOUT_PARSE',
-            message='The output file contains invalid output.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(320, 'ERROR_OUTPUT_XML_READ',
             message='The required XML file could not be read.')
         spec.exit_code(330, 'ERROR_READING_POS_FILE',

src/aiida_quantumespresso/calculations/dos.py

@@ -27,10 +27,6 @@ def define(cls, spec):
         spec.output('output_parameters', valid_type=orm.Dict)
         spec.output('output_dos', valid_type=orm.XyData)
         spec.default_output_node = 'output_parameters'
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(330, 'ERROR_READING_DOS_FILE',
             message='The dos file could not be read from the retrieved folder.')
         # yapf: enable

src/aiida_quantumespresso/calculations/matdyn.py

@@ -37,10 +37,6 @@ def define(cls, spec):
         spec.output('output_phonon_bands', valid_type=orm.BandsData)
         spec.default_output_node = 'output_parameters'
 
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(330, 'ERROR_OUTPUT_FREQUENCIES',
             message='The output frequencies file could not be read from the retrieved folder.')
         spec.exit_code(410, 'ERROR_OUTPUT_KPOINTS_MISSING',

src/aiida_quantumespresso/calculations/namelists.py

@@ -58,6 +58,14 @@ def define(cls, spec):
             help='Use an additional node for special settings')
         spec.input('parent_folder', valid_type=(RemoteData, FolderData, SinglefileData), required=False,
             help='Use a local or remote folder as parent folder (for restarts and similar)')
+        spec.exit_code(302, 'ERROR_OUTPUT_STDOUT_MISSING',
+            message='The retrieved folder did not contain the required stdout output file.')
+        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
+            message='An exception was raised while reading the `stdout` file: {exception}')
+        spec.exit_code(311, 'ERROR_OUTPUT_STDOUT_PARSE',
+            message='An exception was raised while parsing the `stdout` file: {exception}')
+        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
+            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         # yapf: enable
 
     @classmethod

src/aiida_quantumespresso/calculations/neb.py

@@ -78,12 +78,6 @@ def define(cls, spec):
         spec.default_output_node = 'output_parameters'
         spec.exit_code(303, 'ERROR_MISSING_XML_FILE',
             message='The required XML file is not present in the retrieved folder.')
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(311, 'ERROR_OUTPUT_STDOUT_PARSE',
-            message='The output file contains invalid output.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(320, 'ERROR_OUTPUT_XML_READ',
             message='The XML output file could not be read.')
         spec.exit_code(321, 'ERROR_OUTPUT_XML_PARSE',

src/aiida_quantumespresso/calculations/open_grid.py

@@ -28,13 +28,7 @@ def define(cls, spec):
 
         spec.exit_code(300, 'ERROR_NO_RETRIEVED_FOLDER',
             message='The retrieved folder data node could not be accessed.')
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(311, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(312, 'ERROR_INCOMPATIBLE_FFT_GRID',
             message='Found rotation or fractional translation not compatible with FFT grid.')
-        spec.exit_code(340, 'ERROR_GENERIC_QE_ERROR',
-            message='Encountered a generic error message.')
         spec.exit_code(350, 'ERROR_OUTPUT_KPOINTS_MISMATCH',
             message='Mismatch between kmesh dimensions and number of kpoints.')

src/aiida_quantumespresso/calculations/pp.py

@@ -98,6 +98,8 @@ def define(cls, spec):
             message='The parent folder did not contain the required XML output file.')
         spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
             message='The stdout output file could not be read.')
+        spec.exit_code(311, 'ERROR_OUTPUT_STDOUT_PARSE',
+            message='The stdout output file could not be parsed.')
         spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
             message='The stdout output file was incomplete.')
         spec.exit_code(340, 'ERROR_OUT_OF_WALLTIME_INTERRUPTED',

src/aiida_quantumespresso/calculations/projwfc.py

@@ -62,10 +62,6 @@ def define(cls, spec):
             message='The retrieved temporary folder could not be accessed.')
         spec.exit_code(303, 'ERROR_OUTPUT_XML_MISSING',
             message='The retrieved folder did not contain the required XML file.')
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(320, 'ERROR_OUTPUT_XML_READ',
             message='The XML output file could not be read.')
         spec.exit_code(321, 'ERROR_OUTPUT_XML_PARSE',

src/aiida_quantumespresso/calculations/pw.py

@@ -87,20 +87,12 @@ def define(cls, spec):
         # Unrecoverable errors: required retrieved files could not be read, parsed or are otherwise incomplete
         spec.exit_code(301, 'ERROR_NO_RETRIEVED_TEMPORARY_FOLDER',
             message='The retrieved temporary folder could not be accessed.')
-        spec.exit_code(302, 'ERROR_OUTPUT_STDOUT_MISSING',
-            message='The retrieved folder did not contain the required stdout output file.')
         spec.exit_code(303, 'ERROR_OUTPUT_XML_MISSING',
             message='The retrieved folder did not contain the required XML file.')
         spec.exit_code(304, 'ERROR_OUTPUT_XML_MULTIPLE',
             message='The retrieved folder contained multiple XML files.')
         spec.exit_code(305, 'ERROR_OUTPUT_FILES',
             message='Both the stdout and XML output files could not be read or parsed.')
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(311, 'ERROR_OUTPUT_STDOUT_PARSE',
-            message='The stdout output file could not be parsed.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(320, 'ERROR_OUTPUT_XML_READ',
             message='The XML output file could not be read.')
         spec.exit_code(321, 'ERROR_OUTPUT_XML_PARSE',
@@ -116,8 +108,6 @@ def define(cls, spec):
             message='The code failed in finding a valid reciprocal lattice vector.')
 
         # Significant errors but calculation can be used to restart
-        spec.exit_code(400, 'ERROR_OUT_OF_WALLTIME',
-            message='The calculation stopped prematurely because it ran out of walltime.')
         spec.exit_code(410, 'ERROR_ELECTRONIC_CONVERGENCE_NOT_REACHED',
             message='The electronic minimization cycle did not reach self-consistency.')
 

src/aiida_quantumespresso/calculations/pw2gw.py

@@ -38,16 +38,8 @@ def define(cls, spec):
         spec.output('eps', valid_type=orm.ArrayData,
             help='The `eps` output node containing 5 arrays `energy`, `epsX`, `epsY`, `epsZ`, `epsTOT`')
 
-        spec.exit_code(302, 'ERROR_OUTPUT_STDOUT_MISSING',
-            message='The retrieved folder did not contain the required stdout output file.')
         spec.exit_code(305, 'ERROR_OUTPUT_FILES',
             message='The eps*.dat output files could not be read or parsed.')
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(311, 'ERROR_OUTPUT_STDOUT_PARSE',
-            message='The stdout output file could not be parsed.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(330, 'ERROR_OUTPUT_FILES_INVALID_FORMAT',
             message='The eps*.dat output files do not have the expected shape (N, 2).')
         spec.exit_code(331, 'ERROR_OUTPUT_FILES_ENERGY_MISMATCH',

src/aiida_quantumespresso/calculations/pw2wannier90.py

@@ -31,10 +31,6 @@ def define(cls, spec):
                    help='The output folder of a pw.x calculation')
         spec.output('output_parameters', valid_type=Dict)
         spec.default_output_node = 'output_parameters'
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(340, 'ERROR_GENERIC_QE_ERROR',
             message='Encountered a generic error message')
         spec.exit_code(350, 'ERROR_UNEXPECTED_PARSER_EXCEPTION',

src/aiida_quantumespresso/calculations/q2r.py

@@ -32,10 +32,6 @@ def define(cls, spec):
         super().define(spec)
         spec.input('parent_folder', valid_type=(orm.RemoteData, orm.FolderData), required=True)
         spec.output('force_constants', valid_type=ForceConstantsData)
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ',
-            message='The stdout output file could not be read.')
-        spec.exit_code(312, 'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.')
         spec.exit_code(330, 'ERROR_READING_FORCE_CONSTANTS_FILE',
             message='The force constants file could not be read.')
         # yapf: enable

src/aiida_quantumespresso/calculations/xspectra.py

@@ -61,12 +61,6 @@ def define(cls, spec):
         spec.output('spectra', valid_type=XyData)
         spec.default_output_node = 'output_parameters'
 
-        spec.exit_code(310, 'ERROR_OUTPUT_STDOUT_READ', message='The stdout output file could not be read.')
-        spec.exit_code(
-            312,
-            'ERROR_OUTPUT_STDOUT_INCOMPLETE',
-            message='The stdout output file was incomplete probably because the calculation got interrupted.'
-        )
         spec.exit_code(
             313,
             'ERROR_OUTPUT_ABSORBING_SPECIES_WRONG',