Merge pull request #260 from ecmwf-ifs/naml-transformation-pipeline

Introducing the Pipeline class

loki/batch/scheduler.py

@@ -18,7 +18,7 @@
 )
 from loki.frontend import FP, REGEX, RegexParserClass
 from loki.tools import as_tuple, CaseInsensitiveDict, flatten
-from loki.logging import info, perf, warning, debug
+from loki.logging import info, perf, warning, debug, error
 
 
 __all__ = ['Scheduler']
@@ -377,6 +377,46 @@ def rekey_item_cache(self):
         )
 
     def process(self, transformation):
+        """
+        Process all :attr:`items` in the scheduler's graph with either
+        a :any:`Pipeline` or a single :any:`Transformation`.
+
+        A single :any:`Transformation` pass invokes
+        :meth:`process_transformation` individually, while a
+        :any:`Pipeline` will apply each contained transformation in
+        turn over the full dependency graph of the scheduler.
+
+        Parameters
+        ----------
+        transformation : :any:`Transformation` or :any:`Pipeline`
+            The transformation or transformation pipeline to apply
+        """
+        from loki.transform import Transformation, Pipeline  # pylint: disable=import-outside-toplevel
+
+        if isinstance(transformation, Transformation):
+            self.process_transformation(transformation=transformation)
+
+        elif isinstance(transformation, Pipeline):
+            self.process_pipeline(pipeline=transformation)
+
+        else:
+            error('[Loki::Scheduler] Batch processing requires Transformation or Pipeline object')
+            raise RuntimeError('[Loki] Could not batch process {transformation_or_pipeline}')
+
+    def process_pipeline(self, pipeline):
+        """
+        Process a given :any:`Pipeline` by applying its assocaited
+        transformations in turn.
+
+        Parameters
+        ----------
+        transformation : :any:`Pipeline`
+            The transformation pipeline to apply
+        """
+        for transformation in pipeline.transformations:
+            self.process_transformation(transformation)
+
+    def process_transformation(self, transformation):
         """
         Process all :attr:`items` in the scheduler's graph
 
@@ -396,6 +436,11 @@ def process(self, transformation):
         to ``True``. This uses the :attr:`filegraph` to process the dependency tree.
         If combined with a :any:`Transformation.item_filter`, only source files with
         at least one object corresponding to an item of that type are processed.
+
+        Parameters
+        ----------
+        transformation : :any:`Transformation`
+            The transformation to apply over the dependency tree
         """
         def _get_definition_items(_item, sgraph_items):
             # For backward-compatibility with the DependencyTransform and LinterTransformation

loki/transform/__init__.py

@@ -21,3 +21,4 @@
 from loki.transform.transform_extract_contained_procedures import * # noqa
 from loki.transform.transform_dead_code import * # noqa
 from loki.transform.transform_sanitise import * # noqa
+from loki.transform.pipeline import * # noqa

loki/transform/dependency_transform.py

@@ -8,7 +8,6 @@
 from pathlib import Path
 
 from loki.backend import fgen
-from loki.batch import SchedulerConfig
 from loki.expression import Variable, FindInlineCalls, SubstituteExpressions
 from loki.ir import (
     CallStatement, Import, Section, Interface, FindNodes, Transformer
@@ -250,6 +249,8 @@ def rename_calls(self, routine, targets=None, item=None):
             Optional list of subroutine names for which to modify the corresponding
             calls. If not provided, all calls are updated
         """
+        from loki.batch import SchedulerConfig  # pylint: disable=import-outside-toplevel,cyclic-import
+
         def _update_item(orig_name, new_name):
             # Update the ignore property if necessary
             if item and (matched_keys := SchedulerConfig.match_item_keys(orig_name, item.ignore)):
@@ -468,6 +469,8 @@ def update_imports(self, source, imports, **kwargs):
         """
         Update imports of wrapped subroutines.
         """
+        from loki.batch import SchedulerConfig  # pylint: disable=import-outside-toplevel,cyclic-import
+
         targets = tuple(str(t).lower() for t in as_tuple(kwargs.get('targets')))
         if self.replace_ignore_items and (item := kwargs.get('item')):
             targets += tuple(str(i).lower() for i in item.ignore)

loki/transform/pipeline.py

@@ -0,0 +1,74 @@
+# (C) Copyright 2018- ECMWF.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from inspect import signature, Parameter
+
+
+class Pipeline:
+    """
+    A transformation pipeline that combines multiple :any:`Transformation`
+    passes and allows to apply them in unison.
+
+    The associated :any:`Transformation` objects are constructed from keyword
+    arguments in the constructor, so shared keywords get same initial value.
+
+    Attributes
+    ----------
+    transformations : list of :any:`Transformation`
+        The list of transformations applied to a source in this pipeline
+
+    Parameters
+    ----------
+    classes : tuple of types
+        A tuple of types from which to instantiate :any:`Transformation` objects.
+    *args : optional
+        Positional arguments that are passed on to the constructors of
+        all transformations
+    **kwargs : optional
+        Keyword arguments that are matched to the constructor
+        signature of the transformations.
+    """
+
+    def __init__(self, *args, classes=None, **kwargs):
+        self.transformations = []
+        for cls in classes:
+
+            # Get all relevant constructor parameters from teh MRO,
+            # but exclude catch-all keyword args, like ``**kwargs``
+            t_parameters = {
+                k: v for c in cls.__mro__ for k, v in signature(c).parameters.items()
+                if not v.kind == Parameter.VAR_KEYWORD
+            }
+            # Filter kwargs for this transformation class specifically
+            t_kwargs = {k: v for k, v in kwargs.items() if k in t_parameters}
+
+            # We need to apply our own default, if we are to honour inheritance
+            t_kwargs.update({
+                k: param.default for k, param in t_parameters.items()
+                if k not in t_kwargs and param.default is not None
+            })
+
+            # Then instantiate with the default *args and the derived **t_kwargs
+            self.transformations.append(cls(*args, **t_kwargs))
+
+    def apply(self, source, **kwargs):
+        """
+        Apply each associated :any:`Transformation` to :data:`source`
+
+        It dispatches to the respective :meth:`apply` of each
+        :any:`Transformation` in the order specified in the constructor.
+
+        Parameters
+        ----------
+        source : :any:`Sourcefile` or :any:`Module` or :any:`Subroutine`
+            The source item to transform.
+        **kwargs : optional
+            Keyword arguments that are passed on to the methods defining the
+            actual transformation.
+        """
+        for trafo in self.transformations:
+            trafo.apply(source, **kwargs)

loki/transform/transform_hoist_variables.py

@@ -101,12 +101,6 @@ class HoistVariablesAnalysis(Transformation):
     Traverses all subroutines to find the variables to be hoisted.
     Create a derived class and override :func:`find_variables<HoistVariablesAnalysis.find_variables>`
     to define which variables to be hoisted.
-
-    Parameters
-    ----------
-    key : str
-        Access identifier/key for the ``item.trafo_data`` dictionary. Only necessary to provide if several of
-        these transformations are carried out in succession.
     """
 
     _key = 'HoistVariablesTransformation'
@@ -116,10 +110,6 @@ class HoistVariablesAnalysis(Transformation):
 
     process_ignored_items = True
 
-    def __init__(self, key=None):
-        if key is not None:
-            self._key = key
-
     def transform_subroutine(self, routine, **kwargs):
         """
         Analysis applied to :any:`Subroutine` item.
@@ -197,18 +187,13 @@ class HoistVariablesTransformation(Transformation):
 
     Parameters
     ----------
-    key : str
-        Access identifier/key for the ``item.trafo_data`` dictionary. Only necessary to provide if several of
-        these transformations are carried out in succession.
     as_kwarguments : boolean
         Whether to pass the hoisted arguments as `args` or `kwargs`.
     """
 
     _key = 'HoistVariablesTransformation'
 
-    def __init__(self, key=None, as_kwarguments=False):
-        if key is not None:
-            self._key = key
+    def __init__(self, as_kwarguments=False):
         self.as_kwarguments = as_kwarguments
 
     def transform_subroutine(self, routine, **kwargs):
@@ -371,19 +356,16 @@ class HoistTemporaryArraysAnalysis(HoistVariablesAnalysis):
 
     Parameters
     ----------
-    key : str, optional
-        Access identifier/key for the ``item.trafo_data`` dictionary. Only necessary to provide if several of
-        these transformations are carried out in succession.
     dim_vars: tuple of str, optional
-        Variables to be within the dimensions of the arrays to be hoisted. If not provided, no checks will be done
-        for the array dimensions.
+        Variables to be within the dimensions of the arrays to be
+        hoisted. If not provided, no checks will be done for the array
+        dimensions.
     """
 
     # Apply in reverse order to recursively find all variables to be hoisted.
     reverse_traversal = True
 
-    def __init__(self, key=None, dim_vars=None, **kwargs):
-        super().__init__(key=key, **kwargs)
+    def __init__(self, dim_vars=None):
         self.dim_vars = dim_vars
         if self.dim_vars is not None:
             assert is_iterable(self.dim_vars)
@@ -414,17 +396,8 @@ class HoistTemporaryArraysTransformationAllocatable(HoistVariablesTransformation
     functionality/transformation, to hoist temporary arrays and make
     them ``allocatable``, including the actual *allocation* and
     *de-allocation*.
-
-    Parameters
-    ----------
-    key : str, optional
-        Access identifier/key for the ``item.trafo_data`` dictionary. Only necessary to provide if several of
-        these transformations are carried out in succession.
     """
 
-    def __init__(self, key=None, **kwargs):
-        super().__init__(key=key, **kwargs)
-
     def driver_variable_declaration(self, routine, variables):
         """
         Declares hoisted arrays as ``allocatable``, including *allocation* and *de-allocation*.