diff --git a/loki/bulk/configure.py b/loki/bulk/configure.py index 83c24e6d0..5e9cfbb75 100644 --- a/loki/bulk/configure.py +++ b/loki/bulk/configure.py @@ -42,11 +42,15 @@ class SchedulerConfig: visualisation. These are intended for utility routines that pop up in many routines but can be ignored in terms of program control flow, like ``flush`` or ``abort``. + transformation_configs : dict + Dicts with transformation-specific options + frontend_args : dict + Dicts with file-specific frontend options """ def __init__( self, default, routines, disable=None, dimensions=None, - transformation_configs=None + transformation_configs=None, frontend_args=None ): self.default = default self.disable = as_tuple(disable) @@ -54,6 +58,7 @@ def __init__( self.routines = CaseInsensitiveDict(routines) self.transformation_configs = transformation_configs + self.frontend_args = frontend_args # Resolve the dimensions for trafo configurations for cfg in self.transformation_configs.values(): @@ -82,10 +87,11 @@ def from_dict(cls, config): name: TransformationConfig(name=name, **cfg) for name, cfg in transformation_configs.items() } + frontend_args = config.get('frontend_args', {}) return cls( default=default, routines=routines, disable=disable, dimensions=dimensions, - transformation_configs=transformation_configs + transformation_configs=transformation_configs, frontend_args=frontend_args ) @classmethod @@ -179,6 +185,39 @@ def create_item_config(self, name): item_conf.update(self.routines[key]) return item_conf + def create_frontend_args(self, path, default_args): + """ + Create bespoke ``frontend_args`` to pass to the constructor + or ``make_complete`` method for a file + + The resulting `dict` contains overwrites that have been provided + in the :attr:`frontend_args` of the config. + + Parameters + ---------- + path : str or pathlib.Path + The file path for which to create the frontend arguments. This + can be a fully-qualified path or include :any:`fnmatch`-compatible + patterns. + default_args : dict + The default options to use. Only keys that are explicitly overriden + for the file in the scheduler config are updated. + + Returns + ------- + dict + The frontend arguments, with file-specific overrides of + :data:`default_args` if specified in the Scheduler config. + """ + path = str(path).lower() + frontend_args = default_args.copy() + for key, args in (self.frontend_args or {}).items(): + pattern = key.lower() if key[0] == '/' else f'*{key}'.lower() + if fnmatch(path, pattern): + frontend_args.update(args) + return frontend_args + return frontend_args + def is_disabled(self, name): """ Check if the item with the given :data:`name` is marked as `disabled` diff --git a/loki/bulk/item.py b/loki/bulk/item.py index 0a6a585fe..aae0864fa 100644 --- a/loki/bulk/item.py +++ b/loki/bulk/item.py @@ -1094,6 +1094,8 @@ def get_or_create_file_item_from_path(self, path, config, frontend_args=None): if not frontend_args: frontend_args = {} + if config: + frontend_args = config.create_frontend_args(path, frontend_args) source = Sourcefile.from_file(path, **frontend_args) item_conf = config.create_item_config(item_name) if config else None diff --git a/loki/bulk/scheduler.py b/loki/bulk/scheduler.py index 2ff5ca7ab..828efb197 100644 --- a/loki/bulk/scheduler.py +++ b/loki/bulk/scheduler.py @@ -6,16 +6,16 @@ # nor does it submit to any jurisdiction. from collections import deque, defaultdict -from pathlib import Path from os.path import commonpath +from pathlib import Path import networkx as nx from codetiming import Timer +from loki.bulk.configure import SchedulerConfig from loki.bulk.item import ( Item, FileItem, ModuleItem, ProcedureItem, ProcedureBindingItem, InterfaceItem, TypeDefItem, ItemFactory ) -from loki.bulk.configure import SchedulerConfig from loki.frontend import FP, REGEX, RegexParserClass from loki.tools import as_tuple, CaseInsensitiveDict, flatten from loki.logging import info, perf, warning, debug @@ -265,10 +265,11 @@ def _parse_items(self): the execution plan and enriching subroutine calls. """ # Force the parsing of the routines - build_args = self.build_args.copy() - build_args['definitions'] = as_tuple(build_args['definitions']) + self.definitions + default_frontend_args = self.build_args.copy() + default_frontend_args['definitions'] = as_tuple(default_frontend_args['definitions']) + self.definitions for item in SFilter(self.sgraph.as_filegraph(self.item_factory, self.config), reverse=True): - item.source.make_complete(**build_args) + frontend_args = self.config.create_frontend_args(item.name, default_frontend_args) + item.source.make_complete(**frontend_args) @Timer(logger=info, text='[Loki::Scheduler] Enriched call tree in {:.2f}s') def _enrich(self): @@ -285,7 +286,8 @@ def _enrich(self): self.sgraph._create_item(name, item_factory=self.item_factory, config=self.config) ) for enrich_item in enrich_items: - enrich_item.source.make_complete(**self.build_args) + frontend_args = self.config.create_frontend_args(enrich_item.source.path, self.build_args) + enrich_item.source.make_complete(**frontend_args) enrich_definitions += tuple(item_.ir for item_ in enrich_items) item.ir.enrich(enrich_definitions, recurse=True) diff --git a/loki/program_unit.py b/loki/program_unit.py index ff10145a9..16f646027 100644 --- a/loki/program_unit.py +++ b/loki/program_unit.py @@ -138,6 +138,9 @@ def from_source(cls, source, definitions=None, preprocess=False, parent : :any:`Scope`, optional The parent scope this module or subroutine is nested into """ + if isinstance(frontend, str): + frontend = Frontend[frontend.upper()] + if preprocess: # Trigger CPP-preprocessing explicitly, as includes and # defines can also be used by our OMNI frontend @@ -278,6 +281,8 @@ def make_complete(self, **frontend_args): if not self._incomplete: return frontend = frontend_args.pop('frontend', Frontend.FP) + if isinstance(frontend, str): + frontend = Frontend[frontend.upper()] definitions = frontend_args.get('definitions') xmods = frontend_args.get('xmods') parser_classes = frontend_args.get('parser_classes', RegexParserClass.AllClasses) diff --git a/loki/sourcefile.py b/loki/sourcefile.py index 1b23bbe5b..f89f63e9e 100644 --- a/loki/sourcefile.py +++ b/loki/sourcefile.py @@ -15,7 +15,7 @@ from loki.backend.fgen import fgen from loki.backend.cufgen import cufgen from loki.frontend import ( - OMNI, OFP, FP, REGEX, sanitize_input, Source, read_file, preprocess_cpp, + Frontend, OMNI, OFP, FP, REGEX, sanitize_input, Source, read_file, preprocess_cpp, parse_omni_source, parse_ofp_source, parse_fparser_source, parse_omni_ast, parse_ofp_ast, parse_fparser_ast, parse_regex_source, RegexParserClass @@ -107,6 +107,8 @@ def from_file(cls, filename, definitions=None, preprocess=False, frontend : :any:`Frontend`, optional Frontend to use for producing the AST (default :any:`FP`). """ + if isinstance(frontend, str): + frontend = Frontend[frontend.upper()] # Log full parses at INFO and regex scans at PERF level log = f'[Loki::Sourcefile] Constructed from {filename}' + ' in {:.2f}s' @@ -323,6 +325,9 @@ def from_source(cls, source, definitions=None, preprocess=False, frontend : :any:`Frontend`, optional Frontend to use for producing the AST (default :any:`FP`). """ + if isinstance(frontend, str): + frontend = Frontend[frontend.upper()] + if preprocess: # Trigger CPP-preprocessing explicitly, as includes and # defines can also be used by our OMNI frontend @@ -362,6 +367,8 @@ def make_complete(self, **frontend_args): # Sanitize frontend_args frontend = frontend_args.pop('frontend', FP) + if isinstance(frontend, str): + frontend = Frontend[frontend.upper()] if frontend == REGEX: frontend_argnames = ['parser_classes'] elif frontend == OMNI: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index cb31ae8b9..07e8a12a3 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -53,17 +53,18 @@ from pathlib import Path import re from shutil import rmtree +from subprocess import CalledProcessError import pytest from conftest import available_frontends, graphviz_present from loki import ( - Scheduler, SchedulerConfig, DependencyTransformation, FP, OFP, - HAVE_FP, HAVE_OFP, REGEX, Sourcefile, FindNodes, CallStatement, + Scheduler, SchedulerConfig, DependencyTransformation, FP, OFP, OMNI, + HAVE_FP, HAVE_OFP, HAVE_OMNI, REGEX, Sourcefile, FindNodes, CallStatement, fexprgen, Transformation, BasicType, Subroutine, gettempdir, ProcedureSymbol, Item, ProcedureItem, ProcedureBindingItem, InterfaceItem, ProcedureType, DerivedType, TypeDef, Scalar, Array, FindInlineCalls, Import, flatten, as_tuple, TypeDefItem, SFilter, CaseInsensitiveDict, Comment, - ModuleWrapTransformation, Dimension + ModuleWrapTransformation, Dimension, PreprocessorDirective ) pytestmark = pytest.mark.skipif(not HAVE_FP and not HAVE_OFP, reason='Fparser and OFP not available') @@ -2536,3 +2537,198 @@ def transform_file(self, sourcefile, **kwargs): scheduler.process(transformation=MyFileTrafo()) rmtree(workdir) + + +@pytest.mark.parametrize('frontend', available_frontends()) +@pytest.mark.parametrize('frontend_args,defines,preprocess,has_cpp_directives,additional_dependencies', [ + # No preprocessing, thus all call dependencies are included + (None, None, False, [ + '#test_scheduler_frontend_args1', '#test_scheduler_frontend_args2', '#test_scheduler_frontend_args4' + ], { + '#test_scheduler_frontend_args2': ('#test_scheduler_frontend_args3',), + '#test_scheduler_frontend_args3': (), + '#test_scheduler_frontend_args4': ('#test_scheduler_frontend_args3',), + }), + # Global preprocessing setting SOME_DEFINITION, removing dependency on 3 + (None, ['SOME_DEFINITION'], True, [], {}), + # Global preprocessing with local definition for one file, re-adding a dependency on 3 + ( + {'test_scheduler_frontend_args/file3_4.F90': {'defines': ['SOME_DEFINITION','LOCAL_DEFINITION']}}, + ['SOME_DEFINITION'], + True, + [], + { + '#test_scheduler_frontend_args3': (), + '#test_scheduler_frontend_args4': ('#test_scheduler_frontend_args3',), + } + ), + # Global preprocessing with preprocessing switched off for 2 + ( + {'test_scheduler_frontend_args/file2.F90': {'preprocess': False}}, + ['SOME_DEFINITION'], + True, + ['#test_scheduler_frontend_args2'], + { + '#test_scheduler_frontend_args2': ('#test_scheduler_frontend_args3',), + '#test_scheduler_frontend_args3': (), + } + ), + # No preprocessing except for 2 + ( + {'test_scheduler_frontend_args/file2.F90': {'preprocess': True, 'defines': ['SOME_DEFINITION']}}, + None, + False, + ['#test_scheduler_frontend_args1', '#test_scheduler_frontend_args4'], + { + '#test_scheduler_frontend_args3': (), + '#test_scheduler_frontend_args4': ('#test_scheduler_frontend_args3',), + } + ), +]) +def test_scheduler_frontend_args(frontend, frontend_args, defines, preprocess, + has_cpp_directives, additional_dependencies, config): + """ + Test overwriting frontend options via Scheduler config + """ + + fcode1 = """ +subroutine test_scheduler_frontend_args1 + implicit none +#ifdef SOME_DEFINITION + call test_scheduler_frontend_args2 +#endif +end subroutine test_scheduler_frontend_args1 + """.strip() + + fcode2 = """ +subroutine test_scheduler_frontend_args2 + implicit none +#ifndef SOME_DEFINITION + call test_scheduler_frontend_args3 +#endif + call test_scheduler_frontend_args4 +end subroutine test_scheduler_frontend_args2 + """.strip() + + fcode3_4 = """ +subroutine test_scheduler_frontend_args3 +implicit none +end subroutine test_scheduler_frontend_args3 + +subroutine test_scheduler_frontend_args4 +implicit none +#ifdef LOCAL_DEFINITION + call test_scheduler_frontend_args3 +#endif +end subroutine test_scheduler_frontend_args4 + """.strip() + + workdir = gettempdir()/'test_scheduler_frontend_args' + if workdir.exists(): + rmtree(workdir) + workdir.mkdir() + (workdir/'file1.F90').write_text(fcode1) + (workdir/'file2.F90').write_text(fcode2) + (workdir/'file3_4.F90').write_text(fcode3_4) + + expected_dependencies = { + '#test_scheduler_frontend_args1': ('#test_scheduler_frontend_args2',), + '#test_scheduler_frontend_args2': ('#test_scheduler_frontend_args4',), + '#test_scheduler_frontend_args4': (), + } + + for key, value in additional_dependencies.items(): + expected_dependencies[key] = expected_dependencies.get(key, ()) + value + + config['frontend_args'] = frontend_args + + scheduler = Scheduler( + paths=[workdir], config=config, seed_routines=['test_scheduler_frontend_args1'], + frontend=frontend, defines=defines, preprocess=preprocess, xmods=[workdir] + ) + + assert set(scheduler.items) == set(expected_dependencies) + assert set(scheduler.dependencies) == { + (a, b) for a, deps in expected_dependencies.items() for b in deps + } + + for item in scheduler.items: + cpp_directives = FindNodes(PreprocessorDirective).visit(item.ir.ir) + assert bool(cpp_directives) == (item in has_cpp_directives and frontend != OMNI) + # NB: OMNI always does preprocessing, therefore we won't find the CPP directives + # after the full parse + + rmtree(workdir) + + +@pytest.mark.skipif(not (HAVE_OMNI and HAVE_FP), reason="OMNI or FP not available") +def test_scheduler_frontend_overwrite(config): + """ + Test the use of a different frontend via Scheduler config + """ + fcode_header = """ +module test_scheduler_frontend_overwrite_header + implicit none + type some_type + ! We have a comment + real, dimension(:,:), pointer :: arr + end type some_type +end module test_scheduler_frontend_overwrite_header + """.strip() + fcode_kernel = """ +subroutine test_scheduler_frontend_overwrite_kernel + use test_scheduler_frontend_overwrite_header, only: some_type + implicit none + type(some_type) :: var +end subroutine test_scheduler_frontend_overwrite_kernel + """.strip() + + workdir = gettempdir()/'test_scheduler_frontend_overwrite' + if workdir.exists(): + rmtree(workdir) + workdir.mkdir() + (workdir/'test_scheduler_frontend_overwrite_header.F90').write_text(fcode_header) + (workdir/'test_scheduler_frontend_overwrite_kernel.F90').write_text(fcode_kernel) + + # Make sure that OMNI cannot parse the header file + with pytest.raises(CalledProcessError): + Sourcefile.from_source(fcode_header, frontend=OMNI, xmods=[workdir]) + + # ...and that the problem exists also during Scheduler traversal + with pytest.raises(CalledProcessError): + Scheduler( + paths=[workdir], config=config, seed_routines=['test_scheduler_frontend_overwrite_kernel'], + frontend=OMNI, xmods=[workdir] + ) + + # Strip the comment from the header file and parse again to generate an xmod + fcode_header_lines = fcode_header.split('\n') + Sourcefile.from_source('\n'.join(fcode_header_lines[:3] + fcode_header_lines[4:]), frontend=OMNI, xmods=[workdir]) + + # Setup the config with the frontend overwrite + config['frontend_args'] = { + 'test_scheduler_frontend_overwrite_header.F90': {'frontend': 'FP'} + } + + # ...and now it works fine + scheduler = Scheduler( + paths=[workdir], config=config, seed_routines=['test_scheduler_frontend_overwrite_kernel'], + frontend=OMNI, xmods=[workdir] + ) + + assert set(scheduler.items) == { + '#test_scheduler_frontend_overwrite_kernel', 'test_scheduler_frontend_overwrite_header', + 'test_scheduler_frontend_overwrite_header#some_type' + } + + assert set(scheduler.dependencies) == { + ('#test_scheduler_frontend_overwrite_kernel', 'test_scheduler_frontend_overwrite_header'), + ('#test_scheduler_frontend_overwrite_kernel', 'test_scheduler_frontend_overwrite_header#some_type') + } + + # ...and the derived type has it's comment + comments = FindNodes(Comment).visit(scheduler['test_scheduler_frontend_overwrite_header#some_type'].ir.body) + assert len(comments) == 1 + assert comments[0].text == '! We have a comment' + + rmtree(workdir) diff --git a/transformations/tests/test_cloudsc.py b/transformations/tests/test_cloudsc.py index f05068996..4632e5376 100644 --- a/transformations/tests/test_cloudsc.py +++ b/transformations/tests/test_cloudsc.py @@ -49,10 +49,7 @@ def fixture_bundle_create(here, local_loki_bundle): @pytest.mark.usefixtures('bundle_create') @pytest.mark.parametrize('frontend', available_frontends( xfail=[(OFP, 'Lack of elemental support makes C-transpilation impossible')], - skip=[(OMNI, 'OMNI needs FParser for parsing headers')] if True or not HAVE_FP else None # pylint: disable=condition-evals-to-constant - # NB: OMNI has been temporarily disabled until we can provide config-file override of the - # frontend for individual files. This is required for OMNI to parse header files - # with comments in derived types + skip=[(OMNI, 'OMNI needs FParser for parsing headers')] if not HAVE_FP else None )) def test_cloudsc(here, frontend): build_cmd = [