#3 Start to use new compiler objects for Fortran compilation.

source/fab/build_config.py

@@ -144,6 +144,10 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self._finalise_metrics(self._start_time, self._build_timer)
         self._finalise_logging()
 
+    @property
+    def tool_box(self):
+        return self._tool_box
+
     @property
     def build_output(self):
         return self.project_workspace / BUILD_OUTPUT

source/fab/newtools/compiler.py

@@ -9,6 +9,8 @@
 
 """
 
+from typing import List
+
 from fab.newtools.categories import Categories
 from fab.newtools.tool import Tool
 
@@ -17,9 +19,75 @@ class Compiler(Tool):
     '''This is the base class for any compiler.
     '''
 
-    def __init__(self, name: str, exec_name: str, category: Categories):
+    def __init__(self, name: str, exec_name: str, category: Categories,
+                 compile_flag="-c", output_flag="-o", module_folder_flag=None,
+                 omp_flag=None, syntax_only_flag=None):
         super().__init__(name, exec_name, category)
         self._version = None
+        self._compile_flag = compile_flag
+        self._output_flag = output_flag
+        self._module_folder_flag = module_folder_flag
+        self._omp_flag = omp_flag
+        self._syntax_only_flag = syntax_only_flag
+        self._module_output_path: List[str] = []
+
+    @property
+    def has_syntax_only(self):
+        return self._syntax_only_flag is not None
+
+    def set_module_output_path(self, path):
+        path = str(path)
+        self._module_output_path = path
+
+    def _remove_managed_flags(self, flags: List[str]):
+        '''Removes all flags in `flags` that will be managed by FAB.
+        This is atm only the module output path. The list will be
+        modified in-place.
+
+        :param flags: the list of flags from which to remove managed flags.
+        '''
+        i = 0
+        flag_len = len(self._module_output_path)
+        while i < len(flags):
+            flag = flags[i]
+            # E.g. "-J/tmp" and "-J /tmp" are both accepted.
+            # First check for two parameter, i.e. with space after the flag
+            if flag == self._module_folder_flag:
+                if i + 1 == len(flags):
+                    # We have a flag, but no path. Issue a warning:
+                    self.logger.warning(f"Flags '{' '. join(flags)} contain "
+                                        f"module path "
+                                        f"'{self._module_folder_flag}' but "
+                                        f"no path.")
+                    break
+                del flag[i:i+2]
+                continue
+
+            if flag[:flag_len] == self._module_output_path:
+                del flag[i]
+                continue
+            i += 1
+
+    def compile_file(self, input_file, output_file, add_flags=None,
+                     syntax_only=False):
+        # Do we need to remove compile flag or module_folder_flag from
+        # add_flags??
+        params = [input_file.fpath.name, self._compile_flag,
+                  self._output_flag, str(output_file)]
+        if syntax_only and self._syntax_only_flag:
+            params.append(self._syntax_only_flag)
+        if add_flags:
+            # Don't modify the user's list:
+            new_flags = add_flags[:]
+            self._remove_managed_flags(new_flags)
+            params += new_flags
+
+        # Append module output path
+        params.append(self._module_folder_flag)
+        params.append(self._module_output_path)
+
+        return self.run(cwd=input_file.fpath.parent,
+                        additional_parameters=params)
 
     def get_version(self):
         """
@@ -82,7 +150,10 @@ class Gfortran(Compiler):
     '''Class for GNU's gfortran compiler.
     '''
     def __init__(self):
-        super().__init__("gfortran", "gfortran", Categories.FORTRAN_COMPILER)
+        super().__init__("gfortran", "gfortran", Categories.FORTRAN_COMPILER,
+                         module_folder_flag="-J",
+                         omp_flag="-fopenmp",
+                         syntax_only_flag="-fsyntax-only")
 
 
 # ============================================================================
@@ -98,4 +169,7 @@ class Ifort(Compiler):
     '''Class for Intel's ifort compiler.
     '''
     def __init__(self):
-        super().__init__("ifort", "ifort", Categories.FORTRAN_COMPILER)
+        super().__init__("ifort", "ifort", Categories.FORTRAN_COMPILER,
+                         module_folder_flag="-module",
+                         omp_flag="-qopenmp",
+                         syntax_only_flag="-syntax-only")

source/fab/newtools/tool.py

@@ -11,7 +11,7 @@
 import logging
 from pathlib import Path
 import subprocess
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from fab.newtools.categories import Categories
 from fab.newtools.flags import Flags
@@ -49,7 +49,7 @@ def __str__(self):
         return f"{type(self).__name__} - {self._name}: {self._exec_name}"
 
     def run(self,
-            additional_parameters: Optional[Union[str, list[str]]] = None,
+            additional_parameters: Optional[Union[str, List[str]]] = None,
             env: Optional[dict[str, str]] = None,
             cwd: Optional[Union[Path, str]] = None,
             capture_output=True) -> str:

source/fab/steps/compile_fortran.py

@@ -27,7 +27,8 @@
 from fab.metrics import send_metric
 from fab.parse.fortran import AnalysedFortran
 from fab.steps import check_for_errors, run_mp, step
-from fab.tools import COMPILERS, remove_managed_flags, flags_checksum, run_command, get_tool, get_compiler_version
+from fab.tools import flags_checksum, run_command, get_tool
+from fab.newtools import Categories, Compiler
 from fab.util import CompiledFile, log_or_dot_finish, log_or_dot, Timer, by_type, \
     file_checksum
 
@@ -41,11 +42,9 @@ class MpCommonArgs(object):
     """Arguments to be passed into the multiprocessing function, alongside the filenames."""
     config: BuildConfig
     flags: FlagsConfig
-    compiler: str
-    compiler_version: str
+    compiler: Compiler
     mod_hashes: Dict[str, int]
-    two_stage_flag: Optional[str]
-    stage: Optional[int]
+    syntax_only: bool
 
 
 @step
@@ -71,43 +70,42 @@ def compile_fortran(config: BuildConfig, common_flags: Optional[List[str]] = Non
 
     """
 
-    compiler, compiler_version, flags_config = handle_compiler_args(common_flags, path_flags)
+    compiler, flags_config = handle_compiler_args(config, common_flags,
+                                                  path_flags)
+    # Set module output folder:
+    compiler.set_module_output_path(config.build_output)
 
     source_getter = source or DEFAULT_SOURCE_GETTER
-
-    # todo: move this to the known compiler flags?
-    # todo: this is a misleading name
-    two_stage_flag = None
-    if compiler == 'gfortran' and config.two_stage:
-        two_stage_flag = '-fsyntax-only'
-
     mod_hashes: Dict[str, int] = {}
 
     # get all the source to compile, for all build trees, into one big lump
     build_lists: Dict[str, List] = source_getter(config._artefact_store)
 
+    syntax_only = compiler.has_syntax_only and config.two_stage
     # build the arguments passed to the multiprocessing function
     mp_common_args = MpCommonArgs(
-        config=config, flags=flags_config, compiler=compiler, compiler_version=compiler_version,
-        mod_hashes=mod_hashes, two_stage_flag=two_stage_flag, stage=None)
+        config=config, flags=flags_config, compiler=compiler,
+        mod_hashes=mod_hashes, syntax_only=syntax_only)
 
     # compile everything in multiple passes
     compiled: Dict[Path, CompiledFile] = {}
     uncompiled: Set[AnalysedFortran] = set(sum(build_lists.values(), []))
     logger.info(f"compiling {len(uncompiled)} fortran files")
 
-    if two_stage_flag:
+    if syntax_only:
         logger.info("Starting two-stage compile: mod files, multiple passes")
-        mp_common_args.stage = 1
+    elif config.two_stage:
+        logger.info(f"Compiler {compiler.name} does not support syntax-only, "
+                    f"disabling two-stage compile.")
 
     while uncompiled:
         uncompiled = compile_pass(config=config, compiled=compiled, uncompiled=uncompiled,
                                   mp_common_args=mp_common_args, mod_hashes=mod_hashes)
     log_or_dot_finish(logger)
 
-    if two_stage_flag:
+    if syntax_only:
         logger.info("Finalising two-stage compile: object files, single pass")
-        mp_common_args.stage = 2
+        mp_common_args.syntax_only = False
 
         # a single pass should now compile all the object files in one go
         uncompiled = set(sum(build_lists.values(), []))  # todo: order by last compile duration
@@ -122,32 +120,19 @@ def compile_fortran(config: BuildConfig, common_flags: Optional[List[str]] = Non
     store_artefacts(compiled, build_lists, config._artefact_store)
 
 
-def handle_compiler_args(common_flags=None, path_flags=None):
+def handle_compiler_args(config: BuildConfig, common_flags=None,
+                         path_flags=None):
 
     # Command line tools are sometimes specified with flags attached.
-    compiler, compiler_flags = get_fortran_compiler()
-
-    compiler_version = get_compiler_version(compiler)
-    logger.info(f'fortran compiler is {compiler} {compiler_version}')
+    compiler = config.tool_box[Categories.FORTRAN_COMPILER]
+    logger.info(f'fortran compiler is {compiler} {compiler.get_version()}')
 
     # collate the flags from 1) compiler env, 2) flags env and 3) params
     env_flags = os.getenv('FFLAGS', '').split()
-    common_flags = compiler_flags + env_flags + (common_flags or [])
-
-    # Do we know this compiler? If so we can manage the flags a little, to avoid duplication or misconfiguration.
-    # todo: This has been raised for discussion - we might never want to modify incoming flags...
-    known_compiler = COMPILERS.get(os.path.basename(compiler))
-    if known_compiler:
-        common_flags = remove_managed_flags(compiler, common_flags)
-    else:
-        logger.warning(f"Unknown compiler {compiler}. Fab cannot control certain flags."
-                       "Please ensure you specify the flag `-c` equivalent flag to only compile."
-                       "Please ensure the module output folder is set to your config's build_output folder."
-                       "or please extend fab.tools.COMPILERS in your build script.")
-
+    common_flags = env_flags + (common_flags or [])
     flags_config = FlagsConfig(common_flags=common_flags, path_flags=path_flags)
 
-    return compiler, compiler_version, flags_config
+    return compiler, flags_config
 
 
 def compile_pass(config, compiled: Dict[Path, CompiledFile], uncompiled: Set[AnalysedFortran],
@@ -244,9 +229,11 @@ def process_file(arg: Tuple[AnalysedFortran, MpCommonArgs]) \
     """
     with Timer() as timer:
         analysed_file, mp_common_args = arg
+        config = mp_common_args.config
+        compiler = config.tool_box[Categories.FORTRAN_COMPILER]
+        flags = mp_common_args.flags.flags_for_path(path=analysed_file.fpath, config=config)
 
-        flags = mp_common_args.flags.flags_for_path(path=analysed_file.fpath, config=mp_common_args.config)
-        mod_combo_hash = _get_mod_combo_hash(analysed_file, mp_common_args=mp_common_args)
+        mod_combo_hash = _get_mod_combo_hash(analysed_file, compiler=compiler)
         obj_combo_hash = _get_obj_combo_hash(analysed_file, mp_common_args=mp_common_args, flags=flags)
 
         # calculate the incremental/prebuild artefact filenames
@@ -289,7 +276,10 @@ def process_file(arg: Tuple[AnalysedFortran, MpCommonArgs]) \
         artefacts = [obj_file_prebuild] + mod_file_prebuilds
 
     # todo: probably better to record both mod and obj metrics
-    metric_name = "compile fortran" + (f' stage {mp_common_args.stage}' if mp_common_args.stage else '')
+    metric_name = "compile fortran"
+    if mp_common_args.syntax_only:
+        metric_name += " syntax-only"
+
     send_metric(
         group=metric_name,
         name=str(analysed_file.fpath),
@@ -308,21 +298,21 @@ def _get_obj_combo_hash(analysed_file, mp_common_args: MpCommonArgs, flags):
             analysed_file.file_hash,
             flags_checksum(flags),
             sum(mod_deps_hashes.values()),
-            zlib.crc32(mp_common_args.compiler.encode()),
-            zlib.crc32(mp_common_args.compiler_version.encode()),
+            zlib.crc32(mp_common_args.compiler.name.encode()),
+            zlib.crc32(mp_common_args.compiler.get_version().encode()),
         ])
     except TypeError:
         raise ValueError("could not generate combo hash for object file")
     return obj_combo_hash
 
 
-def _get_mod_combo_hash(analysed_file, mp_common_args: MpCommonArgs):
+def _get_mod_combo_hash(analysed_file, compiler: Compiler):
     # get a combo hash of things which matter to the mod files we define
     try:
         mod_combo_hash = sum([
             analysed_file.file_hash,
-            zlib.crc32(mp_common_args.compiler.encode()),
-            zlib.crc32(mp_common_args.compiler_version.encode()),
+            zlib.crc32(compiler.name.encode()),
+            zlib.crc32(compiler.get_version().encode()),
         ])
     except TypeError:
         raise ValueError("could not generate combo hash for mod files")
@@ -341,30 +331,12 @@ def compile_file(analysed_file, flags, output_fpath, mp_common_args):
     output_fpath.parent.mkdir(parents=True, exist_ok=True)
 
     # tool
-    command = [mp_common_args.compiler]
-    known_compiler = COMPILERS.get(os.path.basename(mp_common_args.compiler))
-
-    # Compile flag.
-    # If it's an unknown compiler, we rely on the user config to specify this.
-    if known_compiler:
-        command.append(known_compiler.compile_flag)
-
-    # flags
-    command.extend(flags)
-    if mp_common_args.two_stage_flag and mp_common_args.stage == 1:
-        command.append(mp_common_args.two_stage_flag)
-
-    # Module folder.
-    # If it's an unknown compiler, we rely on the user config to specify this.
-    if known_compiler:
-        command.extend([known_compiler.module_folder_flag, str(mp_common_args.config.build_output)])
-
-    # files
-    command.append(analysed_file.fpath.name)
-    command.extend(['-o', str(output_fpath)])
-
-    run_command(command, cwd=analysed_file.fpath.parent)
+    config = mp_common_args.config
+    compiler = config.tool_box[Categories.FORTRAN_COMPILER]
 
+    compiler.compile_file(input_file=analysed_file, output_file=output_fpath,
+                          add_flags=flags,
+                          syntax_only=mp_common_args.syntax_only)
 
 # todo: move this