Merge pull request #408 from ecmwf-ifs/nabr-update-omni-compiler

Add option to install "plain OMNI" to install script and upgrade Github actions runners
ecmwf-ifs · Oct 17, 2024 · f3b7327 · f3b7327
2 parents d90b5a7 + 04dd86b
commit f3b7327
Show file tree

Hide file tree

Showing 13 changed files with 219 additions and 56 deletions.
diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml
@@ -17,7 +17,7 @@ jobs:
   regression_tests:
     name: Python ${{ matrix.python-version }}
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false  # false: try to complete all jobs
       matrix:
@@ -66,7 +66,9 @@ jobs:
 
       - name: Install Loki
         run: |
-          ./install --with-claw --with-ofp --with-tests --without-dace
+          export JAVA_HOME=${JAVA_HOME_11_X64}
+          ./install --with-omni --with-ofp --with-tests --without-dace
+          echo "export JAVA_HOME=${JAVA_HOME_11_X64}" >> loki-activate
 
       - name: Install up-to-date CMake
         run: |
@@ -81,7 +83,6 @@ jobs:
       - name: Run CLOUDSC and ECWAM regression tests
         env:
           CLOUDSC_DIR: ${{ github.workspace }}/cloudsc
-          CLOUDSC_ARCH: ${{ github.workspace }}/cloudsc/arch/github/ubuntu/gnu/9.4.0
           ECWAM_DIR: ${{ github.workspace }}/ecwam
           OMP_STACKSIZE: 4G
         run: |

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -29,16 +29,16 @@ jobs:
 
         include:
           - name: linux python-3.8
-            os: ubuntu-20.04
+            os: ubuntu-24.04
             python-version: '3.8'
           - name: linux python-3.9
-            os: ubuntu-20.04
+            os: ubuntu-24.04
             python-version: '3.9'
           - name: linux python-3.10
-            os: ubuntu-20.04
+            os: ubuntu-24.04
             python-version: '3.10'
           - name: linux python-3.11
-            os: ubuntu-20.04
+            os: ubuntu-24.04
             python-version: '3.11'
           - name: macos python-3.11
             os: macos-14
@@ -84,7 +84,9 @@ jobs:
             echo "export F90=gfortran-13" >> loki-activate
             echo "export LD=gfortran-13" >> loki-activate
           else
-            ./install --with-claw --with-ofp --with-examples --with-tests --with-dace
+            export JAVA_HOME=${JAVA_HOME_11_X64}
+            ./install --with-omni --with-ofp --with-examples --with-tests --with-dace
+            echo "export JAVA_HOME=${JAVA_HOME_11_X64}" >> loki-activate
           fi
 
       - name: Install up-to-date CMake

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -72,7 +72,13 @@ ecbuild_add_option(
 ecbuild_add_option(
     FEATURE CLAW
     DEFAULT OFF
-    DESCRIPTION "Build OMNI Compiler and CLAW Compiler"
+    DESCRIPTION "Build OMNI compiler and CLAW compiler"
+)
+ecbuild_add_option(
+    FEATURE OMNI
+    DEFAULT OFF
+    DESCRIPTION "Build OMNI compiler as Loki frontend"
+    CONDITION NOT HAVE_CLAW
 )
 
 include( loki_transform )
@@ -98,6 +104,11 @@ if( NOT HAVE_NO_INSTALL )
         install_claw_compiler( mlange-dev )
     endif()
 
+    if( HAVE_OMNI )
+        include( omni_compiler )
+        install_omni_compiler( master )
+    endif()
+
     # Setup Python virtual environment
     include( python_venv )
     set( loki_VENV_PATH ${CMAKE_CURRENT_BINARY_DIR}/loki_env )
@@ -111,6 +122,9 @@ if( NOT HAVE_NO_INSTALL )
         if( HAVE_CLAW )
             set( _TEST_SELECTOR "not ofp" )
             set( _TEST_PATH "${CLAW_DIR}/bin:$ENV{PATH}" )
+        elseif( HAVE_OMNI )
+            set( _TEST_SELECTOR "not ofp" )
+            set( _TEST_PATH "${OMNI_DIR}/bin:$ENV{PATH}" )
         else()
             set( _TEST_SELECTOR "not ofp and not omni" )
             set( _TEST_PATH "$ENV{PATH}" )

diff --git a/INSTALL.md b/INSTALL.md
@@ -108,6 +108,7 @@ Available options:
   --with[out]-jdk              Install JDK instead of using system version (default: use system version)
   --with[out]-ant              Install ant instead of using system version (default: use system version)
   --with[out]-claw             Install CLAW and OMNI Compiler (default: disabled)
+  --with[out]-omni             Install OMNI Compiler; cannot be combined with --with-claw (default: disabled)
   --with[out]-ofp              Install Open Fortran Parser (default: disabled)
   --with[out]-dace             Install DaCe (default: enabled)
   --with[out]-tests            Install dependencies to run tests (default: enabled)
@@ -128,19 +129,19 @@ to bring up the virtual environment and set paths for the external dependencies.
 The default command on ECMWF's Atos HPC facility for a full stack installation is
 
 ```bash
-./install --hpc2020 --with-ant --with-claw --with-ofp
+./install --hpc2020 --with-ant --with-omni --with-ofp
 ```
 
 On standard Linux hosts with up-to-date JDK and ant, it is as easy as
 
 ```bash
-./install --with-claw --with-ofp
+./install --with-omni --with-ofp
 ```
 
 To update the installation (e.g., to add JDK), the existing virtual environment can be provided, e.g.,
 
 ```bash
-./install --with-claw --with-jdk --with-ant --use-venv=loki_env --with-ofp
+./install --with-omni --with-jdk --with-ant --use-venv=loki_env --with-ofp
 ```
 
 ### Installation using CMake/ecbuild
@@ -161,7 +162,10 @@ The following options are available and can be enabled/disabled by providing `-D
   elsewhere and only the CMake integration is required
 - `EDITABLE` (default: `OFF`): Install Loki in editable mode, i.e. without
   copying any files
-- `CLAW` (default: `OFF`): Install the CLAW and OMNI Compiler as well as its
+- `OMNI` (default: `OFF`): Install the OMNI compiler as well as its
+  Java dependencies as required. Note that this is an experimental setup and comes
+  with no support or guarantees.
+- `CLAW` (default: `OFF`): Install the CLAW with the included OMNI Compiler and their
   Java dependencies as required. Note that this is an experimental setup and comes
   with no support or guarantees.
 
@@ -293,18 +297,29 @@ pip install -e ./lint_rules
 popd
 ```
 
-### 4.  Install CLAW with OMNI compiler -- optional
+### 4.  Install OMNI compiler -- optional
+
+#### Option a: install latest xcodeml-tools
+
+```bash
+git clone --recursive --single-branch https://github.com/omni-compiler/xcodeml-tools.git xcodeml-tools
+pushd xcodeml-tools
+# Now build and install OMNI in the venv:
+cmake -S . -B build -DCMAKE_INSTALL_PREFIX=../loki_env
+cmake --build build
+cmake --install build
+popd
+```
+
+#### Option b: install (older) OMNI version and CLAW
 
 ```bash
 git clone --recursive --single-branch --branch=mlange-dev https://github.com/mlange05/claw-compiler.git claw-compiler
 pushd claw-compiler
-mkdir build
-pushd build
 # Now build and install CLAW in the venv:
-cmake -DCMAKE_INSTALL_PREFIX=../../loki_env ..
-make
-make install
-popd
+cmake -S . -B build -DCMAKE_INSTALL_PREFIX=../loki_env
+cmake --build build
+cmake --install build
 popd
 ```
 

diff --git a/cmake/claw_compiler.cmake b/cmake/claw_compiler.cmake
@@ -79,9 +79,11 @@ function(install_claw_compiler VERSION)
         CMAKE_ARGS -DOMNI_CONF_OPTION=JAR=${Java_JAR_EXECUTABLE} -DCMAKE_INSTALL_PREFIX=${CLAW_DIR} -DJAVA_HOME=${JAVA_HOME} -DANT_HOME=${ANT_HOME}
     )
 
-    add_executable( clawfc IMPORTED GLOBAL )
-    set_property( TARGET clawfc PROPERTY IMPORTED_LOCATION ${CLAW_DIR}/bin/clawfc )
-    add_dependencies( clawfc claw )
+    foreach ( _bin clawfc F_Front )
+        add_executable( ${_bin} IMPORTED GLOBAL )
+        set_property( TARGET ${_bin} PROPERTY IMPORTED_LOCATION ${CLAW_DIR}/bin/${_bin} )
+        add_dependencies( ${_bin} claw )
+    endforeach()
 
     # Forward variables to parent scope
     foreach ( _VAR_NAME CLAW_DIR )

diff --git a/cmake/cmake-jdk-ant/CMakeLists.txt b/cmake/cmake-jdk-ant/CMakeLists.txt
@@ -19,7 +19,7 @@ set( FORCE_ANT_INSTALL OFF CACHE BOOL "Force installation of OpenJDK" )
 set( OPEN_JDK_INSTALL_VERSION 11.0.2 CACHE STRING "OpenJDK version to install if Java >= ${MINIMUM_JAVA_VERSION} not found" )
 set( OPEN_JDK_MIRROR https://download.java.net/java/GA/jdk11/9/GPL/ CACHE STRING "OpenJDK download mirror" )
 
-set( ANT_INSTALL_VERSION 1.10.12 CACHE STRING "ant version to install if Ant >= ${MINIMUM_ANT_VERSION} not found" )
+set( ANT_INSTALL_VERSION 1.10.15 CACHE STRING "ant version to install if Ant >= ${MINIMUM_ANT_VERSION} not found" )
 set( ANT_MIRROR https://archive.apache.org/dist/ant/binaries/ CACHE STRING "ant download mirror" )
 
 list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/module" )

diff --git a/cmake/loki_transform.cmake b/cmake/loki_transform.cmake
@@ -452,7 +452,7 @@ endfunction()
 #
 #  ..warning::
 #      loki_transform_transpile() was removed!
-# 
+#
 #  Please use
 #       loki_transform( COMMAND convert [...] )
 #   or
@@ -592,7 +592,7 @@ function( generate_xmod )
         endforeach()
     endif()
 
-    if( TARGET clawfc )
+    if( TARGET clawfc AND NOT TARGET F_Front )
         get_target_property( _CLAWFC_EXECUTABLE clawfc IMPORTED_LOCATION )
         get_filename_component( _CLAWFC_LOCATION ${_CLAWFC_EXECUTABLE} DIRECTORY )
         set( _F_FRONT_EXECUTABLE ${_CLAWFC_LOCATION}/F_Front )

diff --git a/cmake/omni_compiler.cmake b/cmake/omni_compiler.cmake
@@ -0,0 +1,89 @@
+# (C) Copyright 2018- ECMWF.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+##############################################################################
+# .rst:
+#
+# install_omni_compiler
+# =====================
+#
+# Download and install OMNI Compiler. ::
+#
+#   install_omni_compiler(VERSION)
+#
+# Installation procedure
+# ----------------------
+#
+# OMNI will be installed during the build step into folder
+# `omni-compiler` in the current binary directory ``${CMAKE_CURRENT_BINARY_DIR}``.
+#
+# Options
+# -------
+#
+# :VERSION:     The git branch or tag to download
+#
+# Output variables
+# ----------------
+# :OMNI_DIR:    The directory into which OMNI has been installed
+#
+##############################################################################
+
+include( FetchContent )
+include( ExternalProject )
+
+function(install_omni_compiler VERSION)
+
+    set( OMNI_DIR "" )
+    message( STATUS "Downloading OMNI Compiler")
+
+    # Bootstrap OpenJDK and Ant, if necessary
+    add_subdirectory( cmake/cmake-jdk-ant )
+
+    # Build OMNI Compiler
+    FetchContent_Declare(
+        omni_compiler
+        GIT_REPOSITORY  https://github.com/omni-compiler/xcodeml-tools.git
+        GIT_TAG         ${VERSION}
+        GIT_SHALLOW     ON
+    )
+
+    # Need to fetch manually to be able to do an "in-build installation"
+    FetchContent_GetProperties( omni_compiler )
+    if( NOT omni_compiler_POPULATED )
+        FetchContent_Populate( omni_compiler )
+
+        set( OMNI_DIR ${CMAKE_CURRENT_BINARY_DIR}/omni-compiler )
+
+    endif()
+
+    find_program(MAKE_EXECUTABLE NAMES gmake make mingw32-make REQUIRED)
+
+    ExternalProject_Add(
+        omni
+        SOURCE_DIR ${omni_compiler_SOURCE_DIR}
+        BINARY_DIR ${omni_compiler_BINARY_DIR}
+        INSTALL_DIR ${OMNI_DIR}
+
+        # Can skip this as FetchContent will take care of it at configure time
+        DOWNLOAD_COMMAND ""
+        UPDATE_COMMAND ""
+        PATCH_COMMAND ""
+
+        # Specify in-build installation target
+        CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${OMNI_DIR} -DJAVA_HOME=${JAVA_HOME}
+    )
+
+    add_executable( F_Front IMPORTED GLOBAL )
+    set_property( TARGET F_Front PROPERTY IMPORTED_LOCATION ${OMNI_DIR}/bin/F_Front )
+    add_dependencies( F_Front omni )
+
+    # Forward variables to parent scope
+    foreach ( _VAR_NAME OMNI_DIR )
+        set( ${_VAR_NAME} ${${_VAR_NAME}} PARENT_SCOPE )
+    endforeach()
+
+endfunction()