Parsing ROM related options (#228)

* librom compilation setup.

* rom config options.

* parsing rom config option to Control.

.gitignore

@@ -0,0 +1 @@
+src/mgmol_config.h

CMakeLists.txt

@@ -136,6 +136,10 @@ if(USE_LIBROM)
   endif(NOT LIBROM_PATH)
 
   find_package(libROM REQUIRED)
+
+  if(libROM_FOUND)
+    set(MGMOL_HAS_LIBROM 1)
+  endif(libROM_FOUND)
 endif(USE_LIBROM)
 
 # ARPACK (optional)
@@ -261,6 +265,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra")
 include_directories("${PROJECT_SOURCE_DIR}/src/tools")
 include_directories("${PROJECT_SOURCE_DIR}/src")
 
+include_directories("${LIBROM_PATH}/lib")
+link_libraries(${LIBROM_LIB})
+
 # add subdirectories for source files, tests
 add_subdirectory(src)
 

src/CMakeLists.txt

@@ -1,4 +1,7 @@
 
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h.in
+  ${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h @ONLY)
+
 add_subdirectory(DistMatrix)
 add_subdirectory(linear_algebra)
 add_subdirectory(local_matrices)
@@ -8,7 +11,8 @@ add_subdirectory(radial)
 add_subdirectory(sparse_linear_algebra)
 add_subdirectory(tools)
 
-set(link_libs mgmol_distmatrix 
+set(link_libs
+   mgmol_distmatrix 
    mgmol_linear_algebra 
    mgmol_local_matrices 
    mgmol_numerical_kernels 
@@ -161,7 +165,11 @@ add_library(mgmol_src ${SOURCES})
 
 target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS})
 target_include_directories(mgmol_src PRIVATE ${Boost_INCLUDE_DIRS})
-target_include_directories (mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+target_include_directories(mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+if(USE_LIBROM)
+  target_include_directories (mgmol_src PRIVATE ${LIBROM_INCLUDES})
+  target_link_libraries(mgmol_src ${LIBROM_LIB})
+endif(USE_LIBROM)
 
 target_link_libraries(mgmol_src ${link_libs})
 if(${MGMOL_WITH_MAGMA})
@@ -196,7 +204,7 @@ install(TARGETS mgmol-opt DESTINATION bin)
 # build ROM executable
 if(USE_LIBROM)
   add_executable(mgmol-rom rom_main.cc)
-  target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES})
+  target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS})
 
   target_link_libraries(mgmol-rom mgmol_src ${link_libs})
   target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES})

src/Control.cc

@@ -1917,6 +1917,10 @@ void Control::setOptions(const boost::program_options::variables_map& vm)
 
     // synchronize all processors
     sync();
+
+#ifdef MGMOL_HAS_LIBROM
+    setROMOptions(vm);
+#endif
 }
 
 int Control::checkOptions()
@@ -2051,3 +2055,66 @@ void Control::printPoissonOptions(std::ostream& os)
     }
     os << std::endl;
 }
+
+void Control::setROMOptions(const boost::program_options::variables_map& vm)
+{
+    printWithTimeStamp("Control::setROMOptions()...", std::cout);
+
+    if (onpe0)
+    {
+        std::string str = vm["ROM.stage"].as<std::string>();
+        if (str.compare("offline") == 0)
+            rom_pri_option.rom_stage = ROMStage::OFFLINE;
+        else if (str.compare("online") == 0)
+            rom_pri_option.rom_stage = ROMStage::ONLINE;
+        else if (str.compare("build") == 0)
+            rom_pri_option.rom_stage = ROMStage::BUILD;
+        else if (str.compare("none") == 0)
+            rom_pri_option.rom_stage = ROMStage::UNSUPPORTED;
+
+        rom_pri_option.restart_file_fmt = vm["ROM.offline.restart_filefmt"].as<std::string>();
+        rom_pri_option.restart_file_minidx = vm["ROM.offline.restart_min_idx"].as<int>();
+        rom_pri_option.restart_file_maxidx = vm["ROM.offline.restart_max_idx"].as<int>();
+        rom_pri_option.basis_file = vm["ROM.offline.basis_file"].as<std::string>();
+
+        rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as<bool>();
+    }  // onpe0
+
+    // synchronize all processors
+    syncROMOptions();
+}
+
+void Control::syncROMOptions()
+{
+    if (onpe0 && verbose > 0)
+        (*MPIdata::sout) << "Control::syncROMOptions()" << std::endl;
+
+    MGmol_MPI& mmpi = *(MGmol_MPI::instance());
+
+    mmpi.bcast(rom_pri_option.restart_file_fmt, comm_global_);
+    mmpi.bcast(rom_pri_option.basis_file, comm_global_);
+
+    auto bcast_check = [](int mpirc) {
+        if (mpirc != MPI_SUCCESS)
+        {
+            (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl;
+            MPI_Abort(comm_global_, 2);
+        }
+    };
+
+    short rom_stage = (short)static_cast<int>(rom_pri_option.rom_stage);
+    int mpirc;
+    mpirc = MPI_Bcast(&rom_stage, 1, MPI_SHORT, 0, comm_global_);
+    bcast_check(mpirc);
+
+    mpirc = MPI_Bcast(&rom_pri_option.restart_file_minidx, 1, MPI_INT, 0, comm_global_);
+    bcast_check(mpirc);
+
+    mpirc = MPI_Bcast(&rom_pri_option.restart_file_maxidx, 1, MPI_INT, 0, comm_global_);
+    bcast_check(mpirc);
+
+    mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_);
+    bcast_check(mpirc);
+
+    rom_pri_option.rom_stage = static_cast<ROMStage>(rom_stage);
+}

src/Control.h

@@ -13,6 +13,10 @@
 #include "Species.h"
 #include "Timeout.h"
 
+/* enumeration and option variables for libROM */
+#include "mgmol_config.h"
+#include "rom_Control.h"
+
 #include <cassert>
 #include <fstream>
 #include <iostream>
@@ -220,6 +224,9 @@ class Control
 
     void printRestartLink();
 
+    /* libROM related options */
+    ROMPrivateOptions rom_pri_option;
+
 public:
     static Control* instance()
     {
@@ -707,6 +714,11 @@ class Control
     }
 
     bool AtomsMove() { return (atoms_dyn_ != 0); }
+
+    /* ROM-related options */
+    void setROMOptions(const boost::program_options::variables_map& vm);
+    void syncROMOptions();
+    const ROMPrivateOptions getROMOptions() { return rom_pri_option; }
 };
 
 #endif

src/MGmol_prototypes.h

@@ -10,9 +10,11 @@
 #ifndef MGMOL_PROTOTYPES_H
 #define MGMOL_PROTOTYPES_H
 
+#include "mgmol_config.h"
 #include "global.h"
 
 #include <boost/program_options.hpp>
+namespace po = boost::program_options;
 
 class Ions;
 class KBPsiMatrixSparse;
@@ -24,4 +26,8 @@ int read_config(int argc, char** argv,
     boost::program_options::variables_map& vm, std::string& input_file,
     std::string& lrs_filename, std::string& constraints_filename,
     float& total_spin, bool& with_spin, bool& tcheck);
+#ifdef MGMOL_HAS_LIBROM
+void setupROMConfigOption(po::options_description &rom_cfg);
+#endif
+
 #endif

src/mgmol_config.h.in

@@ -0,0 +1,19 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC and
+// UT-Battelle, LLC.
+// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge
+// National Laboratory.
+// LLNL-CODE-743438
+// All rights reserved.
+// This file is part of MGmol. For details, see https://github.com/llnl/mgmol.
+// Please also read this link https://github.com/llnl/mgmol/LICENSE
+
+// Description: A configuration header for mgmol.
+//
+
+#ifndef INCLUDED_MGMOL_CONFIG_H
+#define INCLUDED_MGMOL_CONFIG_H
+
+/* Have google test library. */
+#cmakedefine MGMOL_HAS_LIBROM
+
+#endif

src/read_config.cc

@@ -14,6 +14,7 @@
 #include <iterator>
 #include <sys/cdefs.h>
 #include <vector>
+#include "MGmol_prototypes.h"
 
 #include <boost/program_options.hpp>
 namespace po = boost::program_options;
@@ -201,6 +202,10 @@ int read_config(int argc, char** argv, po::variables_map& vm,
             po::value<std::string>()->default_value(""),
             "Output file for dumping cluster information in vtk format");
 
+#ifdef MGMOL_HAS_LIBROM
+        setupROMConfigOption(config);
+#endif
+
         // Hidden options, will be allowed in config file, but will not be
         // shown to the user.
         po::options_description hidden("Hidden options");
@@ -409,3 +414,22 @@ int read_config(int argc, char** argv, po::variables_map& vm,
 
     return 0;
 }
+
+#ifdef MGMOL_HAS_LIBROM
+void setupROMConfigOption(po::options_description &rom_cfg)
+{
+    rom_cfg.add_options()
+        ("ROM.stage", po::value<std::string>()->default_value("none"),
+            "ROM workflow stage: offline; build; online; none.")
+        ("ROM.offline.restart_filefmt", po::value<std::string>(),
+            "File name format to read for snapshots.")
+        ("ROM.offline.restart_min_idx", po::value<int>(),
+            "Minimum index for snapshot file format.")
+        ("ROM.offline.restart_max_idx", po::value<int>(),
+            "Maximum index for snapshot file format.")
+        ("ROM.offline.basis_file", po::value<std::string>(),
+            "File name for libROM snapshot/POD matrices.")
+        ("ROM.offline.save_librom_snapshot", po::value<bool>()->default_value(false),
+            "Save libROM snapshot file at FOM simulation.");
+}
+#endif

src/rom_Control.h

@@ -0,0 +1,42 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC and
+// UT-Battelle, LLC.
+// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge
+// National Laboratory.
+// LLNL-CODE-743438
+// All rights reserved.
+// This file is part of MGmol. For details, see https://github.com/llnl/mgmol.
+// Please also read this link https://github.com/llnl/mgmol/LICENSE
+
+#ifndef ROM_CONTROL_H
+#define ROM_CONTROL_H
+
+#include <cassert>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+
+enum class ROMStage
+{
+    OFFLINE,
+    ONLINE,
+    RESTORE,    // TODO(kevin): what stage is this?
+    BUILD,
+    UNSUPPORTED
+};
+
+/* Stored as a private member variable of Control class */
+struct ROMPrivateOptions
+{
+    ROMStage rom_stage = ROMStage::UNSUPPORTED;
+
+    std::string restart_file_fmt = "";
+    int restart_file_minidx = -1;
+    int restart_file_maxidx = -1;
+    std::string basis_file = "";
+
+    /* save librom snapshot matrix at FOM simulation. */
+    bool save_librom_snapshot = false;
+};
+
+#endif  // ROM_CONTROL_H