Merge pull request #342 from electronic-structure/mixer

Refactor density mixing in pseudopotential case

CMakeLists.txt

@@ -1,6 +1,9 @@
 cmake_minimum_required(VERSION 3.9) # Intel C++11 support starts from 3.6 or even later version
 project(SIRIUS)
 
+# allow {module}_ROOT variables to be set
+cmake_policy(SET CMP0074 NEW)
+
 # set language and standard
 enable_language(CXX Fortran)
 set(CMAKE_CXX_STANDARD 11)
@@ -12,6 +15,7 @@ set(BUILD_DOCS OFF CACHE BOOL "build doxygen doc")
 set(USE_ELPA OFF CACHE BOOL "use scalapack")
 set(USE_MAGMA OFF CACHE BOOL "use MAGMA")
 set(USE_CUDA OFF CACHE BOOL "use CUDA")
+set(USE_ROCM OFF CACHE BOOL "use ROCM AMD GPU code")
 set(USE_VDWXC OFF CACHE BOOL "use libvdwxc for van der Walls corrections")
 set(USE_MKL OFF CACHE BOOL "use Intel MKL")
 set(USE_CRAY_LIBSCI OFF CACHE BOOL "use LAPACK/SCALAPACK from Cray LIBSCI")
@@ -133,6 +137,24 @@ if(USE_MAGMA)
   set(SYSTEM_LIBRARIES "${SYSTEM_LIBRARIES};${MAGMA_LIBRARIES}")
 endif(USE_MAGMA)
 
+if(USE_CUDA AND USE_ROCM)
+  message(FATAL_ERROR "USE_CUDA and USE_ROCM must not be enabled at the same time!")
+endif()
+
+if(USE_ROCM)
+    message(STATUS "WARNING: ROCM enabled, prototype feature! Only limited functionality available.")
+    find_package(ROCM COMPONENTS rocfft hipblas)
+    if(NOT ${ROCM_HIP_PLATFORM} STREQUAL hcc)
+	message(FATAL_ERROR "Compilation on Nvidia platform not supported with ROCM enabled!")
+    endif()
+    add_definitions("-D__GPU")
+    add_definitions("-D__ROCM")
+    add_definitions(${ROCM_DEFINITIONS})
+    include_directories(${ROCM_INCLUDE_DIRS})
+    add_subdirectory(src/SDDK/GPU/hipblas_port)
+    include_directories(src/SDDK/GPU/hipblas_port)
+endif()
+
 # add required libraries
 set(SYSTEM_LIBRARIES "${SYSTEM_LIBRARIES};${MPI_CXX_LIBRARIES};${GSL_LIBRARY};${LIBXC_LIBRARIES};${LIBSPG_LIBRARIES}")
 
@@ -235,7 +257,15 @@ MACRO(SIRIUS_SETUP_TARGET _target)
       message(FATAL_ERROR "Unsupported compiler")
     endif()
   else()
-    target_link_libraries(${_target} PRIVATE "${sirius_cu_location};${SYSTEM_LIBRARIES};${LAPACK_LIBRARIES};${FFTW_LIBRARIES};${HDF5_C_LIBRARIES};${HDF5_C_HL_LIBRARIES}")
+    target_link_libraries(${_target} PRIVATE "${SYSTEM_LIBRARIES};${LAPACK_LIBRARIES};${FFTW_LIBRARIES};${HDF5_C_LIBRARIES};${HDF5_C_HL_LIBRARIES}")
+  endif()
+
+  if(USE_CUDA)
+    target_link_libraries(${_target} PRIVATE "${sirius_cu_location}")
+  endif()
+
+  if(USE_ROCM)
+    target_link_libraries(${_target} PRIVATE sirius_rocm sirius_rocm_interface hipblas_port ${ROCM_LIBRARIES})
   endif()
 
   if(OpenMP_CXX_FOUND)
@@ -254,12 +284,13 @@ if(USE_CUDA)
 endif()
 
 # applications
-add_subdirectory(apps/atoms)
-add_subdirectory(apps/dft_loop)
 if(BUILD_TESTS)
   add_subdirectory(apps/tests)
   add_subdirectory(apps/unit_tests)
 endif(BUILD_TESTS)
+
+add_subdirectory(apps/atoms)
+add_subdirectory(apps/dft_loop)
 add_subdirectory(apps/utils)
 add_subdirectory(python_module)
 add_subdirectory(doc)

VERSION

@@ -1 +1 @@
-6.1.0
+6.1.3

apps/dft_loop/sirius.scf.cpp

@@ -41,15 +41,6 @@ std::unique_ptr<Simulation_context> create_sim_ctx(std::string     fname__,
     auto gen_evp_solver_name = args__.value("gen_evp_solver_name", ctx.control().gen_evp_solver_name_);
     ctx.gen_evp_solver_name(gen_evp_solver_name);
 
-//    auto pu = args__.value("processing_unit", ctx.control().processing_unit_);
-//    if (pu == "") {
-//#ifdef __GPU
-//        pu = "gpu";
-//#else
-//        pu = "cpu";
-//#endif
-//    }
-//    ctx.set_processing_unit(pu);
     ctx.import(args__);
 
     return std::move(ctx_ptr);
@@ -97,6 +88,14 @@ double ground_state(Simulation_context& ctx,
     /* launch the calculation */
     auto result = dft.find(inp.potential_tol_, inp.energy_tol_, inp.num_dft_iter_, write_state);
 
+    auto repeat_update = args.value<int>("repeat_update", 0);
+    if (repeat_update) {
+        for (int i = 0; i < repeat_update; i++) {
+            dft.update();
+            result = dft.find(inp.potential_tol_, inp.energy_tol_, inp.num_dft_iter_, write_state);
+        }
+    }
+
     dft.print_magnetic_moment();
 
     if (!ctx.full_potential()) {
@@ -146,8 +145,8 @@ double ground_state(Simulation_context& ctx,
             }
             if (diff > 1e-6) {
                 printf("total stress is different!");
-                //std::cout << "  reference: " << dict_ref["ground_state"]["stress"] << "\n";
-                //std::cout << "  computed: " << result["stress"] << "\n";
+                std::cout << "  reference: " << dict_ref["ground_state"]["stress"] << "\n";
+                std::cout << "  computed: " << result["stress"] << "\n";
                 ctx.comm().abort(2);
             }
         }
@@ -162,8 +161,8 @@ double ground_state(Simulation_context& ctx,
             }
             if (diff > 1e-6) {
                 printf("total force is different!");
-                //std::cout << "  reference: " << dict_ref["ground_state"]["stress"] << "\n";
-                //std::cout << "  computed: " << result["stress"] << "\n";
+                std::cout << "  reference: " << dict_ref["ground_state"]["forces"] << "\n";
+                std::cout << "  computed: " << result["forces"] << "\n";
                 ctx.comm().abort(3);
             }
         }
@@ -357,6 +356,7 @@ int main(int argn, char** argv)
     args.register_key("--std_evp_solver_name=", "{string} standard eigen-value solver");
     args.register_key("--gen_evp_solver_name=", "{string} generalized eigen-value solver");
     args.register_key("--processing_unit=", "{string} type of the processing unit");
+    args.register_key("--repeat_update=", "{int} number of times to repeat update()");
     args.register_key("--control.processing_unit=", "");
     args.register_key("--control.mpi_grid_dims=","");
     args.register_key("--control.std_evp_solver_name=", "");

apps/tests/test_hloc.cpp

@@ -1,4 +1,6 @@
 #include <sirius.h>
+#include <fstream>
+#include <string>
 
 using namespace sirius;
 
@@ -79,9 +81,6 @@ void test_hloc(std::vector<int> mpi_grid_dims__, double cutoff__, int num_bands_
     t1.stop();
     hloc.dismiss();
 
-    if (pu == GPU && !phi.pw_coeffs(0).is_remapped()) {
-        hphi.pw_coeffs(0).copy_to(memory_t::host, 0, 4 * num_bands__);
-    }
 
     double diff{0};
     for (int i = 0; i < 4 * num_bands__; i++) {
@@ -116,6 +115,7 @@ int main(int argn, char** argv)
     args.register_key("--use_gpu=", "{int} 0: CPU only, 1: hybrid CPU+GPU");
     args.register_key("--gpu_ptr=", "{int} 0: start from CPU, 1: start from GPU");
     args.register_key("--repeat=", "{int} number of repetitions");
+    args.register_key("--t_file=", "{string} name of timing output file");
 
     args.parse_args(argn, argv);
     if (args.exist("help")) {
@@ -130,6 +130,7 @@ int main(int argn, char** argv)
     auto use_gpu = args.value<int>("use_gpu", 0);
     auto gpu_ptr = args.value<int>("gpu_ptr", 0);
     auto repeat = args.value<int>("repeat", 3);
+    auto t_file = args.value<std::string>("t_file", std::string(""));
 
     sirius::initialize(1);
     for (int i = 0; i < repeat; i++) {
@@ -138,6 +139,11 @@ int main(int argn, char** argv)
     Communicator::world().barrier();
     if (Communicator::world().rank() == 0) {
         utils::timer::print();
+
+        if (!t_file.empty()) {
+            std::ofstream json_file(t_file);
+            json_file << std::setw(2) << utils::timer::serialize() << std::endl;
+        }
     }
     Communicator::world().barrier();
     sirius::finalize();

apps/unit_tests/test_wf_ortho.cpp

@@ -37,7 +37,7 @@ void test_wf_ortho(std::vector<int> mpi_grid_dims__,
     linalg_t la{linalg_t::blas};
     memory_t mem{memory_t::host};
     if (pu == device_t::GPU) {
-        la = linalg_t::cublas;
+        la = linalg_t::gpublas;
         mem = memory_t::device;
     }