add the option of IJ base (#1056)

AUTOTEST/cmake.sh

@@ -6,7 +6,7 @@
 
 testname=`basename $0 .sh`
 
-drivers="ij sstruct struct ams_driver maxwell_unscaled sstruct_fac ij_mv struct_migrate"
+drivers="ij sstruct struct ams_driver maxwell_unscaled sstruct_fac ij_mv struct_migrate ij_assembly"
 
 # Echo usage information
 case $1 in

src/IJ_mv/IJMatrix_parcsr_device.c

@@ -579,6 +579,7 @@ hypre_IJMatrixAssembleParCSRDevice(hypre_IJMatrix *matrix)
    HYPRE_BigInt row_end   = row_partitioning[1];
    HYPRE_BigInt col_start = col_partitioning[0];
    HYPRE_BigInt col_end   = col_partitioning[1];
+   HYPRE_BigInt col_first = hypre_IJMatrixGlobalFirstCol(matrix);
    HYPRE_Int nrows = row_end - row_start;
    HYPRE_Int ncols = col_end - col_start;
 
@@ -743,6 +744,24 @@ hypre_IJMatrixAssembleParCSRDevice(hypre_IJMatrix *matrix)
                          _1 - row_start );
 #endif
 
+      /* adjust col indices wrt the global first index */
+      if (col_first)
+      {
+#if defined(HYPRE_USING_SYCL)
+         HYPRE_ONEDPL_CALL( std::transform,
+                            new_j,
+                            new_j + new_nnz,
+                            new_j,
+         [col_first = col_first] (const auto & x) {return x - col_first;} );
+#else
+         HYPRE_THRUST_CALL( transform,
+                            new_j,
+                            new_j + new_nnz,
+                            new_j,
+                            _1 - col_first );
+#endif
+      }
+
       hypre_TFree(new_i, HYPRE_MEMORY_DEVICE);
 
       HYPRE_Int      num_cols_offd_new;
@@ -769,9 +788,9 @@ hypre_IJMatrixAssembleParCSRDevice(hypre_IJMatrix *matrix)
                                        new_j,
                                        NULL,
                                        NULL,
-                                       col_start,
-                                       col_end - 1,
-                                       hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(par_matrix)),
+                                       col_start - col_first,
+                                       col_end - col_first - 1,
+                                       -1,
                                        NULL,
                                        NULL,
                                        NULL,
@@ -817,8 +836,8 @@ hypre_IJMatrixAssembleParCSRDevice(hypre_IJMatrix *matrix)
                                        new_j,
                                        new_data,
                                        diag_nnz_existed || offd_nnz_existed ? new_sora : NULL,
-                                       col_start,
-                                       col_end - 1,
+                                       col_start - col_first,
+                                       col_end - col_first - 1,
                                        hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(par_matrix)),
                                        hypre_ParCSRMatrixDeviceColMapOffd(par_matrix),
                                        &col_map_offd_map,

src/test/CMakeLists.txt

@@ -18,6 +18,7 @@ set(TEST_SRCS
   struct_migrate.c
   sstruct_fac.c
   ij_mv.c
+  ij_assembly.c
 )
 
 add_hypre_executables(TEST_SRCS)

src/test/TEST_ij/assembly.jobs

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
+# HYPRE Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+#=============================================================================
+# ij_assembly: Run default case
+#=============================================================================
+
+mpirun -np 1  ./ij_assembly > assembly.out.0
+
+mpirun -np 2  ./ij_assembly > assembly.out.1
+
+mpirun -np 3  ./ij_assembly > assembly.out.2
+
+mpirun -np 7  ./ij_assembly > assembly.out.3
+

src/test/TEST_ij/assembly.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
+# HYPRE Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+TNAME=`basename $0 .sh`
+RTOL=$1
+ATOL=$2
+
+#=============================================================================
+# compare with baseline case
+#=============================================================================
+
+FILES="\
+ ${TNAME}.out.0\
+ ${TNAME}.out.1\
+ ${TNAME}.out.2\
+ ${TNAME}.out.3\
+"
+
+for i in $FILES
+do
+  echo "# Output file: $i"
+  tail -1 $i
+done > ${TNAME}.out
+
+# Make sure that the output file is reasonable
+RUNCOUNT=`echo $FILES | wc -w`
+OUTCOUNT=`grep "error code" ${TNAME}.out | wc -l`
+if [ "$OUTCOUNT" != "$RUNCOUNT" ]; then
+  echo "Incorrect number of runs in ${TNAME}.out" >&2
+fi
+
+# Make sure that the output file has no errors
+for i in $FILES
+do
+  echo "# Output file: $i"
+  echo "Test error code = 0"
+done > ${TNAME}.testdata
+
+diff ${TNAME}.testdata ${TNAME}.out >&2
+
+#=============================================================================
+# remove temporary files
+#=============================================================================
+
+rm -f ${TNAME}.testdata