diff --git a/examples/test_tpetra_mesh.cpp b/examples/test_tpetra_mesh.cpp index 98a5a736..d05cfcb8 100644 --- a/examples/test_tpetra_mesh.cpp +++ b/examples/test_tpetra_mesh.cpp @@ -105,7 +105,6 @@ void run_test(mesh_data &mesh) TpetraPartitionMap<> all_node_map = mesh.nodes_in_elem_distributed.pmap; size_t nlocal_nodes = mesh.nlocal_nodes; int ntimesteps = 1000; - real_t constant_velocity = 0.0001; real_t timestep = 0.001; //arbitrary calculation done by looping over all local elements for all timesteps diff --git a/src/include/tpetra_wrapper_types.h b/src/include/tpetra_wrapper_types.h index b73618dc..e2a95027 100644 --- a/src/include/tpetra_wrapper_types.h +++ b/src/include/tpetra_wrapper_types.h @@ -2776,393 +2776,6 @@ TpetraDFArray::~TpetraDFArray() {} // End of TpetraDFArray //////////////////////////////////////////////////////////////////////////////// -///////////////////////// -// TpetraCArray: CArray Tpetra wrapper around Tpetra CRSMatrix; it was the only distributed Layout Right type we knew about in Trilinos. -///////////////////////// -template -class TpetraCArray { - - // this is managed - //using TArray1D = CArrayKokkos; - using TArray1D = Kokkos::View ; - using TArray1D_Host = CArrayKokkos; - - using row_map_type = Kokkos::View; - using values_array = Kokkos::View; - using global_indices_array = Kokkos::View; - using indices_array = Kokkos::View; - - size_t dims_[7]; - size_t global_dim0_; - size_t submap_size_; - size_t length_; - size_t order_; // tensor order (rank) - size_t column_map_size_; - MPI_Comm mpi_comm_; - MPI_Datatype mpi_datatype_; - TArray1D this_array_; - row_map_type start_index_; - indices_array crs_local_indices_; - - // Trilinos type definitions - typedef Tpetra::CrsMatrix MAT; //stands for matrix - typedef const Tpetra::CrsMatrix const_MAT; - typedef Tpetra::MultiVector MV; - typedef MV::dual_view_type::t_dev vec_array; - typedef MV::dual_view_type::t_host host_vec_array; - typedef Kokkos::View const_host_vec_array; - typedef Kokkos::View const_vec_array; - typedef Kokkos::View const_host_ivec_array; - typedef Kokkos::View host_ivec_array; - typedef MV::dual_view_type dual_vec_array; - - Teuchos::RCP> importer; // tpetra comms object - - -public: - - //data for arrays that own both shared and local data and aren't intended to communicate with another MATAR type - //This is simplifying for cases such as a local + ghost storage vector where you need to update the ghost entries - bool own_comms; //This Mapped MPI Array contains its own communication plan; just call array_comms() - - void set_mpi_type(); - TpetraPartitionMap pmap; - TpetraPartitionMap column_pmap; - TpetraPartitionMap comm_pmap; - Teuchos::RCP> tpetra_pmap; - Teuchos::RCP> tpetra_column_pmap; - Teuchos::RCP> tpetra_comm_pmap; - Teuchos::RCP tpetra_crs_matrix; - - TpetraCArray(); - - //Copy Constructor - TpetraCArray(const TpetraCArray &temp){ - *this = temp; - } - - TpetraCArray(size_t global_dim0, - const std::string& tag_string = DEFAULTSTRINGARRAY, MPI_Comm mpi_comm = MPI_COMM_WORLD); - - //CRS matrix constructor for banded matrix case - TpetraCArray(size_t global_dim0, size_t dim2, - const std::string& tag_string = DEFAULTSTRINGARRAY, MPI_Comm mpi_comm = MPI_COMM_WORLD); - - //CRS matrix constructor with arbitrary row graph and column map supplied - TpetraCArray(TpetraPartitionMap &input_pmap, size_t dim1, const std::string& tag_string = DEFAULTSTRINGARRAY); - - //CRS matric constructor with arbitrary row graph; builds column map for you and thus one less arg - TpetraCArray(Teuchos::RCP> input_pmap, size_t dim1, const std::string& tag_string = DEFAULTSTRINGARRAY); - - KOKKOS_INLINE_FUNCTION - T& operator()(size_t i) const; - - KOKKOS_INLINE_FUNCTION - T& operator()(size_t i, size_t j) const; - - // T& host(size_t i, size_t j) const; - - KOKKOS_INLINE_FUNCTION - TpetraCArray& operator=(const TpetraCArray& temp); - - // Method that returns size - KOKKOS_INLINE_FUNCTION - size_t size() const; - - KOKKOS_INLINE_FUNCTION - long long int getColumnMapGlobalIndex(int local_index) const; - - KOKKOS_INLINE_FUNCTION - long long int getMapGlobalIndex(int local_index) const; - - KOKKOS_INLINE_FUNCTION - int getColumnMapLocalIndex(long long int local_index) const; - - KOKKOS_INLINE_FUNCTION - int getMapLocalIndex(long long int local_index) const; - - KOKKOS_INLINE_FUNCTION - size_t dims(size_t i) const; - - size_t global_dim() const; - - // Method returns the raw device pointer of the Kokkos DualView - KOKKOS_INLINE_FUNCTION - T* device_pointer() const; - - // Method returns the raw host pointer of the Kokkos DualView - // KOKKOS_INLINE_FUNCTION - // T* host_pointer() const; - - // Method returns kokkos dual view - KOKKOS_INLINE_FUNCTION - Kokkos::View get_kokkos_view() const; - - // // Method that update host view - // void update_host(); - - // Method that update device view - void update_device(); - - //print vector data - void print() const; - - // Deconstructor - virtual KOKKOS_INLINE_FUNCTION - ~TpetraCArray (); -}; // End of TpetraCArray - - -// Default constructor -template -TpetraCArray::TpetraCArray(): tpetra_pmap(NULL){ - length_ = 0; - for (int i = 0; i < 7; i++) { - dims_[i] = 0; - } -} - -// 1D Constructor -template -TpetraCArray::TpetraCArray(size_t global_dim0, const std::string& tag_string, MPI_Comm mpi_comm) { -} - -// Constructor that takes local data in a matar ragged type -template -TpetraCArray::TpetraCArray(size_t global_dim0, size_t dim1, const std::string& tag_string, MPI_Comm mpi_comm) { - mpi_comm_ = mpi_comm; - global_dim0_ = global_dim0; - Teuchos::RCP> teuchos_comm = Teuchos::rcp(new Teuchos::MpiComm(mpi_comm_)); - tpetra_pmap = Teuchos::rcp(new Tpetra::Map((long long int) global_dim0, 0, teuchos_comm)); - pmap = TpetraPartitionMap(tpetra_pmap); - dims_[0] = tpetra_pmap->getLocalNumElements(); - dims_[1] = dim1; - order_ = 2; - - //this_array_ = TArray1D(dims_[0], dims_[1], "tag_string"); - this_array_ = TArray1D("tag_string", dims_[0]*dims_[1]); - size_t nnz = dims_[0]*dims_[1]; - - //assign indices 0 to dim1-1 since tpetra constructor still needs a graph - //row offsets with compatible template arguments - row_map_type row_offsets_pass("row_offsets", dims_[0] + 1); - - Kokkos::parallel_for("StartValuesInit", dims_[0]+1, KOKKOS_CLASS_LAMBDA(const int i) { - row_offsets_pass(i) = 0; - }); - - Kokkos::parallel_scan("StartValuesSetup", dims_[0], KOKKOS_CLASS_LAMBDA(const int i, int& update, const bool final) { - // Load old value in case we update it before accumulating - const size_t count = dims_[1]; - update += count; - if (final) { - row_offsets_pass((i+1)) = update; - } - - }); - - row_offsets_pass(0) = 0; - for(int ipass = 1; ipass < dims_[0] + 1; ipass++){ - row_offsets_pass(ipass) = row_offsets_pass(ipass-1) + dims_[1]; - } - - global_indices_array input_crs_graph = global_indices_array("dummy_graph", nnz); - FOR_ALL(i,0,dims_[0], { - for(int igraph = 0; igraph < dims_[1]; igraph++){ - input_crs_graph(row_offsets_pass(i) + igraph) = igraph; - } - }); - //build column map for the global conductivity matrix - Teuchos::RCP > colmap; - const Teuchos::RCP > dommap = tpetra_pmap; - - Tpetra::Details::makeColMap(colmap, tpetra_pmap, input_crs_graph, nullptr); - tpetra_column_pmap = colmap; - - //debug print - //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - - //local indices in the graph using the constructed column map - crs_local_indices_ = indices_array("crs_local_indices", nnz); - FOR_ALL(i,0,dims_[0], { - for(int igraph = 0; igraph < dims_[1]; igraph++){ - crs_local_indices_(row_offsets_pass(i) + igraph) = tpetra_column_pmap->getLocalElement(input_crs_graph(row_offsets_pass(i) + igraph)); - } - }); - - //sort values and indices - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, crs_local_indices_, this_array_); - - tpetra_crs_matrix = Teuchos::rcp(new MAT(tpetra_pmap, tpetra_column_pmap, row_offsets_pass, crs_local_indices_, this_array_)); - tpetra_crs_matrix->fillComplete(); -} - -// Overloaded 2D constructor where you provide a partition map -template -TpetraCArray::TpetraCArray(TpetraPartitionMap &input_pmap, - size_t dim1, const std::string& tag_string) { -} - -// Overloaded 2D constructor taking an RPC pointer to a Tpetra Map -template -TpetraCArray::TpetraCArray(Teuchos::RCP> input_pmap, - size_t dim1, const std::string& tag_string) { - -} - -template -void TpetraCArray::set_mpi_type() { - if (typeid(T).name() == typeid(bool).name()) { - mpi_datatype_ = MPI_C_BOOL; - } - else if (typeid(T).name() == typeid(int).name()) { - mpi_datatype_ = MPI_INT; - } - else if (typeid(T).name() == typeid(long int).name()) { - mpi_datatype_ = MPI_LONG; - } - else if (typeid(T).name() == typeid(long long int).name()) { - mpi_datatype_ = MPI_LONG_LONG_INT; - } - else if (typeid(T).name() == typeid(float).name()) { - mpi_datatype_ = MPI_FLOAT; - } - else if (typeid(T).name() == typeid(double).name()) { - mpi_datatype_ = MPI_DOUBLE; - } - else { - printf("Your entered TpetraCArray type is not a supported type for MPI communications and is being set to int\n"); - mpi_datatype_ = MPI_INT; - } -} - -template -KOKKOS_INLINE_FUNCTION -T& TpetraCArray::operator()(size_t i) const { - assert(order_ == 1 && "Tensor order (rank) does not match constructor in TpetraCArray 1D!"); - assert(i >= 0 && i < dims_[0] && "i is out of bounds in TpetraCArray 1D!"); - return this_array_(i); -} - -template -KOKKOS_INLINE_FUNCTION -T& TpetraCArray::operator()(size_t i, size_t j) const { - assert(order_ == 2 && "Tensor order (rank) does not match constructor in TpetraCArray 2D!"); - assert(i >= 0 && i < dims_[0] && "i is out of bounds in TpetraCArray 2D!"); - assert(j >= 0 && j < dims_[1] && "j is out of bounds in TpetraCArray 2D!"); - return this_array_(i*dims_[1] + j); -} - -// Return global index corresponding to the input local (on this process/rank) index for the sub map this vector comms from -template -KOKKOS_INLINE_FUNCTION -long long int TpetraCArray::getColumnMapGlobalIndex(int local_index) const { - long long int global_index = tpetra_column_pmap->getGlobalElement(local_index); - return global_index; -} - -// Return global index corresponding to the input local (on this process/rank) index -template -KOKKOS_INLINE_FUNCTION -long long int TpetraCArray::getMapGlobalIndex(int local_index) const { - long long int global_index = tpetra_pmap->getGlobalElement(local_index); - return global_index; -} - -// Return global index corresponding to the input local (on this process/rank) index for the sub map this vector comms from -template -KOKKOS_INLINE_FUNCTION -int TpetraCArray::getColumnMapLocalIndex(long long int global_index) const { - int local_index = tpetra_column_pmap->getLocalElement(global_index); - return local_index; -} - -// Return global index corresponding to the input local (on this process/rank) index -template -KOKKOS_INLINE_FUNCTION -int TpetraCArray::getMapLocalIndex(long long int global_index) const { - int local_index = tpetra_pmap->getLocalElement(global_index); - return local_index; -} - -template -KOKKOS_INLINE_FUNCTION -TpetraCArray& TpetraCArray::operator= (const TpetraCArray& temp) { - - // Do nothing if the assignment is of the form x = x - if (this != &temp) { - for (int iter = 0; iter < temp.order_; iter++){ - dims_[iter] = temp.dims_[iter]; - } // end for - global_dim0_ = temp.global_dim0_; - order_ = temp.order_; - start_index_ = temp.start_index_; - crs_local_indices_ = temp.crs_local_indices_; - length_ = temp.length_; - this_array_ = temp.this_array_; - mpi_comm_ = temp.mpi_comm_; - mpi_datatype_ = temp.mpi_datatype_; - tpetra_crs_matrix = temp.tpetra_crs_matrix; - pmap = temp.pmap; - column_pmap = temp.column_pmap; - tpetra_pmap = temp.tpetra_pmap; - tpetra_column_pmap = temp.tpetra_column_pmap; - importer = temp.importer; - own_comms = temp.own_comms; - column_map_size_ = temp.column_map_size_; - } - - return *this; -} - -// Return size -template -KOKKOS_INLINE_FUNCTION -size_t TpetraCArray::size() const { - return length_; -} - -template -KOKKOS_INLINE_FUNCTION -size_t TpetraCArray::dims(size_t i) const { - assert(i < order_ && "TpetraCArray order (rank) does not match constructor, dim[i] does not exist!"); - assert(i >= 0 && dims_[i]>0 && "Access to TpetraCArray dims is out of bounds!"); - return dims_[i]; -} - -template -size_t TpetraCArray::global_dim() const { - return global_dim0_; -} - -template -KOKKOS_INLINE_FUNCTION -T* TpetraCArray::device_pointer() const { - return this_array_.data(); -} - -template -KOKKOS_INLINE_FUNCTION -Kokkos::View TpetraCArray::get_kokkos_view() const { - return this_array_; -} - -template -void TpetraCArray::print() const { - std::ostream &out = std::cout; - Teuchos::RCP fos; - fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - tpetra_crs_matrix->describe(*fos,Teuchos::VERB_EXTREME); -} - -template -KOKKOS_INLINE_FUNCTION -TpetraCArray::~TpetraCArray() {} - -//////////////////////////////////////////////////////////////////////////////// -// End of TpetraCArray -//////////////////////////////////////////////////////////////////////////////// - ///////////////////////// // TpetraCRSMatrix: CRS Matrix Tpetra wrapper. /////////////////////////