diff --git a/tests/cpgrid/addLgrsOnDistributedGrid_test.cpp b/tests/cpgrid/addLgrsOnDistributedGrid_test.cpp index 73824abda..eddbf7e56 100644 --- a/tests/cpgrid/addLgrsOnDistributedGrid_test.cpp +++ b/tests/cpgrid/addLgrsOnDistributedGrid_test.cpp @@ -397,9 +397,12 @@ void refinePatch_and_check(Dune::CpGrid& coarse_grid, BOOST_CHECK( static_cast(allGlobalIds_cells.size()) == global_cells_count); BOOST_CHECK( allGlobalIds_cells.size() == allGlobalIds_cells_set.size() ); - /** [Bug] Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1, + /** Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1, which in particular means that cells that share corners or edges (and not faces) with interior cells are not considered/ - seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points.*/ + seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points. + To achieve unique point global ids, IN SOME CASES, we invoke loanBalance( parts, false, true) where parts determines + the rank of each cell, false represents ownerFisrt, true represents addCornerCells. The last defaulted argument is + overlapLayerSize = 1. */ // Local/Global id sets for level grids (level 0, 1, ..., maxLevel). For level grids, local might differ from global id. for (int level = 0; level < coarse_grid.maxLevel() +1; ++level) @@ -456,6 +459,7 @@ BOOST_AUTO_TEST_CASE(threeLgrs) const std::array grid_dim = {10,8,8}; grid.createCartesian(grid_dim, cell_sizes); + // Fully interior LGRs std::vector parts(640); for (int k = 0; k < 8; ++k) { for (int j = 0; j < 8; ++j) { @@ -483,7 +487,8 @@ BOOST_AUTO_TEST_CASE(threeLgrs) } if(grid.comm().size()>1) { - grid.loadBalance(parts); + grid.loadBalance(parts); // ownerFirst = false, addCornerCells = false, overlapLayerSize = 1 + // IT's not necessary to change the default values since the LGRs are fully interior. const std::vector> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}}; const std::vector> startIJK_vec = {{0,0,0}, {0,0,3}, {3,2,2}}; @@ -545,6 +550,7 @@ BOOST_AUTO_TEST_CASE(atLeastOneLgr_per_process_attempt) {12,13,17,24,25,28,29,32,33}, {2,3,6,7,10,11,18,22,23}, {14,15,19,26,27,30,31,34,35} }; + // Fully interior LGRs for (int rank = 0; rank < 4; ++rank) { for (const auto& elemIdx : cells_per_rank[rank]) { parts[elemIdx] = rank; @@ -552,7 +558,8 @@ BOOST_AUTO_TEST_CASE(atLeastOneLgr_per_process_attempt) } if(grid.comm().size()>1) { - grid.loadBalance(parts); + grid.loadBalance(parts); // ownerFirst = false, addCornerCells = false, overlapLayerSize =1 + // IT's not necessary to change the default values since the LGRs are fully interior. const std::vector> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}, {2,2,2}}; const std::vector> startIJK_vec = {{0,1,0}, {0,0,2}, {3,2,0}, {3,0,2}}; @@ -607,7 +614,8 @@ BOOST_AUTO_TEST_CASE(atLeastOneLgr_per_process_attempt) } } -BOOST_AUTO_TEST_CASE(throw_not_fully_interior_lgr) + +BOOST_AUTO_TEST_CASE(not_fully_interior_lgr) { // Create a grid Dune::CpGrid grid; @@ -620,6 +628,7 @@ BOOST_AUTO_TEST_CASE(throw_not_fully_interior_lgr) {12,13,17,24,25,28,29,32,33}, {2,3,6,7,10,11,18,22,23}, {14,15,19,26,27,30,31,34,35} }; + // NOT fully interior LGRs for (int rank = 0; rank < 4; ++rank) { for (const auto& elemIdx : cells_per_rank[rank]) { parts[elemIdx] = rank; @@ -627,7 +636,8 @@ BOOST_AUTO_TEST_CASE(throw_not_fully_interior_lgr) } if(grid.comm().size()>1) { - grid.loadBalance(parts); + grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1 + // We set addCornersCells to achieve unique global ids for points belonging to refined level grids. const std::vector> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}, {2,2,2}}; const std::vector> startIJK_vec = {{0,1,0}, {0,0,2}, {3,1,0}, {3,0,2}}; @@ -759,7 +769,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr) } if(grid.comm().size()>1) { - grid.loadBalance(parts); + grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1 + // We set addCornersCells to achieve unique global ids for points belonging to refined level grids. const std::vector> cells_per_dim_vec = {{2,2,2}}; const std::vector> startIJK_vec = {{1,0,0}}; @@ -784,29 +795,7 @@ BOOST_AUTO_TEST_CASE(distributed_lgr) } auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm()); const std::set all_point_ids_set(all_point_ids.begin(), all_point_ids.end()); - // Coarse cells 1 in rank 0 and 2 in rank 2 share an I_FACE where (LGR1_dim[1]+1)*(LGR1_dim[2]+ 1), - // here (2 +1)*(2 +1) = 9 points lying on, 4 of them being the 4 corners of the coarse I_FACE shared - // by cell 1 and cell 2. Leaving us with 9 - 4 = 5 potential duplicated ids. - // - // Global cell ids of cells to be refined = {1, 2} - // cell 1 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3} - // cell 2 = { overlap in P0, does not exist in P1, interior in P2, overlap in P3} - // Remark: the LGR is distributed in only two of the four processes, P0 and P2. - // - P0 sees the entire LGR. - // - P1 does NOT see cell 2, but sees cell 1 - // - P2 does NOT see the LGR at all. - // - P3 does NOT see cell 1, but sees cell 2 - // P3 creates: - // +5 duplicated ids on {I_FACE, false} cell 2 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 1] - // That means that the "unfortunate" expected point ids count is 45 (desired value) + 5 (duplicated laying on - // shared I_FACE) = 50. - BOOST_CHECK( static_cast(all_point_ids_set.size()) == 50); - - // Current approach avoids duplicated point ids when - // 1. the LGR is distributed in P_{i_0}, ..., P_{i_n}, with n+1 < grid.comm().size(), - // AND - // 2. there is no coarse cell seen by a process P with P != P_{i_j}, j = 0, ..., n. - // Otherwise, there will be points with multiple ids.*/ + BOOST_CHECK( static_cast(all_point_ids_set.size()) == 45); // LGR1 dim 4x2x2 -> 5x3x3 = 45 points // Check global id is not duplicated for points std::vector localPointIds_vec; @@ -817,8 +806,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr) } auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm()); const std::set allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end()); - // Total global ids in leaf grid view for points: 80 + (45 - 12) + 5 (undesired duplicated ids on shared I_FACE) = 118 - BOOST_CHECK( allGlobalIds_points_set.size() == 118 ); + // Total global ids in leaf grid view for points: 80 + (45 - 12) = 113 + BOOST_CHECK( allGlobalIds_points_set.size() == 113 ); } } @@ -842,7 +831,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr_II) } if(grid.comm().size()>1) { - grid.loadBalance(parts); + grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1 + // We set addCornersCells to achieve unique global ids for points belonging to refined level grids. const std::vector> cells_per_dim_vec = {{2,2,2}}; const std::vector> startIJK_vec = {{0,2,0}}; @@ -863,16 +853,6 @@ BOOST_AUTO_TEST_CASE(distributed_lgr_II) auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm()); const std::set all_point_ids_set(all_point_ids.begin(), all_point_ids.end()); BOOST_CHECK( static_cast(all_point_ids_set.size()) == 63); - // Difference with previous test case: - // Global cell ids of cells to be refined = {8, 9, 10} - // cell 8 = { interior in P0, does not exist in P1, does not exist in P2, does not exist in P3} - // cell 9 = { interior in P0, does not exist in P1, overlap in P2, does not exist in P3} - // cell 10 = { overlap in P0, does not exist in P1, interior in P2, does not exist in P3} - // Remark: the LGR is distributed only in 2 processes which are the only two processes seeing these cells. - // - P0 sees the entire LGR. - // - P1 does NOT see the LGR at all. - // - P2 does NOT see cell 8, but NO OTHER process sees cell 8 since it's fully inteior of P0. - // - P3 does NOT see the LGR at all. // Check global id is not duplicated for points std::vector localPointIds_vec; @@ -907,7 +887,10 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr) } if(grid.comm().size()>1) { - grid.loadBalance(parts); + grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1 + // We set addCornersCells to achieve unique global ids for points belonging to refined level grids. + /** This is not enough to get unique point ids. */ + const std::vector> cells_per_dim_vec = {{2,2,2}}; const std::vector> startIJK_vec = {{1,0,0}}; const std::vector> endIJK_vec = {{3,2,2}}; @@ -926,6 +909,7 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr) // LGR1 dim 4x4x4 -> 5x5x5 = 125 points std::vector local_point_ids; local_point_ids.reserve(125); // expected_point_ids in LGR1 + for (const auto& element : elements(grid.levelGridView(1))) { for (int corner = 0; corner < 8; ++corner) { @@ -936,34 +920,13 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr) auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm()); const std::set all_point_ids_set(all_point_ids.begin(), all_point_ids.end()); - /** [Bug] Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1, - which in particular means that cells that share corners or edges (and not faces) with interior cells are not considered/ - seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points.*/ - // Difference with previous test case: - // Global cell ids of cells to be refined = {1,2,5,6,13,14,17,18} - // cell 1 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3} - // cell 2 = { overlap in P0, does not exist in P1, interior in P2, overlap in P3} - // cell 5 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3} - // cell 6 = { overlap in P0, does not exist in P1, interior in P2, does not exist in P3} - // cell 13 = { overlap in P0, interior in P1, does not exist in P2, overlap in P3} - // cell 14 = { does not exist in P0, overlap in P1, overlap in P2, interior in P3} - // cell 17 = { overlap in P0, interior in P1, overlap in P2, does not exist in P3} - // cell 18 = { does not exist in P0, overlap in P1, interior in P2, overlap in P3} // Remark: the LGR is distributed in ALL processes BUT - // - P0 does NOT see cell 18, but sees all the others. - // - P1 does NOT see cells 2 and 6, but sees the rest of them. - // - P2 does NOT see cell 13, but sees all the others. - // - P3 does NOT see cell 1, 5 and 17, but sees all the others. - // More details: - // P0 creates +1 duplicated id (middle point edge) - // P1 creates +2 duplicated ids (middle point of an edge in cell 2, cell 6 respectively). - // P2 creates +1 duplicated id (middle point edge) - // P3 creates: - // +5 duplicated ids on {I_FACE, false} cell 2 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 1] - // +1 duplicated id on edge unseen-in-P3 cell 5 - // +5 duplicated ids on {J_FACE, true} cell 13 (seen in P3) [equivalent face: {J_FACE, false} of unseen-in-P3 cell 17] - // +5 duplicated ids on {I_FACE, false} cell 18 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 17] - BOOST_CHECK( static_cast(all_point_ids_set.size()) == 145); // Desired value: 125; 20 (=1+2+1+5+1+5+5) duplicated ids. + // - P0, P1, P2 see the entire LGR (due to the argument addCornerCells = true). + // However, P3 does NOT see cell 5, but sees all the other parent cells {1,2,6,13,14,17,18}, + // In P3, {1,6,11,17,21} are (overlap) added corner cells, and {2,3,7,13, 18,22,,25,29,33} are overlap + // cells coming from overlapLayerSize = 1. + // P3 creates +7 duplicated ids on the unseen-in-P3 cell 5 + BOOST_CHECK( static_cast(all_point_ids_set.size()) == (125 + 7) ); // expected + 7 "multi-ids" // Check global id is not duplicated for points std::vector localPointIds_vec; @@ -974,12 +937,86 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr) } auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm()); const std::set allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end()); - // Total global ids in leaf grid view for points: 80 + (125 - 27) = 178 desired value; +20 duplicated ids. - BOOST_CHECK( allGlobalIds_points_set.size() == 198 ); + // Total global ids in leaf grid view for points: 80 + (125 - 27) = 178 + BOOST_CHECK( allGlobalIds_points_set.size() == (178 + 7) ); // expected + 7 "multi-ids" } } -BOOST_AUTO_TEST_CASE(call_adapt_with_args_on_distributed_grid) +BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr_II) +{ + // Create a grid + Dune::CpGrid grid; + const std::array cell_sizes = {1.0, 1.0, 1.0}; + const std::array grid_dim = {4,3,3}; + grid.createCartesian(grid_dim, cell_sizes); + std::vector parts(36); + std::vector> cells_per_rank = { {0,1,4,5,8,9,16,20,21}, + {12,13,17,24,25,28,29,32,33}, + {2,3,6,7,10,11,18,22,23}, + {14,15,19,26,27,30,31,34,35} }; + for (int rank = 0; rank < 4; ++rank) { + for (const auto& elemIdx : cells_per_rank[rank]) { + parts[elemIdx] = rank; + } + } + if(grid.comm().size()>1) + { + grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1 + // We set addCornersCells to achieve unique global ids for points belonging to refined level grids. + /** This is not enough to get unique point ids. */ + + const std::vector> cells_per_dim_vec = {{2,2,2}}; + const std::vector> startIJK_vec = {{1,0,1}}; + const std::vector> endIJK_vec = {{3,2,3}}; + const std::vector lgr_name_vec = {"LGR1"}; + // LGR1 element indices = {13,14,17,18,25,26,29,30} where + // 13,17,25,29 in rank 1, + // 18 in rank 2, + // 14,26,30 in rank 3. + // Block of cells to refine dim 2x2x2. LGR1 dim 4x4x4. + // 64 new refined cells. 5x5x5 = 125 points (only 98 = 125 - 3x3x3 parent corners new points - new global ids). + grid.addLgrsUpdateLeafView(cells_per_dim_vec, startIJK_vec, endIJK_vec, lgr_name_vec); + refinePatch_and_check(grid, cells_per_dim_vec, startIJK_vec, endIJK_vec, lgr_name_vec); + + // Check global id is not duplicated for points for each LGR + // LGR1 dim 4x4x4 -> 5x5x5 = 125 points + std::vector local_point_ids; + local_point_ids.reserve(125); // expected_point_ids in LGR1 + + for (const auto& element : elements(grid.levelGridView(1))) { + for (int corner = 0; corner < 8; ++corner) + { + const auto& point = element.subEntity<3>(corner); + local_point_ids.push_back(grid.currentData()[1]->globalIdSet().id(point)); + } + } + auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm()); + const std::set all_point_ids_set(all_point_ids.begin(), all_point_ids.end()); + + // Remark: the LGR is distributed in P1-P3 processes BUT + // - P0 does NOT see cells 25,26,30 R (even though the argument addCornerCells = true has been used). + // - P2 does NOT see cell 25. + // - P1 and P3 sees the entire LGR (due to the argument addCornerCells = true). + // 4 extra unnecessary point ids are created. + BOOST_CHECK( static_cast(all_point_ids_set.size()) == (125 + 4) ); // expected + 4 "multi-ids" + + // Check global id is not duplicated for points + std::vector localPointIds_vec; + localPointIds_vec.reserve(grid.currentData().back()->size(3)); + for (const auto& point : vertices(grid.leafGridView())) { + // Notice that all partition type points are pushed back. Selecting only interior points does not bring us to the expected value. + localPointIds_vec.push_back(grid.currentData().back()->globalIdSet().id(point)); + } + auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm()); + const std::set allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end()); + // Total global ids in leaf grid view for points: 80 + (125 - 27) = 178 + BOOST_CHECK( allGlobalIds_points_set.size() == (178 + 4) ); // expected + 4 "multi-ids" + } +} + + + +BOOST_AUTO_TEST_CASE(call_adapt_on_distributed_grid) { // Only for testing assignment of new global ids for refined entities (cells and point belonging to // refined level grids). @@ -1001,7 +1038,9 @@ BOOST_AUTO_TEST_CASE(call_adapt_with_args_on_distributed_grid) } if(grid.comm().size()>1) { - grid.loadBalance(parts); + grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1 + // We set addCornersCells to achieve unique global ids for points belonging to refined level grids. + const std::vector> cells_per_dim_vec = {{2,2,2}}; const std::vector> startIJK_vec = {{1,0,0}}; @@ -1031,7 +1070,7 @@ BOOST_AUTO_TEST_CASE(call_adapt_with_args_on_distributed_grid) } // Call adapt() on a distribted grid, marking all the elements (equivalent to call globalRefine). -BOOST_AUTO_TEST_CASE(call_adapt_on_distributed_grid) +BOOST_AUTO_TEST_CASE(call_adapt_on_full_distributed_grid) { // Only for testing assignment of new global ids for refined entities (cells and point belonging to // refined level grids).