Skip to content

Commit

Permalink
USe addCornerCell = true when calling loadBalance(...)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aritorto
aritorto committed Dec 3, 2024
1 parent 31946cc commit 71f3c18
Showing 1 changed file with 114 additions and 75 deletions.
189 changes: 114 additions & 75 deletions tests/cpgrid/addLgrsOnDistributedGrid_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,9 +398,12 @@ void refinePatch_and_check(Dune::CpGrid& coarse_grid,
BOOST_CHECK( static_cast<int>(allGlobalIds_cells.size()) == global_cells_count);
BOOST_CHECK( allGlobalIds_cells.size() == allGlobalIds_cells_set.size() );

/** [Bug] Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1,
/** Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1,
which in particular means that cells that share corners or edges (and not faces) with interior cells are not considered/
seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points.*/
seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points.
To achieve unique point global ids, IN SOME CASES, we invoke loanBalance( parts, false, true) where parts determines
the rank of each cell, false represents ownerFisrt, true represents addCornerCells. The last defaulted argument is
overlapLayerSize = 1. */

// Local/Global id sets for level grids (level 0, 1, ..., maxLevel). For level grids, local might differ from global id.
for (int level = 0; level < coarse_grid.maxLevel() +1; ++level)
Expand Down Expand Up @@ -457,6 +460,7 @@ BOOST_AUTO_TEST_CASE(threeLgrs)
const std::array<int, 3> grid_dim = {10,8,8};
grid.createCartesian(grid_dim, cell_sizes);

// Fully interior LGRs
std::vector<int> parts(640);
for (int k = 0; k < 8; ++k) {
for (int j = 0; j < 8; ++j) {
Expand Down Expand Up @@ -484,7 +488,8 @@ BOOST_AUTO_TEST_CASE(threeLgrs)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts); // ownerFirst = false, addCornerCells = false, overlapLayerSize = 1
// IT's not necessary to change the default values since the LGRs are fully interior.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}};
const std::vector<std::array<int,3>> startIJK_vec = {{0,0,0}, {0,0,3}, {3,2,2}};
Expand Down Expand Up @@ -546,14 +551,16 @@ BOOST_AUTO_TEST_CASE(atLeastOneLgr_per_process_attempt)
{12,13,17,24,25,28,29,32,33},
{2,3,6,7,10,11,18,22,23},
{14,15,19,26,27,30,31,34,35} };
// Fully interior LGRs
for (int rank = 0; rank < 4; ++rank) {
for (const auto& elemIdx : cells_per_rank[rank]) {
parts[elemIdx] = rank;
}
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts); // ownerFirst = false, addCornerCells = false, overlapLayerSize =1
// IT's not necessary to change the default values since the LGRs are fully interior.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}, {2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{0,1,0}, {0,0,2}, {3,2,0}, {3,0,2}};
Expand All @@ -572,7 +579,7 @@ BOOST_AUTO_TEST_CASE(atLeastOneLgr_per_process_attempt)
// Total global ids in leaf grid view for cells: 36-(6 marked cells) + 16 + 27 + 64 + 16 = 153
// Total global ids in leaf grid view for points: 80 + 33 + 56 + 117 + 33 = 319
refinePatch_and_check(grid, cells_per_dim_vec, startIJK_vec, endIJK_vec, lgr_name_vec);

// Check global id is not duplicated for points for each LGR
// LGR1 dim 2x4x2 -> 3x5x3 = 45 points
// LGR2 dim 3x3x3 -> 4x4x4 = 64 points
Expand Down Expand Up @@ -608,7 +615,8 @@ BOOST_AUTO_TEST_CASE(atLeastOneLgr_per_process_attempt)
}
}

BOOST_AUTO_TEST_CASE(throw_not_fully_interior_lgr)

BOOST_AUTO_TEST_CASE(not_fully_interior_lgr)
{
// Create a grid
Dune::CpGrid grid;
Expand All @@ -621,14 +629,16 @@ BOOST_AUTO_TEST_CASE(throw_not_fully_interior_lgr)
{12,13,17,24,25,28,29,32,33},
{2,3,6,7,10,11,18,22,23},
{14,15,19,26,27,30,31,34,35} };
// NOT fully interior LGRs
for (int rank = 0; rank < 4; ++rank) {
for (const auto& elemIdx : cells_per_rank[rank]) {
parts[elemIdx] = rank;
}
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}, {2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{0,1,0}, {0,0,2}, {3,1,0}, {3,0,2}};
Expand Down Expand Up @@ -685,6 +695,7 @@ BOOST_AUTO_TEST_CASE(globalRefine2)
const std::array<double, 3> cell_sizes = {1.0, 1.0, 1.0};
const std::array<int, 3> grid_dim = {4,3,3};
grid.createCartesian(grid_dim, cell_sizes);

// Distribute the grid
if(grid.comm().size()>1)
{
Expand Down Expand Up @@ -714,7 +725,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{1,0,0}};
Expand All @@ -739,29 +751,7 @@ BOOST_AUTO_TEST_CASE(distributed_lgr)
}
auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm());
const std::set<int> all_point_ids_set(all_point_ids.begin(), all_point_ids.end());
// Coarse cells 1 in rank 0 and 2 in rank 2 share an I_FACE where (LGR1_dim[1]+1)*(LGR1_dim[2]+ 1),
// here (2 +1)*(2 +1) = 9 points lying on, 4 of them being the 4 corners of the coarse I_FACE shared
// by cell 1 and cell 2. Leaving us with 9 - 4 = 5 potential duplicated ids.
//
// Global cell ids of cells to be refined = {1, 2}
// cell 1 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3}
// cell 2 = { overlap in P0, does not exist in P1, interior in P2, overlap in P3}
// Remark: the LGR is distributed in only two of the four processes, P0 and P2.
// - P0 sees the entire LGR.
// - P1 does NOT see cell 2, but sees cell 1
// - P2 does NOT see the LGR at all.
// - P3 does NOT see cell 1, but sees cell 2
// P3 creates:
// +5 duplicated ids on {I_FACE, false} cell 2 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 1]
// That means that the "unfortunate" expected point ids count is 45 (desired value) + 5 (duplicated laying on
// shared I_FACE) = 50.
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == 50);

/** Current approach avoids duplicated point ids when
// 1. the LGR is distributed in P_{i_0}, ..., P_{i_n}, with n+1 < grid.comm().size(),
// AND
// 2. there is no coarse cell seen by a process P with P != P_{i_j}, j = 0, ..., n.
// Otherwise, there will be points with multiple ids.*/
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == 45); // LGR1 dim 4x2x2 -> 5x3x3 = 45 points

// Check global id is not duplicated for points
std::vector<int> localPointIds_vec;
Expand All @@ -772,8 +762,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr)
}
auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm());
const std::set<int> allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end());
// Total global ids in leaf grid view for points: 80 + (45 - 12) + 5 (undesired duplicated ids on shared I_FACE) = 118
BOOST_CHECK( allGlobalIds_points_set.size() == 118 );
// Total global ids in leaf grid view for points: 80 + (45 - 12) = 113
BOOST_CHECK( allGlobalIds_points_set.size() == 113 );
}
}

Expand All @@ -797,7 +787,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr_II)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{0,2,0}};
Expand All @@ -818,16 +809,6 @@ BOOST_AUTO_TEST_CASE(distributed_lgr_II)
auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm());
const std::set<int> all_point_ids_set(all_point_ids.begin(), all_point_ids.end());
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == 63);
// Difference with previous test case:
// Global cell ids of cells to be refined = {8, 9, 10}
// cell 8 = { interior in P0, does not exist in P1, does not exist in P2, does not exist in P3}
// cell 9 = { interior in P0, does not exist in P1, overlap in P2, does not exist in P3}
// cell 10 = { overlap in P0, does not exist in P1, interior in P2, does not exist in P3}
// Remark: the LGR is distributed only in 2 processes which are the only two processes seeing these cells.
// - P0 sees the entire LGR.
// - P1 does NOT see the LGR at all.
// - P2 does NOT see cell 8, but NO OTHER process sees cell 8 since it's fully inteior of P0.
// - P3 does NOT see the LGR at all.

// Check global id is not duplicated for points
std::vector<int> localPointIds_vec;
Expand Down Expand Up @@ -862,7 +843,10 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.
/** This is not enough to get unique point ids. */

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{1,0,0}};
const std::vector<std::array<int,3>> endIJK_vec = {{3,2,2}};
Expand All @@ -881,6 +865,7 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr)
// LGR1 dim 4x4x4 -> 5x5x5 = 125 points
std::vector<int> local_point_ids;
local_point_ids.reserve(125); // expected_point_ids in LGR1

for (const auto& element : elements(grid.levelGridView(1))) {
for (int corner = 0; corner < 8; ++corner)
{
Expand All @@ -891,34 +876,85 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr)
auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm());
const std::set<int> all_point_ids_set(all_point_ids.begin(), all_point_ids.end());

/** [Bug] Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1,
which in particular means that cells that share corners or edges (and not faces) with interior cells are not considered/
seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points.*/
// Difference with previous test case:
// Global cell ids of cells to be refined = {1,2,5,6,13,14,17,18}
// cell 1 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3}
// cell 2 = { overlap in P0, does not exist in P1, interior in P2, overlap in P3}
// cell 5 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3}
// cell 6 = { overlap in P0, does not exist in P1, interior in P2, does not exist in P3}
// cell 13 = { overlap in P0, interior in P1, does not exist in P2, overlap in P3}
// cell 14 = { does not exist in P0, overlap in P1, overlap in P2, interior in P3}
// cell 17 = { overlap in P0, interior in P1, overlap in P2, does not exist in P3}
// cell 18 = { does not exist in P0, overlap in P1, interior in P2, overlap in P3}
// Remark: the LGR is distributed in ALL processes BUT
// - P0 does NOT see cell 18, but sees all the others.
// - P1 does NOT see cells 2 and 6, but sees the rest of them.
// - P2 does NOT see cell 13, but sees all the others.
// - P3 does NOT see cell 1, 5 and 17, but sees all the others.
// More details:
// P0 creates +1 duplicated id (middle point edge)
// P1 creates +2 duplicated ids (middle point of an edge in cell 2, cell 6 respectively).
// P2 creates +1 duplicated id (middle point edge)
// P3 creates:
// +5 duplicated ids on {I_FACE, false} cell 2 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 1]
// +1 duplicated id on edge unseen-in-P3 cell 5
// +5 duplicated ids on {J_FACE, true} cell 13 (seen in P3) [equivalent face: {J_FACE, false} of unseen-in-P3 cell 17]
// +5 duplicated ids on {I_FACE, false} cell 18 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 17]
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == 145); // Desired value: 125; 20 (=1+2+1+5+1+5+5) duplicated ids.
// - P0, P1, P2 see the entire LGR (due to the argument addCornerCells = true).
// However, P3 does NOT see cell 5, but sees all the other parent cells {1,2,6,13,14,17,18},
// In P3, {1,6,11,17,21} are (overlap) added corner cells, and {2,3,7,13, 18,22,,25,29,33} are overlap
// cells coming from overlapLayerSize = 1.
// P3 creates +7 duplicated ids on the unseen-in-P3 cell 5
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == (125 + 7) ); // expected + 7 "multi-ids"

// Check global id is not duplicated for points
std::vector<int> localPointIds_vec;
localPointIds_vec.reserve(grid.currentData().back()->size(3));
for (const auto& point : vertices(grid.leafGridView())) {
// Notice that all partition type points are pushed back. Selecting only interior points does not bring us to the expected value.
localPointIds_vec.push_back(grid.currentData().back()->globalIdSet().id(point));
}
auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm());
const std::set<int> allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end());
// Total global ids in leaf grid view for points: 80 + (125 - 27) = 178
BOOST_CHECK( allGlobalIds_points_set.size() == (178 + 7) ); // expected + 7 "multi-ids"
}
}

BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr_II)
{
// Create a grid
Dune::CpGrid grid;
const std::array<double, 3> cell_sizes = {1.0, 1.0, 1.0};
const std::array<int, 3> grid_dim = {4,3,3};
grid.createCartesian(grid_dim, cell_sizes);
std::vector<int> parts(36);
std::vector<std::vector<int>> cells_per_rank = { {0,1,4,5,8,9,16,20,21},
{12,13,17,24,25,28,29,32,33},
{2,3,6,7,10,11,18,22,23},
{14,15,19,26,27,30,31,34,35} };
for (int rank = 0; rank < 4; ++rank) {
for (const auto& elemIdx : cells_per_rank[rank]) {
parts[elemIdx] = rank;
}
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.
/** This is not enough to get unique point ids. */

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{1,0,1}};
const std::vector<std::array<int,3>> endIJK_vec = {{3,2,3}};
const std::vector<std::string> lgr_name_vec = {"LGR1"};
// LGR1 element indices = {13,14,17,18,25,26,29,30} where
// 13,17,25,29 in rank 1,
// 18 in rank 2,
// 14,26,30 in rank 3.
// Block of cells to refine dim 2x2x2. LGR1 dim 4x4x4.
// 64 new refined cells. 5x5x5 = 125 points (only 98 = 125 - 3x3x3 parent corners new points - new global ids).
grid.addLgrsUpdateLeafView(cells_per_dim_vec, startIJK_vec, endIJK_vec, lgr_name_vec);
refinePatch_and_check(grid, cells_per_dim_vec, startIJK_vec, endIJK_vec, lgr_name_vec);

// Check global id is not duplicated for points for each LGR
// LGR1 dim 4x4x4 -> 5x5x5 = 125 points
std::vector<int> local_point_ids;
local_point_ids.reserve(125); // expected_point_ids in LGR1

for (const auto& element : elements(grid.levelGridView(1))) {
for (int corner = 0; corner < 8; ++corner)
{
const auto& point = element.subEntity<3>(corner);
local_point_ids.push_back(grid.currentData()[1]->globalIdSet().id(point));
}
}
auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm());
const std::set<int> all_point_ids_set(all_point_ids.begin(), all_point_ids.end());

// Remark: the LGR is distributed in P1-P3 processes BUT
// - P0 does NOT see cells 25,26,30 R (even though the argument addCornerCells = true has been used).
// - P2 does NOT see cell 25.
// - P1 and P3 sees the entire LGR (due to the argument addCornerCells = true).
// 4 extra unnecessary point ids are created.
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == (125 + 4) ); // expected + 4 "multi-ids"

// Check global id is not duplicated for points
std::vector<int> localPointIds_vec;
Expand All @@ -929,12 +965,13 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr)
}
auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm());
const std::set<int> allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end());
// Total global ids in leaf grid view for points: 80 + (125 - 27) = 178 desired value; +20 duplicated ids.
BOOST_CHECK( allGlobalIds_points_set.size() == 198 );
// Total global ids in leaf grid view for points: 80 + (125 - 27) = 178
BOOST_CHECK( allGlobalIds_points_set.size() == (178 + 4) ); // expected + 4 "multi-ids"
}
}



BOOST_AUTO_TEST_CASE(call_adapt_on_distributed_grid)
{
// Only for testing assignment of new global ids for refined entities (cells and point belonging to
Expand All @@ -957,7 +994,9 @@ BOOST_AUTO_TEST_CASE(call_adapt_on_distributed_grid)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.

// grid.adapt(); It does not throw an exeption. Note: adapt() implements global refinement.
//
// The following test fails. TODO: Move the assignment of global IDs for refined level grids and the leaf grid view
Expand Down

0 comments on commit 71f3c18

Please sign in to comment.