Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

On loadBalance(...) "addCornerCells" should be called "addEdgeCells"? #808

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

On loadBalance(...) "addCornerCells" should be called "addEdgeCells"? #808

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 115 additions & 76 deletions tests/cpgrid/addLgrsOnDistributedGrid_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,9 +397,12 @@ void refinePatch_and_check(Dune::CpGrid& coarse_grid,
BOOST_CHECK( static_cast<int>(allGlobalIds_cells.size()) == global_cells_count);
BOOST_CHECK( allGlobalIds_cells.size() == allGlobalIds_cells_set.size() );

/** [Bug] Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did I get that right?
We call grid.loadBalance(parts, false, true) with addCornerCells = true to get unique IDs, correct?
Can you add that to the explanation here?
Otherwise I'd merge this PR :)

/** Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1,
which in particular means that cells that share corners or edges (and not faces) with interior cells are not considered/
seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points.*/
seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points.
To achieve unique point global ids, IN SOME CASES, we invoke loanBalance( parts, false, true) where parts determines
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo: loadBalance

the rank of each cell, false represents ownerFisrt, true represents addCornerCells. The last defaulted argument is
overlapLayerSize = 1. */

// Local/Global id sets for level grids (level 0, 1, ..., maxLevel). For level grids, local might differ from global id.
for (int level = 0; level < coarse_grid.maxLevel() +1; ++level)
Expand Down Expand Up @@ -456,6 +459,7 @@ BOOST_AUTO_TEST_CASE(threeLgrs)
const std::array<int, 3> grid_dim = {10,8,8};
grid.createCartesian(grid_dim, cell_sizes);

// Fully interior LGRs
std::vector<int> parts(640);
for (int k = 0; k < 8; ++k) {
for (int j = 0; j < 8; ++j) {
Expand Down Expand Up @@ -483,7 +487,8 @@ BOOST_AUTO_TEST_CASE(threeLgrs)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts); // ownerFirst = false, addCornerCells = false, overlapLayerSize = 1
// IT's not necessary to change the default values since the LGRs are fully interior.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}};
const std::vector<std::array<int,3>> startIJK_vec = {{0,0,0}, {0,0,3}, {3,2,2}};
Expand Down Expand Up @@ -545,14 +550,16 @@ BOOST_AUTO_TEST_CASE(atLeastOneLgr_per_process_attempt)
{12,13,17,24,25,28,29,32,33},
{2,3,6,7,10,11,18,22,23},
{14,15,19,26,27,30,31,34,35} };
// Fully interior LGRs
for (int rank = 0; rank < 4; ++rank) {
for (const auto& elemIdx : cells_per_rank[rank]) {
parts[elemIdx] = rank;
}
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts); // ownerFirst = false, addCornerCells = false, overlapLayerSize =1
// IT's not necessary to change the default values since the LGRs are fully interior.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo: It


const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}, {2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{0,1,0}, {0,0,2}, {3,2,0}, {3,0,2}};
Expand Down Expand Up @@ -607,7 +614,8 @@ BOOST_AUTO_TEST_CASE(atLeastOneLgr_per_process_attempt)
}
}

BOOST_AUTO_TEST_CASE(throw_not_fully_interior_lgr)

BOOST_AUTO_TEST_CASE(not_fully_interior_lgr)
{
// Create a grid
Dune::CpGrid grid;
Expand All @@ -620,14 +628,16 @@ BOOST_AUTO_TEST_CASE(throw_not_fully_interior_lgr)
{12,13,17,24,25,28,29,32,33},
{2,3,6,7,10,11,18,22,23},
{14,15,19,26,27,30,31,34,35} };
// NOT fully interior LGRs
for (int rank = 0; rank < 4; ++rank) {
for (const auto& elemIdx : cells_per_rank[rank]) {
parts[elemIdx] = rank;
}
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}, {3,3,3}, {4,4,4}, {2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{0,1,0}, {0,0,2}, {3,1,0}, {3,0,2}};
Expand Down Expand Up @@ -759,7 +769,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{1,0,0}};
Expand All @@ -784,29 +795,7 @@ BOOST_AUTO_TEST_CASE(distributed_lgr)
}
auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm());
const std::set<int> all_point_ids_set(all_point_ids.begin(), all_point_ids.end());
// Coarse cells 1 in rank 0 and 2 in rank 2 share an I_FACE where (LGR1_dim[1]+1)*(LGR1_dim[2]+ 1),
// here (2 +1)*(2 +1) = 9 points lying on, 4 of them being the 4 corners of the coarse I_FACE shared
// by cell 1 and cell 2. Leaving us with 9 - 4 = 5 potential duplicated ids.
//
// Global cell ids of cells to be refined = {1, 2}
// cell 1 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3}
// cell 2 = { overlap in P0, does not exist in P1, interior in P2, overlap in P3}
// Remark: the LGR is distributed in only two of the four processes, P0 and P2.
// - P0 sees the entire LGR.
// - P1 does NOT see cell 2, but sees cell 1
// - P2 does NOT see the LGR at all.
// - P3 does NOT see cell 1, but sees cell 2
// P3 creates:
// +5 duplicated ids on {I_FACE, false} cell 2 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 1]
// That means that the "unfortunate" expected point ids count is 45 (desired value) + 5 (duplicated laying on
// shared I_FACE) = 50.
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == 50);

// Current approach avoids duplicated point ids when
// 1. the LGR is distributed in P_{i_0}, ..., P_{i_n}, with n+1 < grid.comm().size(),
// AND
// 2. there is no coarse cell seen by a process P with P != P_{i_j}, j = 0, ..., n.
// Otherwise, there will be points with multiple ids.*/
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == 45); // LGR1 dim 4x2x2 -> 5x3x3 = 45 points

// Check global id is not duplicated for points
std::vector<int> localPointIds_vec;
Expand All @@ -817,8 +806,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr)
}
auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm());
const std::set<int> allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end());
// Total global ids in leaf grid view for points: 80 + (45 - 12) + 5 (undesired duplicated ids on shared I_FACE) = 118
BOOST_CHECK( allGlobalIds_points_set.size() == 118 );
// Total global ids in leaf grid view for points: 80 + (45 - 12) = 113
BOOST_CHECK( allGlobalIds_points_set.size() == 113 );
}
}

Expand All @@ -842,7 +831,8 @@ BOOST_AUTO_TEST_CASE(distributed_lgr_II)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{0,2,0}};
Expand All @@ -863,16 +853,6 @@ BOOST_AUTO_TEST_CASE(distributed_lgr_II)
auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm());
const std::set<int> all_point_ids_set(all_point_ids.begin(), all_point_ids.end());
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == 63);
// Difference with previous test case:
// Global cell ids of cells to be refined = {8, 9, 10}
// cell 8 = { interior in P0, does not exist in P1, does not exist in P2, does not exist in P3}
// cell 9 = { interior in P0, does not exist in P1, overlap in P2, does not exist in P3}
// cell 10 = { overlap in P0, does not exist in P1, interior in P2, does not exist in P3}
// Remark: the LGR is distributed only in 2 processes which are the only two processes seeing these cells.
// - P0 sees the entire LGR.
// - P1 does NOT see the LGR at all.
// - P2 does NOT see cell 8, but NO OTHER process sees cell 8 since it's fully inteior of P0.
// - P3 does NOT see the LGR at all.

// Check global id is not duplicated for points
std::vector<int> localPointIds_vec;
Expand Down Expand Up @@ -907,7 +887,10 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.
/** This is not enough to get unique point ids. */

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{1,0,0}};
const std::vector<std::array<int,3>> endIJK_vec = {{3,2,2}};
Expand All @@ -926,6 +909,7 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr)
// LGR1 dim 4x4x4 -> 5x5x5 = 125 points
std::vector<int> local_point_ids;
local_point_ids.reserve(125); // expected_point_ids in LGR1

for (const auto& element : elements(grid.levelGridView(1))) {
for (int corner = 0; corner < 8; ++corner)
{
Expand All @@ -936,34 +920,13 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr)
auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm());
const std::set<int> all_point_ids_set(all_point_ids.begin(), all_point_ids.end());

/** [Bug] Uniqueness of point global ids cannot be checked in general since current code sets overlap layer size equal to 1,
which in particular means that cells that share corners or edges (and not faces) with interior cells are not considered/
seen by the process. Therefore, depending how the LGRs are distributed, there may be "multiple ids" for the same points.*/
// Difference with previous test case:
// Global cell ids of cells to be refined = {1,2,5,6,13,14,17,18}
// cell 1 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3}
// cell 2 = { overlap in P0, does not exist in P1, interior in P2, overlap in P3}
// cell 5 = { interior in P0, overlap in P1, overlap in P2, does not exist in P3}
// cell 6 = { overlap in P0, does not exist in P1, interior in P2, does not exist in P3}
// cell 13 = { overlap in P0, interior in P1, does not exist in P2, overlap in P3}
// cell 14 = { does not exist in P0, overlap in P1, overlap in P2, interior in P3}
// cell 17 = { overlap in P0, interior in P1, overlap in P2, does not exist in P3}
// cell 18 = { does not exist in P0, overlap in P1, interior in P2, overlap in P3}
// Remark: the LGR is distributed in ALL processes BUT
// - P0 does NOT see cell 18, but sees all the others.
// - P1 does NOT see cells 2 and 6, but sees the rest of them.
// - P2 does NOT see cell 13, but sees all the others.
// - P3 does NOT see cell 1, 5 and 17, but sees all the others.
// More details:
// P0 creates +1 duplicated id (middle point edge)
// P1 creates +2 duplicated ids (middle point of an edge in cell 2, cell 6 respectively).
// P2 creates +1 duplicated id (middle point edge)
// P3 creates:
// +5 duplicated ids on {I_FACE, false} cell 2 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 1]
// +1 duplicated id on edge unseen-in-P3 cell 5
// +5 duplicated ids on {J_FACE, true} cell 13 (seen in P3) [equivalent face: {J_FACE, false} of unseen-in-P3 cell 17]
// +5 duplicated ids on {I_FACE, false} cell 18 (seen in P3) [equivalent face: {I_FACE, true} of unseen-in-P3 cell 17]
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == 145); // Desired value: 125; 20 (=1+2+1+5+1+5+5) duplicated ids.
// - P0, P1, P2 see the entire LGR (due to the argument addCornerCells = true).
// However, P3 does NOT see cell 5, but sees all the other parent cells {1,2,6,13,14,17,18},
// In P3, {1,6,11,17,21} are (overlap) added corner cells, and {2,3,7,13, 18,22,,25,29,33} are overlap
// cells coming from overlapLayerSize = 1.
// P3 creates +7 duplicated ids on the unseen-in-P3 cell 5
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == (125 + 7) ); // expected + 7 "multi-ids"

// Check global id is not duplicated for points
std::vector<int> localPointIds_vec;
Expand All @@ -974,12 +937,86 @@ BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr)
}
auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm());
const std::set<int> allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end());
// Total global ids in leaf grid view for points: 80 + (125 - 27) = 178 desired value; +20 duplicated ids.
BOOST_CHECK( allGlobalIds_points_set.size() == 198 );
// Total global ids in leaf grid view for points: 80 + (125 - 27) = 178
BOOST_CHECK( allGlobalIds_points_set.size() == (178 + 7) ); // expected + 7 "multi-ids"
}
}

BOOST_AUTO_TEST_CASE(call_adapt_with_args_on_distributed_grid)
BOOST_AUTO_TEST_CASE(distributed_in_all_ranks_lgr_II)
{
// Create a grid
Dune::CpGrid grid;
const std::array<double, 3> cell_sizes = {1.0, 1.0, 1.0};
const std::array<int, 3> grid_dim = {4,3,3};
grid.createCartesian(grid_dim, cell_sizes);
std::vector<int> parts(36);
std::vector<std::vector<int>> cells_per_rank = { {0,1,4,5,8,9,16,20,21},
{12,13,17,24,25,28,29,32,33},
{2,3,6,7,10,11,18,22,23},
{14,15,19,26,27,30,31,34,35} };
for (int rank = 0; rank < 4; ++rank) {
for (const auto& elemIdx : cells_per_rank[rank]) {
parts[elemIdx] = rank;
}
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.
/** This is not enough to get unique point ids. */

const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{1,0,1}};
const std::vector<std::array<int,3>> endIJK_vec = {{3,2,3}};
const std::vector<std::string> lgr_name_vec = {"LGR1"};
// LGR1 element indices = {13,14,17,18,25,26,29,30} where
// 13,17,25,29 in rank 1,
// 18 in rank 2,
// 14,26,30 in rank 3.
// Block of cells to refine dim 2x2x2. LGR1 dim 4x4x4.
// 64 new refined cells. 5x5x5 = 125 points (only 98 = 125 - 3x3x3 parent corners new points - new global ids).
grid.addLgrsUpdateLeafView(cells_per_dim_vec, startIJK_vec, endIJK_vec, lgr_name_vec);
refinePatch_and_check(grid, cells_per_dim_vec, startIJK_vec, endIJK_vec, lgr_name_vec);

// Check global id is not duplicated for points for each LGR
// LGR1 dim 4x4x4 -> 5x5x5 = 125 points
std::vector<int> local_point_ids;
local_point_ids.reserve(125); // expected_point_ids in LGR1

for (const auto& element : elements(grid.levelGridView(1))) {
for (int corner = 0; corner < 8; ++corner)
{
const auto& point = element.subEntity<3>(corner);
local_point_ids.push_back(grid.currentData()[1]->globalIdSet().id(point));
}
}
auto [all_point_ids, displPoint ] = Opm::allGatherv(local_point_ids, grid.comm());
const std::set<int> all_point_ids_set(all_point_ids.begin(), all_point_ids.end());

// Remark: the LGR is distributed in P1-P3 processes BUT
// - P0 does NOT see cells 25,26,30 R (even though the argument addCornerCells = true has been used).
// - P2 does NOT see cell 25.
// - P1 and P3 sees the entire LGR (due to the argument addCornerCells = true).
// 4 extra unnecessary point ids are created.
BOOST_CHECK( static_cast<int>(all_point_ids_set.size()) == (125 + 4) ); // expected + 4 "multi-ids"

// Check global id is not duplicated for points
std::vector<int> localPointIds_vec;
localPointIds_vec.reserve(grid.currentData().back()->size(3));
for (const auto& point : vertices(grid.leafGridView())) {
// Notice that all partition type points are pushed back. Selecting only interior points does not bring us to the expected value.
localPointIds_vec.push_back(grid.currentData().back()->globalIdSet().id(point));
}
auto [allGlobalIds_points, displPointLeaf ] = Opm::allGatherv(localPointIds_vec, grid.comm());
const std::set<int> allGlobalIds_points_set(allGlobalIds_points.begin(), allGlobalIds_points.end());
// Total global ids in leaf grid view for points: 80 + (125 - 27) = 178
BOOST_CHECK( allGlobalIds_points_set.size() == (178 + 4) ); // expected + 4 "multi-ids"
}
}



BOOST_AUTO_TEST_CASE(call_adapt_on_distributed_grid)
{
// Only for testing assignment of new global ids for refined entities (cells and point belonging to
// refined level grids).
Expand All @@ -1001,7 +1038,9 @@ BOOST_AUTO_TEST_CASE(call_adapt_with_args_on_distributed_grid)
}
if(grid.comm().size()>1)
{
grid.loadBalance(parts);
grid.loadBalance(parts, false, true); // ownerFirst = false, addCornerCells = true, overlapLayerSize = 1
// We set addCornersCells to achieve unique global ids for points belonging to refined level grids.


const std::vector<std::array<int,3>> cells_per_dim_vec = {{2,2,2}};
const std::vector<std::array<int,3>> startIJK_vec = {{1,0,0}};
Expand Down Expand Up @@ -1031,7 +1070,7 @@ BOOST_AUTO_TEST_CASE(call_adapt_with_args_on_distributed_grid)
}

// Call adapt() on a distribted grid, marking all the elements (equivalent to call globalRefine).
BOOST_AUTO_TEST_CASE(call_adapt_on_distributed_grid)
BOOST_AUTO_TEST_CASE(call_adapt_on_full_distributed_grid)
{
// Only for testing assignment of new global ids for refined entities (cells and point belonging to
// refined level grids).
Expand Down