compiler: Introduce rank_populated switch for MPI sparse operations o…

…n subdomains

conftest.py

@@ -177,7 +177,7 @@ def parallel(item, m):
     args = ["-n", "1", pyversion, "-m", "pytest", "-s", "--runxfail", "-qq", testname]
     if nprocs > 1:
         args.extend([":", "-n", "%d" % (nprocs - 1), pyversion, "-m", "pytest",
-                     "-s", "--runxfail", "--tb=no", "-qq", testname])
+                     "-s", "--runxfail", "--tb=no", "-qq", "--no-summary", testname])
     # OpenMPI requires an explicit flag for oversubscription. We need it as some
     # of the MPI tests will spawn lots of processes
     if mpi_distro == 'OpenMPI':

devito/ir/iet/efunc.py

@@ -65,9 +65,6 @@ def __init__(self, name, body, retval='void', parameters=None, prefix=('static',
                  dynamic_parameters=None):
         super().__init__(name, body, retval, parameters, prefix)
 
-        print("Parameters", parameters)
-        print("Dynamic parameters", dynamic_parameters)
-
         self._mapper = {}
         for i in as_tuple(dynamic_parameters):
             if i.is_Dimension:

devito/mpi/distributed.py

@@ -289,7 +289,6 @@ def all_ranges(self):
         ret = []
         for i in self.all_numb:
             # In cases where ranks are empty, return a range(0, 0)
-            # FIXME: Not sure I like this, seems hacky
             ret.append(EnrichedTuple(*[range(min(j), max(j) + 1) if len(j) > 0
                                        else range(0, 0) for j in i],
                                      getters=self.dimensions))
@@ -308,12 +307,6 @@ def _obj_neighborhood(self):
         """
         return MPINeighborhood(self.neighborhood)
 
-    @cached_property
-    def rank_populated(self):
-        """Constant symbol for a switch indicating that data is allocated on this rank"""
-        return Constant(name='rank_populated', dtype=np.int8,
-                        value=int(not(self.loc_empty)))
-
 
 class Distributor(DenseDistributor):
 
@@ -501,6 +494,8 @@ def __init__(self, subdomain):
 
         super().__init__(subdomain.shape, subdomain.dimensions)
 
+        self._subdomain_name = subdomain.name
+
         self._dimension_map = frozendict({pd: sd for pd, sd
                                           in zip(subdomain.grid.dimensions,
                                                  subdomain.dimensions)})
@@ -697,9 +692,9 @@ def neighborhood(self):
 
     @cached_property
     def rank_populated(self):
-        """Switch for a rank which is populated"""
-        # Needs to use this value but be prepared elsewhere
-        return self.parent.rank_populated
+        """Constant symbol for a switch indicating that data is allocated on this rank"""
+        return Constant(name=f'rank_populated_{self._subdomain_name}', dtype=np.int8,
+                        value=int(not(self.loc_empty)))
 
 
 class SparseDistributor(AbstractDistributor):

devito/mpi/halo_scheme.py

@@ -12,7 +12,6 @@
 from devito.symbolics.manipulation import _uxreplace_registry
 from devito.tools import (Reconstructable, Tag, as_tuple, filter_ordered, flatten,
                           frozendict, is_integer, filter_sorted, EnrichedTuple)
-from devito.types import Grid
 
 __all__ = ['HaloScheme', 'HaloSchemeEntry', 'HaloSchemeException', 'HaloTouch']
 

devito/operations/interpolators.py

@@ -13,10 +13,10 @@
 from devito.finite_differences.differentiable import Mul
 from devito.finite_differences.elementary import floor
 from devito.logger import warning
-from devito.symbolics import retrieve_function_carriers, retrieve_functions, INT, CondEq
+from devito.symbolics import retrieve_function_carriers, retrieve_functions, INT
 from devito.tools import as_tuple, flatten, filter_ordered, Pickable
 from devito.types import (ConditionalDimension, Eq, Inc, Evaluable, Symbol,
-                          CustomDimension, SubFunction, Constant)
+                          CustomDimension, SubFunction)
 from devito.types.utils import DimensionTuple
 
 __all__ = ['LinearInterpolator', 'PrecomputedInterpolator', 'SincInterpolator']
@@ -72,15 +72,7 @@ def adjust_interp_indices(subdomain, mapper, smapper, cdmapper):
     # Insert a check to catch cases where interpolation/injection is
     # into an empty rank. This depends on the injection field or interpolated
     # expression, and so must be inserted here.
-    # FIXME: The resultant switch isn't super obvious in generated code and
-    # results in different code between ranks.
-    # rank_populated = Constant(name='rank_populated', dtype=np.int8,
-    #                           value=int(not(subdomain.distributor.loc_empty)))
     rank_populated = subdomain.distributor.rank_populated
-    # FIXME: This could be checked for the rank before looping over sparse
-    # footprint -> Can one nest ConditionalDimensions?
-    # FIXME: This is also only needed if the function is distributed
-    # rank_populated = CondEq(int(subdomain.distributor.loc_empty), 0)
 
     for d, cd in list(mapper.items()):
         cond = cd.condition.subs(subs)

devito/passes/iet/misc.py

@@ -239,7 +239,7 @@ def minimize_symbols(iet):
     """
     iet = remove_redundant_moddims(iet)
     iet = simplify_iteration_headers(iet)
-    # iet = abridge_dim_names(iet)
+    iet = abridge_dim_names(iet)
 
     return iet, {}
 

tests/test_interpolation.py

@@ -990,7 +990,6 @@ def test_inject_subdomain(self):
         assert np.all(np.isclose(f1.data, check1))
 
     @pytest.mark.parallel(mode=4)
-    # @pytest.mark.parallel(mode=2)
     def test_interpolate_subdomain_mpi(self, mode):
         """
         Test interpolation off of a Function defined on a SubDomain with MPI.
@@ -1034,9 +1033,6 @@ def test_interpolate_subdomain_mpi(self, mode):
 
         op = Operator([rec0, rec1, rec2, rec3])
 
-        if grid.distributor.myrank == 0:
-            print(op.ccode)
-
         op.apply()
 
         if grid.distributor.myrank == 0:
@@ -1055,15 +1051,10 @@ def test_interpolate_subdomain_mpi(self, mode):
             assert np.all(np.isclose(sr2.data, [0., 16.5]))
             assert np.all(np.isclose(sr3.data, [30., 0.]))
         elif grid.distributor.myrank == 3:
-            print(sr0.data)
-            print(sr1.data)
-            print(sr2.data)
-            print(sr3.data)
-            # FIXME: Fails on this last rank
-            assert np.all(np.isclose(sr0.data, [6.75, 0.]))  # Fails
-            assert np.all(np.isclose(sr1.data, [0., 48.75]))  # Fails
-            assert np.all(np.isclose(sr2.data, [0., 112.5]))  # Fails
-            assert np.all(np.isclose(sr3.data, [0., 0.]))  # Works
+            assert np.all(np.isclose(sr0.data, [6.75, 0.]))
+            assert np.all(np.isclose(sr1.data, [0., 48.75]))
+            assert np.all(np.isclose(sr2.data, [0., 112.5]))
+            assert np.all(np.isclose(sr3.data, [0., 0.]))
 
     @pytest.mark.parallel(mode=4)
     def test_inject_subdomain_mpi(self, mode):