Merge pull request #681 from slayoo/freezing

cleanups & refactors, enabling pylint red flags; fix condensation schedule logic

.github/workflows/julia.yml

@@ -24,6 +24,7 @@ jobs:
       - run: pip install exdown pytest
       - run: python -c "import exdown; code=exdown.extract('README.md'); f=open('readme.jl', 'w'); f.writelines(block.code for block in code if block.syntax=='Julia'); f.close()"
       - uses: julia-actions/setup-julia@v1
+      - run: cat -n readme.jl
       - run: julia readme.jl
       - run: sed -i 's/CPU/GPU/g' readme.jl
       - run: julia readme.jl

.github/workflows/pylint.yml

@@ -26,4 +26,4 @@ jobs:
         pip install -r test-time-requirements.txt
     - name: Analysing the code with pylint
       run: |
-        pylint --exit-zero --disable=fixme $(git ls-files '*.py')
+        pylint --disable=fixme,invalid-name,missing-function-docstring,missing-class-docstring,missing-module-docstring,too-many-arguments,too-many-locals,too-many-instance-attributes,too-few-public-methods,protected-access,too-many-statements,unused-argument,duplicate-code,too-many-branches,cyclic-import $(git ls-files '*.py')

PySDM/backends/__init__.py

@@ -50,7 +50,7 @@ def _cuda_is_available():
 
     import numpy as np
 
-    from PySDM.backends.thrust_rtc import ThrustRTC
+    from PySDM.backends.thrust_rtc import ThrustRTC  # pylint: disable=ungrouped-imports
     ThrustRTC.ENABLE = False
 
     class Random:  # pylint: disable=too-few-public-methods

PySDM/storages/pairwise_storage.py → PySDM/backends/impl_common/pairwise_storage.py

@@ -1,13 +1,6 @@
-from .storage_utils import StorageSignature
-
-
 def make_PairwiseStorage(backend):
 
     class PairwiseStorage(backend.Storage):
-
-        def __init__(self, signature: StorageSignature):
-            super().__init__(signature)
-
         @staticmethod
         def empty(shape, dtype):
             result = PairwiseStorage(backend.Storage._get_empty_data(shape, dtype))

PySDM/backends/impl_numba/storage.py

@@ -1,7 +1,8 @@
 import numpy as np
 from PySDM.backends.impl_numba import storage_impl as impl
-from PySDM.storages.storage_utils import get_data_from_ndarray, StorageSignature, StorageBase, empty
-
+from PySDM.backends.impl_common.storage_utils import (
+    get_data_from_ndarray, StorageSignature, StorageBase, empty
+)
 
 class Storage(StorageBase):
 

PySDM/backends/impl_numba/test_helpers/bdf.py

@@ -32,13 +32,13 @@ def _bdf_condensation(particulator, rtol_x, rtol_thd, counters, RH_max, success,
         n=particulator.attributes['n'].data,
         vdry=particulator.attributes["dry volume"].data,
         idx=particulator.attributes._ParticleAttributes__idx.data,
-        rhod=particulator.env["rhod"].data,
-        thd=particulator.env["thd"].data,
-        qv=particulator.env["qv"].data,
-        dv_mean=particulator.env.dv,
-        prhod=particulator.env.get_predicted("rhod").data,
-        pthd=particulator.env.get_predicted("thd").data,
-        pqv=particulator.env.get_predicted("qv").data,
+        rhod=particulator.environment["rhod"].data,
+        thd=particulator.environment["thd"].data,
+        qv=particulator.environment["qv"].data,
+        dv_mean=particulator.environment.dv,
+        prhod=particulator.environment.get_predicted("rhod").data,
+        pthd=particulator.environment.get_predicted("thd").data,
+        pqv=particulator.environment.get_predicted("qv").data,
         kappa=particulator.attributes["kappa"].data,
         f_org=particulator.attributes["dry volume organic fraction"].data,
         rtol_x=rtol_x,

PySDM/backends/impl_thrust_rtc/conf.py

@@ -2,12 +2,12 @@
 
 if not fakeThrustRTC:
     # noinspection PyUnresolvedReferences
-    import ThrustRTC as trtc
+    import ThrustRTC as trtc  # pylint: disable=unused-import
     # noinspection PyUnresolvedReferences
-    import CURandRTC as rndrtc
+    import CURandRTC as rndrtc  # pylint: disable=unused-import
 else:
     # noinspection PyUnresolvedReferences
-    from .test_helpers.fake_thrust_rtc import FakeThrustRTC as trtc
+    from .test_helpers.fake_thrust_rtc import FakeThrustRTC as trtc  # pylint: disable=unused-import
     # noinspection PyUnresolvedReferences
     rndrtc = None
 

PySDM/backends/impl_thrust_rtc/methods/__init__.py

@@ -1 +0,0 @@
-

PySDM/backends/impl_thrust_rtc/methods/collisions_methods.py

@@ -98,7 +98,7 @@ def __init__(self):
             '''
             if (i == 0) {
                 cell_start[cell_id[idx[0]]] = 0;
-            } 
+            }
             else {
                 auto cell_id_curr = cell_id[idx[i]];
                 auto cell_id_next = cell_id[idx[i + 1]];

PySDM/backends/impl_thrust_rtc/methods/condensation_methods.py

@@ -3,7 +3,7 @@
 from PySDM.backends.impl_thrust_rtc.conf import NICE_THRUST_FLAGS
 from PySDM.backends.impl_thrust_rtc.nice_thrust import nice_thrust
 from PySDM.backends.impl_thrust_rtc.bisection import BISECTION
-from PySDM.storages.storage_utils import StorageBase
+from PySDM.backends.impl_common.storage_utils import StorageBase
 from ..conf import trtc
 from ..methods.thrust_rtc_backend_methods import ThrustRTCBackendMethods
 

PySDM/backends/impl_thrust_rtc/methods/index_methods.py

@@ -39,8 +39,8 @@ def shuffle_global(idx, length, u01):
     @staticmethod
     @nice_thrust(**NICE_THRUST_FLAGS)
     def shuffle_local(idx, u01, cell_start):
-        raise NotImplementedError("Unpredictable behavior")  # TODO #358
-        # IndexMethods.__shuffle_local_body.launch_n(cell_start.size() - 1, [cell_start, u01, idx])
+        raise Exception("Unpredictable behavior")  # TODO #358
+        IndexMethods.__shuffle_local_body.launch_n(cell_start.size() - 1, [cell_start, u01, idx])  # pylint: disable=unreachable
 
     @staticmethod
     @nice_thrust(**NICE_THRUST_FLAGS)

PySDM/backends/impl_thrust_rtc/methods/thrust_rtc_backend_methods.py

@@ -1,18 +1,19 @@
+from typing import Callable, Optional
 from ...impl_common.backend_methods import BackendMethods
 
 
 class ThrustRTCBackendMethods(BackendMethods):
     def __init__(self):
         super().__init__()
         if not hasattr(self, '_conv_function'):
-            self._conv_function = None
+            self._conv_function: Optional[Callable] = None
         if not hasattr(self, '_real_type'):
             self._real_type = None
         if not hasattr(self, '_np_dtype'):
             self._np_dtype = None
 
     def _get_floating_point(self, number):
-        return self._conv_function(number)
+        return self._conv_function(number)  # pylint: disable=not-callable
 
     def _get_c_type(self):
         return self._real_type

PySDM/backends/impl_thrust_rtc/storage.py

@@ -1,5 +1,7 @@
 import numpy as np
-from PySDM.storages.storage_utils import get_data_from_ndarray, StorageSignature, StorageBase, empty
+from PySDM.backends.impl_common.storage_utils import (
+    get_data_from_ndarray, StorageSignature, StorageBase, empty
+)
 from PySDM.backends.impl_thrust_rtc.conf import trtc
 from PySDM.backends.impl_thrust_rtc.nice_thrust import nice_thrust
 from PySDM.backends.impl_thrust_rtc.conf import NICE_THRUST_FLAGS
@@ -24,7 +26,12 @@ def thrust(obj):
         @staticmethod
         @nice_thrust(**NICE_THRUST_FLAGS)
         def add(output, addend):
-            trtc.Transform_Binary(Impl.thrust(addend), Impl.thrust(output), Impl.thrust(output), trtc.Plus())
+            trtc.Transform_Binary(
+                Impl.thrust(addend),
+                Impl.thrust(output),
+                Impl.thrust(output),
+                trtc.Plus()
+            )
 
         @staticmethod
         @nice_thrust(**NICE_THRUST_FLAGS)
@@ -39,7 +46,10 @@ def amin(data):
         @staticmethod
         @nice_thrust(**NICE_THRUST_FLAGS)
         def row_modulo(output, divisor):
-            Impl.__row_modulo_body.launch_n(output.shape[0], Impl.thrust((output, divisor, output.shape[1])))
+            Impl.__row_modulo_body.launch_n(
+                output.shape[0],
+                Impl.thrust((output, divisor, output.shape[1]))
+            )
 
         __floor_body = trtc.For(('arr',), "i", '''
                 if (arr[i] >= 0) {
@@ -74,7 +84,10 @@ def floor(output):
         @staticmethod
         @nice_thrust(**NICE_THRUST_FLAGS)
         def floor_out_of_place(output, input_data):
-            Impl.__floor_out_of_place_body.launch_n(output.shape[0], Impl.thrust((output, input_data)))
+            Impl.__floor_out_of_place_body.launch_n(
+                output.shape[0],
+                Impl.thrust((output, input_data))
+            )
 
         __multiply_elementwise_body = trtc.For(('output', 'multiplier'), "i", '''
                 output[i] *= multiplier[i];
@@ -225,7 +238,10 @@ def __getitem__(self, item):
                 assert item[1].start is None or item[1].start == 0
                 assert item[1].stop is None or item[1].stop == self.shape[1]
                 assert item[1].step is None or item[1].step == 1
-                result_data = self.data.range(self.shape[1] * item[0], self.shape[1] * (item[0] + 1))
+                result_data = self.data.range(
+                    self.shape[1] * item[0],
+                    self.shape[1] * (item[0] + 1)
+                )
                 result = Storage(StorageSignature(result_data, (*self.shape[1:],), self.dtype))
             else:
                 result = self.to_ndarray()[item]

PySDM/backends/impl_thrust_rtc/test_helpers/fake_thrust_rtc.py

@@ -1,3 +1,4 @@
+# pylint: disable=no-member,unsupported-assignment-operation,unsubscriptable-object,no-value-for-parameter
 import types
 import warnings
 

PySDM/backends/thrust_rtc.py

@@ -18,7 +18,7 @@
 from PySDM.backends.impl_thrust_rtc.conf import trtc
 
 
-class ThrustRTC(  # pylint: disable=duplicate-code
+class ThrustRTC(  # pylint: disable=duplicate-code,too-many-ancestors
     CollisionsMethods,
     PairMethods,
     IndexMethods,

PySDM/dynamics/ambient_thermodynamics.py

@@ -12,4 +12,4 @@ def register(self, builder):
         self.particulator = builder.particulator
 
     def __call__(self):
-        self.particulator.env.sync()
+        self.particulator.environment.sync()

PySDM/dynamics/condensation.py

@@ -71,7 +71,7 @@ def register(self, builder):
     def __call__(self):
         if self.enable:
             if self.schedule == 'dynamic':
-                self.condensation_cell_order = np.argsort(self.counters['n_substeps'])
+                self.cell_order = np.argsort(self.counters['n_substeps'])
             elif self.schedule == 'static':
                 pass
             else:

PySDM/dynamics/eulerian_advection.py

@@ -13,8 +13,8 @@ def register(self, builder):
         self.particulator = builder.particulator
 
     def __call__(self):
-        self.particulator.env.get_predicted('qv').download(
-            self.particulator.env.get_qv(), reshape=True)
-        self.particulator.env.get_predicted('thd').download(
-            self.particulator.env.get_thd(), reshape=True)
+        self.particulator.environment.get_predicted('qv').download(
+            self.particulator.environment.get_qv(), reshape=True)
+        self.particulator.environment.get_predicted('thd').download(
+            self.particulator.environment.get_thd(), reshape=True)
         self.solvers()

PySDM/environments/parcel.py

@@ -33,6 +33,7 @@ def __init__(
 
         self.formulae = None
         self.dql = None
+        self.params = None
 
     @property
     def dv(self):

PySDM/exporters/vtk_exporter.py

@@ -96,33 +96,33 @@ def export_products(self, particulator):
                 print("Exporting Products to vtk, path: " + path)
             payload = {}
 
-            if particulator.mesh.dimension == 2:
-                data_shape = (particulator.mesh.grid[0], particulator.mesh.grid[1], 1)
-
-                for k in particulator.products.keys():
-                    v = particulator.products[k].get()
-
-                    if isinstance(v, np.ndarray):
-                        if v.shape == particulator.mesh.grid:
-                            payload[k] = v.T[:, :, np.newaxis]
-                        else:
-                            if self.verbose:
-                                print(f'{k} shape {v.shape} not equals data shape {data_shape}'
-                                      f' and will not be exported', file=sys.stderr)
-                    elif isinstance(v, numbers.Number):
-                        if self.verbose:
-                            print(f'{k} is a Number and will not be exported', file=sys.stderr)
+            if particulator.mesh.dimension != 2:
+                raise NotImplementedError("Only 2 dimensions data is supported at the moment.")
+
+            data_shape = (particulator.mesh.grid[0], particulator.mesh.grid[1], 1)
+
+            for k in particulator.products.keys():
+                v = particulator.products[k].get()
+
+                if isinstance(v, np.ndarray):
+                    if v.shape == particulator.mesh.grid:
+                        payload[k] = v.T[:, :, np.newaxis]
                     else:
                         if self.verbose:
-                            print(f'{k} export is not possible', file=sys.stderr)
-
-                x, y, z = np.mgrid[
-                    :particulator.mesh.grid[0] + 1,
-                    :particulator.mesh.grid[1] + 1,
-                    :1
-                ]
-            else:
-                raise NotImplementedError("Only 2 dimensions data is supported at the moment.")
+                            print(f'{k} shape {v.shape} not equals data shape {data_shape}'
+                                  f' and will not be exported', file=sys.stderr)
+                elif isinstance(v, numbers.Number):
+                    if self.verbose:
+                        print(f'{k} is a Number and will not be exported', file=sys.stderr)
+                else:
+                    if self.verbose:
+                        print(f'{k} export is not possible', file=sys.stderr)
+
+            x, y, z = np.mgrid[
+                :particulator.mesh.grid[0] + 1,
+                :particulator.mesh.grid[1] + 1,
+                :1
+            ]
 
             gridToVTK(path, x, y, z, cellData = payload)
         else:

PySDM/impl/mesh.py

@@ -9,15 +9,14 @@ def __init__(self, grid, size):
         self.strides = Mesh.__strides(grid)
         self.n_cell = int(np.prod(grid))
         self.dv = np.prod((np.array(size) / np.array(grid)))
-        self.__dimension = len(self.grid)
 
     @property
     def dz(self):
         return self.size[-1] / self.grid[-1]
 
     @property
     def dimension(self):
-        return self.__dimension
+        return 0 if self.grid == (1,) else len(self.grid)
 
     @property
     def dim(self):
@@ -27,7 +26,6 @@ def dim(self):
     def mesh_0d(dv=None):
         mesh = Mesh((1,), ())
         mesh.dv = dv
-        mesh.__dimension = 0
         return mesh
 
     @staticmethod

PySDM/impl/particle_attributes.py

@@ -13,12 +13,10 @@ def __init__(
             cell_start,
             attributes: Dict[str, Attribute]
     ):
-        self.__n_sd = particulator.n_sd
         self.__valid_n_sd = particulator.n_sd
         self.healthy = True
         self.__healthy_memory = particulator.Storage.from_ndarray(np.full((1,), 1))
         self.__idx = idx
-        self.__strides = particulator.Storage.from_ndarray(particulator.mesh.strides)
 
         self.__extensive_attribute_storage = extensive_attribute_storage
         self.__extensive_keys = extensive_keys

PySDM/impl/particle_attributes_factory.py

@@ -29,7 +29,7 @@ def attributes(particulator, req_attr, attributes):
             idx, (len(maximum_attr), particulator.n_sd), float)
 
         for attr in req_attr.values():
-            if isinstance(attr, DerivedAttribute) or isinstance(attr, DummyAttribute):
+            if isinstance(attr, (DerivedAttribute, DummyAttribute)):
                 attr.allocate(idx)
             if isinstance(attr, DummyAttribute) and attr.name in attributes:
                 raise ValueError(f"attribute '{attr.name}' indicated as dummy"
@@ -44,9 +44,9 @@ def helper(req_attr, all_attr, names, data, keys):
                 req_attr[attr].set_data(data[i, :])
                 try:
                     req_attr[attr].init(all_attr[attr])
-                except KeyError:
+                except KeyError as err:
                     raise ValueError(f"attribute '{attr}' required by one of the dynamics"
-                                     f" but no initial values given")
+                                     f" but no initial values given") from err
 
         helper(req_attr, attributes, extensive_attr, extensive_attribute_storage, extensive_keys)
         helper(req_attr, attributes, maximum_attr, maximum_attributes, maximum_keys)

PySDM/products/product.py → PySDM/impl/product.py

@@ -1,5 +1,5 @@
 import numpy as np
-from ..environments._moist import _Moist
+from PySDM.environments._moist import _Moist
 
 
 class Product:
@@ -96,7 +96,7 @@ def __init__(self, **args):
     def register(self, builder):
         super().register(builder)
         self.particulator.observers.append(self)
-        self.environment = builder.particulator.env
+        self.environment = builder.particulator.environment
         self.source = self.environment[self._name]
 
     def notify(self):

PySDM/initialisation/r_wet_init.py

@@ -6,7 +6,7 @@
 from ..backends.impl_numba.toms748 import toms748_solve
 from ..physics import constants as const
 from ..backends.impl_numba.conf import JIT_FLAGS
-from PySDM.backends.impl_numba.warnings import warn
+from ..backends.impl_numba.warnings import warn
 
 default_rtol = 1e-5
 default_max_iters = 64

PySDM/initialisation/spectral_sampling.py

@@ -17,7 +17,8 @@ def __init__(self, spectrum, size_range: [None, Tuple[float, float]] = None):
                 self.size_range = [np.nan, np.nan]
                 for i in (0, 1):
                     result = optimize.root(
-                        lambda x: spectrum.cdf(x) - default_cdf_range[i], x0=spectrum.median()
+                        lambda x, value=default_cdf_range[i]: spectrum.cdf(x) - value,
+                        x0=spectrum.median()
                     )
                     assert result.success
                     self.size_range[i] = result.x