Merge pull request #194 from thegridelectric/dev

Persister fixes

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gridworks-proactor"
-version = "0.4.4"
+version = "0.4.5"
 description = "Gridworks Proactor"
 authors = ["Jessica Millar <[email protected]>"]
 license = "MIT"

src/gwproactor/links/reuploads.py

@@ -37,14 +37,14 @@ def init(
             self.begin_num_unacked = reuploads.num_reuploaded_unacked
             self.begin_num_pending = reuploads.num_reupload_pending
             if verbose:
-                self.begin_verbose_str = reuploads.get_str(num_events=100)
+                self.begin_verbose_str = reuploads.get_str(num_events=5)
 
     def diff_str(self, path_dbg: int) -> str:
         s = ""
         if self.reuploads is not None:
             if self.verbose:
                 s += f"Begin reuploads:\n{self.begin_verbose_str}\n"
-                s += f"End reuploads:\n{self.reuploads.get_str(num_events=100)}\n"
+                s += f"End reuploads:\n{self.reuploads.get_str(num_events=5)}\n"
             s += (
                 f"path:0x{path_dbg:08X}  "
                 f"reuploading: {int(self.begin_reuploading)} -> {int(self.reuploads.reuploading())}  "
@@ -200,4 +200,4 @@ def _log_start_reupload(self, num_pending_events, num_reupload_now):
                 f"Total events in reupload: {num_pending_events}."
             )
         if self._logger.path_enabled:
-            self._logger.path(self.get_str(num_events=100))
+            self._logger.path(self.get_str(num_events=5))

src/gwproactor/persister.py

@@ -327,8 +327,13 @@ def clear(self, uid: str) -> Result[bool, Problems]:
             if path.exists():
                 self._curr_bytes -= path.stat().st_size
                 path.unlink()
-                if next(path.parent.iterdir(), None) is None:
-                    shutil.rmtree(path.parent, ignore_errors=True)
+                path_dir = path.parent
+                # Remove directory if empty.
+                # This is much faster than using iterdir.
+                try:
+                    path_dir.rmdir()
+                except OSError:
+                    ...
             else:
                 problems.add_warning(FileMissingWarning(uid=uid, path=path))
         else:
@@ -381,7 +386,7 @@ def reindex(self) -> Result[bool, Problems]:
             try:
                 if base_dir_entry.is_dir() and self._is_iso_parseable(base_dir_entry):
                     for day_dir_entry in base_dir_entry.iterdir():
-                        if self._pat_watchdog_args is not None:
+                        if self._pat_watchdog_args:
                             now = time.time()
                             if now > last_pat + self._reindex_pat_seconds:
                                 last_pat = now
@@ -399,7 +404,7 @@ def reindex(self) -> Result[bool, Problems]:
                             )
             except BaseException as e:
                 problems.add_error(e).add_error(ReindexError())
-        self._pending = dict(sorted(paths, key=lambda item: item.path))
+        self._pending = dict(sorted(paths, key=lambda item: item.path))  # noqa
         if problems:
             return Err(problems)
         else:

src/gwproactor/proactor_implementation.py

@@ -117,6 +117,8 @@ def __init__(
         reindex_result = self._event_persister.reindex()
         if reindex_result.is_err():
             self._reindex_problems = reindex_result.err()
+            self._logger.error("ERROR in event persister reindex():")
+            self._logger.error(reindex_result.err())
         self._links = LinkManager(
             publication_name=self.publication_name,
             settings=settings,