Multiple roots (#48)

* Support multiple media roots

* fix divide by zero in IO progress, turn off gauges for reporting programs

dvrprocess/common/__init__.py

@@ -33,46 +33,85 @@
 ANALYZE_DURATION = '20000000'
 PROBE_SIZE = '20000000'
 
-MEDIA_BASE = None
+MEDIA_ROOTS: Union[list[str], None] = None
 
 
-def get_media_base():
-    global MEDIA_BASE
-    if MEDIA_BASE is None:
+def get_media_roots() -> list[str]:
+    global MEDIA_ROOTS
+    if MEDIA_ROOTS is None:
         _once_lock.acquire()
         try:
-            if MEDIA_BASE is None:
-                MEDIA_BASE = find_media_base()
+            if MEDIA_ROOTS is None:
+                MEDIA_ROOTS = _find_media_roots()
         finally:
             _once_lock.release()
-    return MEDIA_BASE
+    return MEDIA_ROOTS
 
 
-def find_media_base() -> str:
+def _find_media_roots() -> list[str]:
     try:
         paths = config.get_global_config_option('media', 'paths').split(',')
         if len(paths) > 0:
-            for p in psutil.disk_partitions(all=True):
-                if os.path.isdir(os.path.join(p.mountpoint, paths[0])):
-                    return p.mountpoint
+            mount_points = []
+            for part in psutil.disk_partitions(all=True):
+                if any([os.path.isdir(os.path.join(part.mountpoint, path)) for path in paths]):
+                    mount_points.append(part.mountpoint)
+            if mount_points:
+                return mount_points
     except KeyError:
         logger.debug('No media.paths in config')
     try:
+        roots = []
         for root in config.get_global_config_option('media', 'root').split(','):
             root = root.replace('$HOME', os.environ['HOME'])
             if os.path.isdir(root):
-                return root
+                roots.append(root)
+        if roots:
+            return roots
     except KeyError:
         logger.debug('No media.root in config')
     logger.info('No valid media.root found, returning user home')
-    return os.environ['HOME']
+    return [os.environ['HOME']]
 
 
-def get_media_paths(base=None):
-    if base is None:
-        base = get_media_base()
-    paths = config.get_global_config_option('media', 'paths').split(',')
-    return list(map(lambda e: os.path.join(base, e), paths))
+def get_media_paths(roots=None, paths=None):
+    if roots is None:
+        roots = get_media_roots()
+    elif isinstance(roots, (list, set)):
+        roots = list(roots)
+    else:
+        roots = [roots]
+
+    if paths is None:
+        paths = config.get_global_config_option('media', 'paths').split(',')
+    elif isinstance(paths, (list, set)):
+        paths = list(paths)
+    else:
+        paths = [paths]
+
+    result = set()
+    for root in roots:
+        for path in paths:
+            if os.path.isabs(path):
+                joined = path
+            else:
+                joined = os.path.join(root, path)
+            if os.path.exists(joined):
+                result.add(joined)
+    return list(result)
+
+
+def get_media_file_relative_to_root(file_name: str, roots: list[str]) -> tuple[str, str]:
+    if not roots:
+        return file_name, '/'
+    for root in roots:
+        if root.endswith('/'):
+            prefix = root
+        else:
+            prefix = root + '/'
+        if file_name.startswith(prefix):
+            return file_name[len(prefix):], prefix
+    return file_name, '/'
 
 
 def fatal(message):
@@ -927,10 +966,11 @@ def setup_logging(level=logging.INFO):
     logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s:%(lineno)d %(message)s', level=level, force=True)
 
 
-def setup_cli(level=logging.INFO):
+def setup_cli(level=logging.INFO, start_gauges=True):
     setup_logging(level)
     setup_debugging()
-    progress.start_compute_gauges()
+    if start_gauges:
+        progress.start_compute_gauges()
 
 
 def cli_wrapper(func):

dvrprocess/common/progress.py

@@ -568,8 +568,12 @@ def update(self):
             return
 
         if self._last_stats is not None:
+            current_total = self._total(current_stats)
+            last_total = self._total(self._last_stats)
+            if current_total == last_total:
+                return
             iowait_pct = ((current_stats['iowait'] - self._last_stats['iowait']) * 100) / (
-                    self._total(current_stats) - self._total(self._last_stats))
+                    current_total - last_total)
             self.gauge.value(iowait_pct)
 
         self._last_stats = current_stats

dvrprocess/edl-normalize.py

@@ -41,5 +41,5 @@ def edl_normalize_cli(args):
 
 
 if __name__ == '__main__':
-    common.setup_cli(level=logging.ERROR)
+    common.setup_cli(level=logging.ERROR, start_gauges=False)
     sys.exit(edl_normalize_cli(sys.argv[1:]))

dvrprocess/find_need_comcut.py

@@ -26,14 +26,14 @@ def usage():
 -t, --terminator="\\n"
     Set the output terminator, defaults to null (0).
 -d, --dir=
-    Directory containing media. Defaults to {common.get_media_base()}
+    Directory containing media. Defaults to {common.get_media_roots()}
 --nagios
     Output for Nagios monitoring. Also human readable with statistics and estimates of transcode time.
 """, file=sys.stderr)
 
 
 def find_need_comcut_cli(argv):
-    host_home = common.get_media_base()
+    roots = []
     terminator = '\0'
     nagios_output = False
 
@@ -48,7 +48,7 @@ def find_need_comcut_cli(argv):
             usage()
             return 2
         elif opt in ("-d", "--dir"):
-            host_home = arg
+            roots.append(arg)
         elif opt == '--nagios':
             nagios_output = True
         elif opt in ("-t", "--terminator"):
@@ -59,14 +59,17 @@ def find_need_comcut_cli(argv):
             else:
                 terminator = arg
 
+    if not roots:
+        roots = common.get_media_roots()
+
     if args:
-        media_paths = list(map(lambda e: e if os.path.isabs(e) else os.path.join(host_home, e), args))
+        media_paths = common.get_media_paths(roots, args)
     else:
-        media_paths = list(filter(lambda path: 'Movies' in path, common.get_media_paths(host_home)))
+        media_paths = list(filter(lambda path: 'Movies' in path, common.get_media_paths(roots)))
     logger.debug("media_paths = %s", media_paths)
 
     if nagios_output:
-        pending_files = list(need_comcut_generator(media_paths=media_paths, host_home=host_home))
+        pending_files = list(need_comcut_generator(media_paths=media_paths, media_roots=roots))
         pending_files.sort(key=lambda e: e.size, reverse=True)
         uncut_length = sum(map(lambda e: e.uncut_length, pending_files))
         cut_length = sum(map(lambda e: e.cut_length, pending_files))
@@ -81,7 +84,7 @@ def find_need_comcut_cli(argv):
                 f"{e.file_name};{e.size};{common.seconds_to_timespec(e.uncut_length)};{common.seconds_to_timespec(e.cut_length)};{e.cut_count}")
         return code
     else:
-        for e in need_comcut_generator(media_paths=media_paths, host_home=host_home):
+        for e in need_comcut_generator(media_paths=media_paths, media_roots=roots):
             sys.stdout.write(e.file_name)
             sys.stdout.write(terminator)
         return 0
@@ -99,7 +102,7 @@ def __init__(self, file_name: str, host_file_path: str, size: float, uncut_lengt
         self.cut_count = cut_count
 
 
-def need_comcut_generator(media_paths: list[str], host_home: str) -> Iterable[ComcutPendingFileInfo]:
+def need_comcut_generator(media_paths: list[str], media_roots: list[str]) -> Iterable[ComcutPendingFileInfo]:
     for media_path in media_paths:
         for root, dirs, files in os.walk(media_path, topdown=True):
             files_set = set(files)
@@ -119,15 +122,16 @@ def need_comcut_generator(media_paths: list[str], host_home: str) -> Iterable[Co
                 input_info = common.find_input_info(filepath)
                 uncut_length = float(input_info[constants.K_FORMAT][constants.K_DURATION])
                 cut_length = uncut_length - sum(map(lambda e: e.length(), edl))
-                file_info = ComcutPendingFileInfo(file_name=filepath.replace(host_home + '/', ''),
-                                                  host_file_path=filepath,
-                                                  size=os.stat(filepath).st_size,
-                                                  uncut_length=uncut_length,
-                                                  cut_length=cut_length,
-                                                  cut_count=len(edl))
+                file_info = ComcutPendingFileInfo(
+                    file_name=common.get_media_file_relative_to_root(filepath, media_roots)[0],
+                    host_file_path=filepath,
+                    size=os.stat(filepath).st_size,
+                    uncut_length=uncut_length,
+                    cut_length=cut_length,
+                    cut_count=len(edl))
                 yield file_info
 
 
 if __name__ == '__main__':
-    common.setup_cli(level=logging.ERROR)
+    common.setup_cli(level=logging.ERROR, start_gauges=False)
     sys.exit(find_need_comcut_cli(sys.argv[1:]))