add workers option to CLI

AugustinMortier · Nov 3, 2023 · 67c22c5 · 67c22c5
1 parent e17c661
commit 67c22c5
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/aprofiles/cli/aprocess.py b/aprofiles/cli/aprocess.py
@@ -42,6 +42,9 @@ def main(
         ['CHM15k', 'Mini-MPL'], "--instruments-type", help="📗 List of specific instruments to be processed."
     ),
     multiprocessing: bool = typer.Option(False, help="⚡ Use multiprocessing mode."),
+    workers: int = typer.Option(
+        2, "--workers", min=1, envvar="NSLOTS", help="👷 workers NSLOTS (if multiprocessing mode is enabled)"
+    ),
     basedir_in: Path = typer.Option(
         "data/e-profile", exists=True, readable=True, help="📂 Base path for input data."
     ),
@@ -104,7 +107,7 @@ def main(
         if update_data:
             if multiprocessing:
                 with tqdm(total=len(onlyfiles), desc=date.strftime("%Y-%m-%d"), disable=disable_progress_bar) as pbar:
-                    with concurrent.futures.ProcessPoolExecutor() as executor:
+                    with concurrent.futures.ProcessPoolExecutor(max_workers=workers) as executor:
                         futures = [executor.submit(
                             utils.workflow.workflow, 
                             path=file, 

diff --git a/main.py b/main.py
@@ -82,6 +82,6 @@ def main(path):
 
 if __name__ == "__main__":
     # read some data
-    path = "examples/data/E-PROFILE/L2_0-20000-006735_A20210908.nc"
+    #path = "examples/data/E-PROFILE/L2_0-20000-006735_A20210908.nc"
     path = "examples/data/E-PROFILE/L2_0-20000-001492_A20210909.nc"
     main(path)