Merge pull request #210 from mesoscope/feature/store-runs-in-time

Feature/store runs in time

cellpack/autopack/Analysis.py

@@ -34,6 +34,7 @@
 from cellpack.autopack.upy import colors as col
 from cellpack.autopack.upy.colors import map_colors
 from cellpack.autopack.utils import check_paired_key, get_paired_key
+from cellpack.autopack.writers import Writer
 from cellpack.autopack.writers.ImageWriter import ImageWriter
 import concurrent.futures
 import multiprocessing
@@ -84,7 +85,7 @@ def __init__(
 
         self.figures_path = self.output_path / "figures"
         self.figures_path.mkdir(parents=True, exist_ok=True)
-
+        self.seed_to_results = {}
         autopack._colors = None
 
     @staticmethod
@@ -1750,7 +1751,8 @@ def pack(
             self.plotly.update_title(self.env.place_method)
 
         t1 = time()
-        self.env.pack_grid(seedNum=seed)
+        results = self.env.pack_grid(seedNum=seed)
+        self.seed_to_results[seed] = results
         t2 = time()
         run_time = t2 - t1
         print(f"time to run pack_grid for {self.env.place_method}: {run_time:0.2f}")
@@ -2427,7 +2429,6 @@ def doloop(
                             seed_index=seed_index,
                             seed_list=seed_list,
                             bounding_box=bounding_box,
-                            packing_basename=packing_basename,
                             center_distance_dict=center_distance_dict,
                             pairwise_distance_dict=pairwise_distance_dict,
                             ingredient_position_dict=ingredient_position_dict,
@@ -2486,6 +2487,8 @@ def doloop(
         self.writeJSON(ingredient_occurences_file, ingredient_occurence_dict)
         self.writeJSON(ingredient_key_file, ingredient_key_dict)
 
+        Writer().save_as_simularium(self.env, self.seed_to_results)
+
         all_ingredient_positions = self.combine_results_from_seeds(
             ingredient_position_dict
         )

cellpack/autopack/Environment.py

@@ -498,7 +498,7 @@ def save_result(
         self,
         free_points,
         distances,
-        all_ingr_as_array,
+        all_objects,
         save_grid_logs=False,
         save_result_as_file=False,
     ):
@@ -519,8 +519,7 @@ def save_result(
             kwds=["compartment_id"],
             result=True,
             quaternion=True,
-            all_ingr_as_array=all_ingr_as_array,
-            compartments=self.compartments,
+            seed_to_results_map={0: all_objects},
         )
 
     def loadResult(
@@ -1803,7 +1802,7 @@ def prep_molecules_for_save(self, distances, free_points, nbFreePoints):
 
         if self.runTimeDisplay and autopack.helper.host == "simularium":
             autopack.helper.writeToFile("./realtime", self.boundingBox)
-        return self.packed_objects.get_ingredients()
+        return self.packed_objects.get_all()
 
     def check_new_placement(self, new_position):
         distances = self.get_all_distances(new_position)
@@ -2207,7 +2206,7 @@ def pack_grid(
                 self.activeIngr = self.activeIngr0 + self.activeIngr12
 
             if dump and ((time() - stime) > dump_freq):
-                all_ingr_as_array = self.prep_molecules_for_save(
+                all_objects = self.prep_molecules_for_save(
                     distances,
                     free_points,
                     nbFreePoints,
@@ -2218,23 +2217,22 @@ def pack_grid(
                     self.save_result(
                         free_points,
                         distances=distances,
-                        all_ingr_as_array=all_ingr_as_array,
+                        all_objects=all_objects,
                     )
 
         t2 = time()
         if self.show_progress_bar:
             pbar.close()
         self.log.info("time to fill %d", t2 - t1)
-        all_ingr_as_array = self.prep_molecules_for_save(
-            distances, free_points, nbFreePoints
-        )
+        all_objects = self.prep_molecules_for_save(distances, free_points, nbFreePoints)
 
         if self.saveResult:
             self.save_result(
                 free_points,
                 distances=distances,
-                all_ingr_as_array=all_ingr_as_array,
+                all_objects=all_objects,
             )
+        return all_objects
 
     def restore_molecules_array(self, ingr):
         pass

cellpack/autopack/loaders/config_loader.py

@@ -34,6 +34,7 @@ class ConfigLoader(object):
         "ordered_packing": False,
         "out": "out/",
         "overwrite_place_method": False,
+        "open_results_in_browser": True,
         "parallel": False,
         "place_method": "spheresSST",
         "randomness_seed": None,
@@ -43,7 +44,7 @@ class ConfigLoader(object):
         "show_sphere_trees": False,
         "show_progress_bar": False,
         "spacing": None,
-        "upload_results": False,
+        "upload_results": True,
         "use_periodicity": False,
         "version": 1.0,
     }

cellpack/autopack/upy/simularium/simularium_helper.py

@@ -482,9 +482,12 @@ def init_scene_with_objects(
         show_sphere_trees=False,
         grid_pt_radius=0.5,
     ):
-        self.time = 0
+        self.increment_time()
         instance_number = 0
         for packed_object in objects:
+            if packed_object.is_compartment:
+                self.add_compartment_to_scene(packed_object.ingredient)
+                continue
             ingr_name = packed_object.name
             ingredient = packed_object.ingredient
             sub_points = None
@@ -1383,7 +1386,7 @@ def raycast(self, **kw):
     def raycast_test(self, obj, start, end, length, **kw):
         return
 
-    def post_and_open_file(self, file_name):
+    def post_and_open_file(self, file_name, open_results_in_browser=True):
         simularium_file = Path(f"{file_name}.simularium")
         url = None
         try:
@@ -1398,7 +1401,7 @@ def post_and_open_file(self, file_name):
                 print(
                     f"An error occurred while storing the file {simularium_file} to S3: {e}"
                 )
-        if url is not None:
+        if url is not None and open_results_in_browser:
             simulariumHelper.open_in_simularium(url)
 
     @staticmethod

cellpack/autopack/writers/__init__.py

@@ -149,23 +149,24 @@ def setValueToJsonNode(value, attrname):
         vdic[attrname] = value
         return vdic
 
-    def save_as_simularium(self, env, all_ingr_as_array, compartments):
+    def save_as_simularium(self, env, seed_to_results_map):
         env.helper.clear()
 
         grid_positions = env.grid.masterGridPositions if env.show_grid_spheres else None
         compartment_ids = env.grid.compartment_ids if env.show_grid_spheres else None
-        env.helper.init_scene_with_objects(
-            objects=all_ingr_as_array,
-            grid_point_positions=grid_positions,
-            grid_point_compartment_ids=compartment_ids,
-            show_sphere_trees=env.show_sphere_trees,
-            grid_pt_radius=env.grid.gridSpacing / 4,
-        )
 
-        if compartments is not None:
-            for compartment in compartments:
-                env.helper.add_compartment_to_scene(compartment)
+        # one packing
+        for _, all_ingr_as_array in seed_to_results_map.items():
+
+            env.helper.init_scene_with_objects(
+                objects=all_ingr_as_array,
+                grid_point_positions=grid_positions,
+                grid_point_compartment_ids=compartment_ids,
+                show_sphere_trees=env.show_sphere_trees,
+                grid_pt_radius=env.grid.gridSpacing / 4,
+            )
 
+        # Same for all packings
         # plots the distances used to calculate gradients
         # TODO: add an option to plot grid points for compartments and for gradients
         if grid_positions is not None and len(env.gradients):
@@ -182,13 +183,19 @@ def save_as_simularium(self, env, all_ingr_as_array, compartments):
                     gradient.weight,
                     env.grid.gridSpacing / 4,
                 )
+        # write to simularium format
+        result_file_name = env.result_file
+        is_aggregate = len(seed_to_results_map) > 1
+        if is_aggregate:
+            result_file_name = f"{env.result_file.split('_seed')[0]}_all"
         file_name = env.helper.writeToFile(
-            env.result_file, env.boundingBox, env.name, env.version
+            result_file_name, env.boundingBox, env.name, env.version
         )
-        upload_results = env.config_data.get("upload_results", False)
         number_of_packings = env.config_data.get("number_of_packings", 1)
-        if upload_results or number_of_packings == 1:
-            autopack.helper.post_and_open_file(file_name)
+        open_results_in_browser = env.config_data.get("open_results_in_browser", True)
+        upload_results = env.config_data.get("upload_results", True)
+        if (number_of_packings == 1 or is_aggregate) and upload_results:
+            autopack.helper.post_and_open_file(file_name, open_results_in_browser)
 
     def save_Mixed_asJson(
         self,
@@ -403,8 +410,7 @@ def save(
         indent=False,
         quaternion=False,
         transpose=False,
-        all_ingr_as_array=None,
-        compartments=None,
+        seed_to_results_map=None,
     ):
         output_format = self.format
         if output_format == "json":
@@ -420,6 +426,6 @@ def save(
                 transpose=transpose,
             )
         elif output_format == "simularium":
-            self.save_as_simularium(env, all_ingr_as_array, compartments)
+            self.save_as_simularium(env, seed_to_results_map)
         else:
             print("format output " + output_format + " not recognized (json,python)")

cellpack/bin/pack.py

@@ -46,7 +46,10 @@ def pack(recipe, config_path=None, analysis_config_path=None):
     env.helper = helper
 
     afviewer = None
-    if packing_config_data["save_analyze_result"]:
+    if (
+        packing_config_data["save_analyze_result"]
+        or packing_config_data["number_of_packings"] > 1
+    ):
         analyze = Analysis(
             env=env,
             viewer=afviewer,

cellpack/tests/packing-configs/test_variable_count_config.json

@@ -13,6 +13,5 @@
     "spacing": null,
     "parallel": false,
     "use_periodicity": false,
-    "show_sphere_trees": true,
-    "upload_results": false
+    "show_sphere_trees": true
 }

examples/packing-configs/run.json

@@ -13,6 +13,5 @@
     "use_periodicity": false,
     "show_sphere_trees": false,
     "load_from_grid_file": false,
-    "save_converted_recipe": true,
-    "upload_results": true
+    "save_converted_recipe": true
 }

examples/recipes/v2/one_sphere.json

@@ -9,9 +9,9 @@
             0
         ],
         [
-            1000,
-            1000,
-            1
+            20,
+            20,
+            20
         ]
     ],
     "objects": {
@@ -58,7 +58,7 @@
                 0.5,
                 0.5
             ],
-            "radius": 100,
+            "radius": 5,
             "max_jitter": [
                 1,
                 1,