diff --git a/.gitignore b/.gitignore index e6f8fb2..05792f8 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,6 @@ dmypy.json # Pyre type checker .pyre/ + +# vs code settings +.vscode/ diff --git a/README.md b/README.md index 3b17f24..93e1952 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,10 @@ Note that unpack needs to be able to identify the images it imports inequivocall `--folder` allows the user to point to a previously-unpacked folder rather than a single file. +`--merge` will use existing Projects, Datasets and Screens if the current user +already owns entities with the same name as ones defined in `transfer.xml`, +effectively merging the "new" unpacked entities with existing ones. + Examples: ``` omero transfer unpack transfer_pack.zip diff --git a/requirements.txt b/requirements.txt index 47a5a57..cc188e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ ezomero>=2.0.0 -ome-types>=0.3.4 +ome-types>=0.4.0 setuptools>=58.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index c24dd83..d8df849 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def read(fname): packages=['', 'omero.plugins'], package_dir={"": "src"}, name="omero-cli-transfer", - version='0.6.0', + version='0.7.0', maintainer="Erick Ratamero", maintainer_email="erick.ratamero@jax.org", description=("A set of utilities for exporting a transfer" @@ -96,7 +96,7 @@ def read(fname): url="https://github.com/TheJacksonLaboratory/omero-cli-transfer", install_requires=[ 'ezomero==2.0.0', - 'ome-types==0.3.4' + 'ome-types==0.4.0' ], extras_require={ "rocrate": ["rocrate==0.7.0"], diff --git a/src/generate_omero_objects.py b/src/generate_omero_objects.py index de40c86..3bd0e47 100644 --- a/src/generate_omero_objects.py +++ b/src/generate_omero_objects.py @@ -25,6 +25,20 @@ import copy +def create_or_set_projects(pjs: List[Project], conn: BlitzGateway, + merge: bool) -> dict: + pj_map = {} + if not merge: + pj_map = create_projects(pjs, conn) + else: + for pj in pjs: + pj_id = find_project(pj, conn) + if not pj_id: + pj_id = ezomero.post_project(conn, pj.name, pj.description) + pj_map[pj.id] = pj_id + return pj_map + + def create_projects(pjs: List[Project], conn: BlitzGateway) -> dict: pj_map = {} for pj in pjs: @@ -33,6 +47,29 @@ def create_projects(pjs: List[Project], conn: BlitzGateway) -> dict: return pj_map +def find_project(pj: Project, conn: BlitzGateway) -> int: + id = 0 + my_exp_id = conn.getUser().getId() + for p in conn.getObjects("Project", opts={'owner': my_exp_id}): + if p.getName() == pj.name: + id = p.getId() + return id + + +def create_or_set_screens(scrs: List[Screen], conn: BlitzGateway, merge: bool + ) -> dict: + scr_map = {} + if not merge: + scr_map = create_screens(scrs, conn) + else: + for scr in scrs: + scr_id = find_screen(scr, conn) + if not scr_id: + scr_id = ezomero.post_screen(conn, scr.name, scr.description) + scr_map[scr.id] = scr_id + return scr_map + + def create_screens(scrs: List[Screen], conn: BlitzGateway) -> dict: scr_map = {} for scr in scrs: @@ -41,6 +78,34 @@ def create_screens(scrs: List[Screen], conn: BlitzGateway) -> dict: return scr_map +def find_screen(sc: Screen, conn: BlitzGateway) -> int: + id = 0 + my_exp_id = conn.getUser().getId() + for s in conn.getObjects("Screen", opts={'owner': my_exp_id}): + if s.getName() == sc.name: + id = s.getId() + return id + + +def create_or_set_datasets(dss: List[Dataset], pjs: List[Project], + conn: BlitzGateway, merge: bool) -> dict: + ds_map = {} + if not merge: + ds_map = create_datasets(dss, conn) + else: + for ds in dss: + ds_id = find_dataset(ds, pjs, conn) + if not ds_id: + dataset = DatasetWrapper(conn, DatasetI()) + dataset.setName(ds.name) + if ds.description is not None: + dataset.setDescription(ds.description) + dataset.save() + ds_id = dataset.getId() + ds_map[ds.id] = ds_id + return ds_map + + def create_datasets(dss: List[Dataset], conn: BlitzGateway) -> dict: """ Currently doing it the non-ezomero way because ezomero always @@ -58,6 +123,31 @@ def create_datasets(dss: List[Dataset], conn: BlitzGateway) -> dict: return ds_map +def find_dataset(ds: Dataset, pjs: List[Project], conn: BlitzGateway) -> int: + id = 0 + my_exp_id = conn.getUser().getId() + orphan = True + for pj in pjs: + for dsref in pj.dataset_refs: + if dsref.id == ds.id: + orphan = False + if not orphan: + for pj in pjs: + for p in conn.getObjects("Project", opts={'owner': my_exp_id}): + if p.getName() == pj.name: + for dsref in pj.dataset_refs: + if dsref.id == ds.id: + for ds_rem in p.listChildren(): + if ds.name == ds_rem.getName(): + id = ds_rem.getId() + else: + for d in conn.getObjects("Dataset", opts={'owner': my_exp_id, + 'orphaned': True}): + if d.getName() == ds.name: + id = d.getId() + return id + + def create_annotations(ans: List[Annotation], conn: BlitzGateway, hash: str, folder: str, metadata: List[str]) -> dict: ann_map = {} @@ -73,7 +163,7 @@ def create_annotations(ans: List[Annotation], conn: BlitzGateway, hash: str, namespace = an.namespace map_ann.setNs(namespace) key_value_data = [] - for v in an.value.m: + for v in an.value.ms: if int(an.id.split(":")[-1]) < 0: if not metadata: key_value_data.append(['empty_metadata', "True"]) @@ -131,7 +221,7 @@ def create_original_file(ann: FileAnnotation, ans: List[Annotation], conn: BlitzGateway, folder: str ) -> OriginalFileWrapper: curr_folder = str(Path('.').resolve()) - for an in ann.annotation_ref: + for an in ann.annotation_refs: clean_id = int(an.id.split(":")[-1]) if clean_id < 0: cmnt_id = an.id @@ -149,7 +239,7 @@ def create_plate_map(ome: OME, img_map: dict, conn: BlitzGateway plate_map = {} map_ref_ids = [] for plate in ome.plates: - ann_ids = [i.id for i in plate.annotation_ref] + ann_ids = [i.id for i in plate.annotation_refs] file_path = "" for ann in ome.structured_annotations: if (ann.id in ann_ids and @@ -201,9 +291,9 @@ def create_plate_map(ome: OME, img_map: dict, conn: BlitzGateway plate_id = create_plate_from_images(plate, img_map, conn) plate_map[plate.id] = plate_id for p in newome.plates: - for ref in p.annotation_ref: + for ref in p.annotation_refs: if ref.id in map_ref_ids: - p.annotation_ref.remove(ref) + p.annotation_refs.remove(ref) return plate_map, newome @@ -342,7 +432,7 @@ def _int_to_rgba(omero_val: int) -> Tuple[int, int, int, int]: def create_rois(rois: List[ROI], imgs: List[Image], img_map: dict, conn: BlitzGateway): for img in imgs: - for roiref in img.roi_ref: + for roiref in img.roi_refs: roi = next(filter(lambda x: x.id == roiref.id, rois)) shapes = create_shapes(roi) img_id_dest = img_map[img.id] @@ -354,10 +444,15 @@ def create_rois(rois: List[ROI], imgs: List[Image], img_map: dict, def link_datasets(ome: OME, proj_map: dict, ds_map: dict, conn: BlitzGateway): for proj in ome.projects: proj_id = proj_map[proj.id] + proj_obj = conn.getObject("Project", proj_id) + existing_ds = [] + for dataset in proj_obj.listChildren(): + existing_ds.append(dataset.getId()) ds_ids = [] - for ds in proj.dataset_ref: + for ds in proj.dataset_refs: ds_id = ds_map[ds.id] - ds_ids.append(ds_id) + if ds_id not in existing_ds: + ds_ids.append(ds_id) ezomero.link_datasets_to_project(conn, ds_ids, proj_id) return @@ -366,10 +461,15 @@ def link_plates(ome: OME, screen_map: dict, plate_map: dict, conn: BlitzGateway): for screen in ome.screens: screen_id = screen_map[screen.id] + scr_obj = conn.getObject("Screen", screen_id) + existing_pl = [] + for pl in scr_obj.listChildren(): + existing_pl.append(pl.getId()) pl_ids = [] - for pl in screen.plate_ref: + for pl in screen.plate_refs: pl_id = plate_map[pl.id] - pl_ids.append(pl_id) + if pl_id not in existing_pl: + pl_ids.append(pl_id) ezomero.link_plates_to_screen(conn, pl_ids, screen_id) return @@ -378,7 +478,7 @@ def link_images(ome: OME, ds_map: dict, img_map: dict, conn: BlitzGateway): for ds in ome.datasets: ds_id = ds_map[ds.id] img_ids = [] - for img in ds.image_ref: + for img in ds.image_refs: try: img_id = img_map[img.id] img_ids.append(img_id) @@ -395,14 +495,14 @@ def link_annotations(ome: OME, proj_map: dict, ds_map: dict, img_map: dict, proj_id = proj_map[proj.id] proj_obj = conn.getObject("Project", proj_id) anns = ome.structured_annotations - for annref in proj.annotation_ref: + for annref in proj.annotation_refs: ann = next(filter(lambda x: x.id == annref.id, anns)) link_one_annotation(proj_obj, ann, ann_map, conn) for ds in ome.datasets: ds_id = ds_map[ds.id] ds_obj = conn.getObject("Dataset", ds_id) anns = ome.structured_annotations - for annref in ds.annotation_ref: + for annref in ds.annotation_refs: ann = next(filter(lambda x: x.id == annref.id, anns)) link_one_annotation(ds_obj, ann, ann_map, conn) for img in ome.images: @@ -410,7 +510,7 @@ def link_annotations(ome: OME, proj_map: dict, ds_map: dict, img_map: dict, img_id = img_map[img.id] img_obj = conn.getObject("Image", img_id) anns = ome.structured_annotations - for annref in img.annotation_ref: + for annref in img.annotation_refs: ann = next(filter(lambda x: x.id == annref.id, anns)) link_one_annotation(img_obj, ann, ann_map, conn) except KeyError: @@ -419,23 +519,23 @@ def link_annotations(ome: OME, proj_map: dict, ds_map: dict, img_map: dict, scr_id = scr_map[scr.id] scr_obj = conn.getObject("Screen", scr_id) anns = ome.structured_annotations - for annref in scr.annotation_ref: + for annref in scr.annotation_refs: ann = next(filter(lambda x: x.id == annref.id, anns)) link_one_annotation(scr_obj, ann, ann_map, conn) for pl in ome.plates: pl_id = pl_map[pl.id] pl_obj = conn.getObject("Plate", pl_id) anns = ome.structured_annotations - for annref in pl.annotation_ref: + for annref in pl.annotation_refs: ann = next(filter(lambda x: x.id == annref.id, anns)) link_one_annotation(pl_obj, ann, ann_map, conn) anns = ome.structured_annotations for well in pl.wells: - if len(well.annotation_ref) > 0: + if len(well.annotation_refs) > 0: row, col = well.row, well.column well_id = ezomero.get_well_id(conn, pl_id, row, col) well_obj = conn.getObject("Well", well_id) - for annref in well.annotation_ref: + for annref in well.annotation_refs: ann = next(filter(lambda x: x.id == annref.id, anns)) link_one_annotation(well_obj, ann, ann_map, conn) return @@ -485,13 +585,13 @@ def rename_plates(pls: List[Plate], pl_map: dict, conn: BlitzGateway): def populate_omero(ome: OME, img_map: dict, conn: BlitzGateway, hash: str, - folder: str, metadata: List[str]): + folder: str, metadata: List[str], merge: bool): plate_map, ome = create_plate_map(ome, img_map, conn) rename_images(ome.images, img_map, conn) rename_plates(ome.plates, plate_map, conn) - proj_map = create_projects(ome.projects, conn) - ds_map = create_datasets(ome.datasets, conn) - screen_map = create_screens(ome.screens, conn) + proj_map = create_or_set_projects(ome.projects, conn, merge) + ds_map = create_or_set_datasets(ome.datasets, ome.projects, conn, merge) + screen_map = create_or_set_screens(ome.screens, conn, merge) ann_map = create_annotations(ome.structured_annotations, conn, hash, folder, metadata) create_rois(ome.rois, ome.images, img_map, conn) diff --git a/src/generate_xml.py b/src/generate_xml.py index 672c2b3..161e84d 100644 --- a/src/generate_xml.py +++ b/src/generate_xml.py @@ -459,7 +459,7 @@ def create_provenance_metadata(conn: BlitzGateway, img_id: int, mmap.append(M(k=_key, value='')) kv, ref = create_kv_and_ref(id=id, namespace=ns, - value=Map(m=mmap)) + value=Map(ms=mmap)) return kv, ref @@ -527,7 +527,7 @@ def parse_files_import(text): def parse_showinf(text, counter_imgs, counter_plates, target): - ome = from_xml(text, parser='xmlschema') + ome = from_xml(text) images = [] plates = [] annotations = [] @@ -552,7 +552,10 @@ def parse_showinf(text, counter_imgs, counter_plates, target): ) annotations.append(an) anref = AnnotationRef(id=an.id) - img.annotation_ref.append(anref) + img.annotation_refs.append(anref) + an, anref = create_prepare_metadata() + annotations.append(an) + img.annotation_refs.append(anref) images.append(img) for plate in ome.plates: pl_id_str = f"Plate:{str(pl_id)}" @@ -568,11 +571,33 @@ def parse_showinf(text, counter_imgs, counter_plates, target): ) annotations.append(an) anref = AnnotationRef(id=an.id) - pl.annotation_ref.append(anref) + pl.annotation_refs.append(anref) plates.append(pl) return images, plates, annotations +def create_prepare_metadata(): + software = "omero-cli-transfer" + version = pkg_resources.get_distribution(software).version + date_time = datetime.now().strftime("%d/%m/%Y, %H:%M:%S") + ns = 'openmicroscopy.org/cli/transfer/prepare' + id = (-1) * uuid4().int + md_dict: Dict[str, Any] = {} + md_dict['software'] = software + md_dict['version'] = version + md_dict['packing_timestamp'] = date_time + mmap = [] + for _key, _value in md_dict.items(): + if _value: + mmap.append(M(k=_key, value=str(_value))) + else: + mmap.append(M(k=_key, value='')) + kv, ref = create_kv_and_ref(id=id, + namespace=ns, + value=Map(ms=mmap)) + return kv, ref + + def create_empty_pixels(image, id): pix_id = f"Pixels:{str(id)}" pixels = Pixels( @@ -630,11 +655,11 @@ def populate_image(obj: ImageI, ome: OME, conn: BlitzGateway, hostname: str, if kv_id not in [i.id for i in ome.structured_annotations]: ome.structured_annotations.append(kv) if ref: - img.annotation_ref.append(ref) + img.annotation_refs.append(ref) filepath_anns, refs = create_filepath_annotations(img_id, conn) for i in range(len(filepath_anns)): ome.structured_annotations.append(filepath_anns[i]) - img.annotation_ref.append(refs[i]) + img.annotation_refs.append(refs[i]) roi_service = conn.getRoiService() rois = roi_service.findByImage(id, None).rois for roi in rois: @@ -668,7 +693,7 @@ def populate_dataset(obj: DatasetI, ome: OME, conn: BlitzGateway, for img in obj.listChildren(): img_obj = conn.getObject('Image', img.getId()) img_ref = populate_image(img_obj, ome, conn, hostname, metadata) - ds.image_ref.append(img_ref) + ds.image_refs.append(img_ref) ds_id = f"Dataset:{str(ds.id)}" if ds_id not in [i.id for i in ome.datasets]: ome.datasets.append(ds) @@ -686,7 +711,7 @@ def populate_project(obj: ProjectI, ome: OME, conn: BlitzGateway, for ds in obj.listChildren(): ds_obj = conn.getObject('Dataset', ds.getId()) ds_ref = populate_dataset(ds_obj, ome, conn, hostname, metadata) - proj.dataset_ref.append(ds_ref) + proj.dataset_refs.append(ds_ref) ome.projects.append(proj) @@ -701,7 +726,7 @@ def populate_screen(obj: ScreenI, ome: OME, conn: BlitzGateway, for pl in obj.listChildren(): pl_obj = conn.getObject('Plate', pl.getId()) pl_ref = populate_plate(pl_obj, ome, conn, hostname, metadata) - scr.plate_ref.append(pl_ref) + scr.plate_refs.append(pl_ref) ome.screens.append(scr) @@ -720,23 +745,23 @@ def populate_plate(obj: PlateI, ome: OME, conn: BlitzGateway, if kv_id not in [i.id for i in ome.structured_annotations]: ome.structured_annotations.append(kv) if ref: - pl.annotation_ref.append(ref) + pl.annotation_refs.append(ref) for well in obj.listChildren(): well_obj = conn.getObject('Well', well.getId()) well_ref = populate_well(well_obj, ome, conn, hostname, metadata) pl.wells.append(well_ref) - last_image_anns = ome.images[-1].annotation_ref + last_image_anns = ome.images[-1].annotation_refs last_image_anns_ids = [i.id for i in last_image_anns] for ann in ome.structured_annotations: if (ann.id in last_image_anns_ids and - type(ann) == CommentAnnotation and + isinstance(ann, CommentAnnotation) and int(ann.id.split(":")[-1]) < 0): plate_path = ann.value filepath_anns, refs = create_filepath_annotations(pl.id, conn, plate_path=plate_path) for i in range(len(filepath_anns)): ome.structured_annotations.append(filepath_anns[i]) - pl.annotation_ref.append(refs[i]) + pl.annotation_refs.append(refs[i]) pl_id = f"Plate:{str(pl.id)}" if pl_id not in [i.id for i in ome.plates]: ome.plates.append(pl) @@ -784,7 +809,7 @@ def add_annotation(obj: Union[Project, Dataset, Image, Plate, Screen, kv, ref = create_kv_and_ref(id=ann.getId(), namespace=ann.getNs(), value=Map( - m=mmap)) + ms=mmap)) if kv.id not in [i.id for i in ome.structured_annotations]: ome.structured_annotations.append(kv) obj.annotation_ref.append(ref) @@ -934,10 +959,10 @@ def generate_columns(ome: OME, ids: dict) -> List[str]: columns.append("comment") anns = ome.structured_annotations for i in ome.images: - for ann_ref in i.annotation_ref: + for ann_ref in i.annotation_refs: ann = next(filter(lambda x: x.id == ann_ref.id, anns)) if isinstance(ann, MapAnnotation): - for v in ann.value.m: + for v in ann.value.ms: if v.k not in columns: columns.append(v.k) return columns @@ -969,7 +994,7 @@ def list_files(ome: OME, ids: dict, top_level: str) -> List[str]: def find_dataset(id: str, ome: OME) -> Union[str, None]: for d in ome.datasets: def lfunc(x): return x.id == id - if any(filter(lfunc, d.image_ref)): + if any(filter(lfunc, d.image_refs)): return d.name return None @@ -1029,7 +1054,7 @@ def generate_lines_and_move(img: Image, ome: OME, ids: dict, folder: str, def get_annotation_vals(cols: List[str], img: Image, ome: OME) -> List[str]: anns = [] - for annref in img.annotation_ref: + for annref in img.annotation_refs: a = next(filter(lambda x: x.id == annref.id, ome.structured_annotations)) anns.append(a) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index 798b0ed..d9524a6 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -95,6 +95,10 @@ --folder allows the user to point to a previously-unpacked folder rather than a single file. +--merge will use existing Projects, Datasets and Screens if the current user +already owns entities with the same name as ones defined in `transfer.xml`, +effectively merging the "new" unpacked entities with existing ones. + --metadata allows you to specify which transfer metadata will be used from `transfer.xml` as MapAnnotation values to the images. Fields that do not exist on `transfer.xml` will be ignored. Default is `all` (equivalent to @@ -198,6 +202,9 @@ def _configure(self, parser): unpack.add_argument( "--ln_s_import", help="Use in-place import", action="store_true") + unpack.add_argument( + "--merge", help="Use existing entities if possible", + action="store_true") unpack.add_argument( "--folder", help="Pass path to a folder rather than a pack", action="store_true") @@ -384,7 +391,7 @@ def __unpack(self, args): args.output) else: folder = Path(args.filepath) - ome = from_xml(folder / "transfer.xml", parser='xmlschema') + ome = from_xml(folder / "transfer.xml") hash = "imported from folder" print("Generating Image mapping and import filelist...") ome, src_img_map, filelist = self._create_image_map(ome) @@ -397,10 +404,10 @@ def __unpack(self, args): ln_s, args.skip, self.gateway) self._delete_all_rois(dest_img_map, self.gateway) print("Matching source and destination images...") - img_map = self._make_image_map(src_img_map, dest_img_map) + img_map = self._make_image_map(src_img_map, dest_img_map, self.gateway) print("Creating and linking OMERO objects...") populate_omero(ome, img_map, self.gateway, - hash, folder, self.metadata) + hash, folder, self.metadata, args.merge) return def _load_from_pack(self, filepath: str, output: Optional[str] = None @@ -433,7 +440,7 @@ def _load_from_pack(self, filepath: str, output: Optional[str] = None raise ValueError("File is not a zip or tar file") else: raise FileNotFoundError("filepath is not a zip file") - ome = from_xml(folder / "transfer.xml", parser='xmlschema') + ome = from_xml(folder / "transfer.xml") return hash, ome, folder def _create_image_map(self, ome: OME @@ -458,9 +465,9 @@ def _create_image_map(self, ome: OME filelist.append(ann.value) newome.structured_annotations.remove(ann) for i in newome.images: - for ref in i.annotation_ref: + for ref in i.annotation_refs: if ref.id in map_ref_ids: - i.annotation_ref.remove(ref) + i.annotation_refs.remove(ref) filelist = list(set(filelist)) img_map = DefaultDict(list, {x: sorted(img_map[x]) for x in img_map.keys()}) @@ -532,7 +539,8 @@ def _get_image_ids(self, file_path: str, conn: BlitzGateway) -> List[str]: image_ids.append(img_id) return image_ids - def _make_image_map(self, source_map: dict, dest_map: dict) -> dict: + def _make_image_map(self, source_map: dict, dest_map: dict, + conn: Optional[BlitzGateway] = None) -> dict: # using both source and destination file-to-image-id maps, # map image IDs between source and destination src_dict = DefaultDict(list) @@ -555,10 +563,24 @@ def _make_image_map(self, source_map: dict, dest_map: dict) -> dict: src_v = src_dict[src_k] if src_k in dest_dict.keys(): dest_v = dest_dict[src_k] - if len(src_v) == len(dest_v): + clean_dest = [] + if not conn: + clean_dest = dest_v + else: + for i in dest_v: + img_obj = conn.getObject("Image", i) + anns = 0 + for j in img_obj.listAnnotations(): + ns = j.getNs() + if ns.startswith( + "openmicroscopy.org/cli/transfer"): + anns += 1 + if not anns: + clean_dest.append(i) + if len(src_v) == len(clean_dest): for count in range(len(src_v)): map_key = f"Image:{src_v[count]}" - imgmap[map_key] = dest_v[count] + imgmap[map_key] = clean_dest[count] return imgmap def __prepare(self, args): diff --git a/test/data/simple_screen.zip b/test/data/simple_screen.zip index 01530e7..52d41da 100644 Binary files a/test/data/simple_screen.zip and b/test/data/simple_screen.zip differ diff --git a/test/integration/test_prepare.py b/test/integration/test_prepare.py index 21c8ae4..c3eb80e 100644 --- a/test/integration/test_prepare.py +++ b/test/integration/test_prepare.py @@ -168,8 +168,8 @@ def run_asserts_edited(self): elif img_name == "edited image name": kvs = ezomero.get_map_annotation_ids(self.gw, "Image", img.getId()) - assert len(kvs) == 1 - kv = ezomero.get_map_annotation(self.gw, kvs[0]) + assert len(kvs) == 2 + kv = ezomero.get_map_annotation(self.gw, kvs[-1]) assert len(kv) == 2 assert kv['key1'] == "value1" assert kv['key2'] == "2" @@ -179,7 +179,7 @@ def run_asserts_edited(self): shapes = ezomero.get_shape_ids(self.gw, rois[0]) assert len(shapes) == 1 shape = ezomero.get_shape(self.gw, shapes[0]) - assert type(shape) == ezomero.rois.Rectangle + assert type(shape) is ezomero.rois.Rectangle assert shape.x == 1 assert shape.y == 2 assert shape.width == 3 @@ -195,12 +195,12 @@ def run_asserts_edited(self): assert len(pl_ids) == 1 def edit_xml(self, filename): - ome = from_xml(filename, parser='xmlschema') + ome = from_xml(filename) new_proj = Project(id="Project:1", name="edited project") new_ds = Dataset(id="Dataset:1", name="edited dataset") newtag1 = TagAnnotation(id="Annotation:1", value="tag for proj") newtag2 = TagAnnotation(id="Annotation:2", value="tag for img") - new_proj.annotation_ref.append(AnnotationRef(id=newtag1.id)) + new_proj.annotation_refs.append(AnnotationRef(id=newtag1.id)) md_dict = {"key1": "value1", "key2": 2} mmap = [] for _key, _value in md_dict.items(): @@ -208,31 +208,31 @@ def edit_xml(self, filename): mmap.append(M(k=_key, value=str(_value))) else: mmap.append(M(k=_key, value='')) - mapann = MapAnnotation(id="Annotation:3", value=Map(m=mmap)) + mapann = MapAnnotation(id="Annotation:3", value=Map(ms=mmap)) rect = Rectangle(id="Shape:1", x=1, y=2, width=3, height=4) roi = ROI(id="ROI:1", union=[rect]) ome.rois.append(roi) ome.structured_annotations.extend([newtag1, newtag2, mapann]) for img in ome.images: if img.name == "test_pyramid.tiff": - img.annotation_ref.append(AnnotationRef(id=newtag2.id)) + img.annotation_refs.append(AnnotationRef(id=newtag2.id)) imref = ImageRef(id=img.id) - new_ds.image_ref.append(imref) + new_ds.image_refs.append(imref) elif img.name == "vsi-ets-test-jpg2k.vsi [001 C405, C488]": img.name = "edited image name" - img.annotation_ref.append(AnnotationRef(id=mapann.id)) + img.annotation_refs.append(AnnotationRef(id=mapann.id)) imref = ImageRef(id=img.id) - new_ds.image_ref.append(imref) + new_ds.image_refs.append(imref) elif img.name == "vsi-ets-test-jpg2k.vsi [macro image]": - img.roi_ref.append(ROIRef(id=roi.id)) + img.roi_refs.append(ROIRef(id=roi.id)) imref = ImageRef(id=img.id) - new_ds.image_ref.append(imref) + new_ds.image_refs.append(imref) dsref = DatasetRef(id=new_ds.id) - new_proj.dataset_ref.append(dsref) + new_proj.dataset_refs.append(dsref) ome.projects.append(new_proj) ome.datasets.append(new_ds) new_scr = Screen(id="Screen:1", name="edited screen") - new_scr.plate_ref.append(PlateRef(id=ome.plates[0].id)) + new_scr.plate_refs.append(PlateRef(id=ome.plates[0].id)) ome.screens.append(new_scr) with open(filename, 'w') as fp: print(to_xml(ome), file=fp) diff --git a/test/integration/test_transfer.py b/test/integration/test_transfer.py index 51f12cb..1e047db 100644 --- a/test/integration/test_transfer.py +++ b/test/integration/test_transfer.py @@ -317,6 +317,40 @@ def test_unpack_skip(self): assert img.getName() == 'combined_result.tiff' self.delete_all() + def test_unpack_merge(self): + proj_args = self.args + ["unpack", + "test/data/valid_single_project.zip"] + self.cli.invoke(proj_args, strict=True) + proj_args += ['--merge'] + self.cli.invoke(proj_args, strict=True) + pj_ids = ezomero.get_project_ids(self.gw) + assert len(pj_ids) == 1 + ds_ids = ezomero.get_dataset_ids(self.gw, pj_ids[0]) + assert len(ds_ids) == 2 + ds_args = self.args + ['unpack', "test/data/valid_single_dataset.zip"] + print(ds_args) + self.cli.invoke(ds_args, strict=True) + orphan = ezomero.get_dataset_ids(self.gw) + assert len(orphan) == 1 + ds_args += ['--merge'] + self.cli.invoke(ds_args, strict=True) + orphan = ezomero.get_dataset_ids(self.gw) + assert len(orphan) == 1 + scr_args = self.args + ['unpack', "test/data/simple_screen.zip"] + self.cli.invoke(scr_args, strict=True) + scr_args += ['--merge'] + self.cli.invoke(scr_args, strict=True) + scr_ids = [] + for screen in self.gw.getObjects("Screen"): + scr_ids.append(screen.getId()) + assert len(scr_ids) == 1 + pl_ids = [] + screen = self.gw.getObject("Screen", scr_ids[0]) + for plate in screen.listChildren(): + pl_ids.append(plate.getId()) + assert len(pl_ids) == 4 + self.delete_all() + @pytest.mark.parametrize('target_name', sorted(SUPPORTED)) def test_pack_unpack(self, target_name, tmpdir): if target_name == "datasetid" or target_name == "projectid" or\ diff --git a/test/unit/test_set.py b/test/unit/test_set.py index 282b54e..726ec21 100644 --- a/test/unit/test_set.py +++ b/test/unit/test_set.py @@ -88,7 +88,7 @@ def test_non_existing_file(self): 'data/output_folder') def test_src_img_map(self): - ome = from_xml('test/data/transfer.xml', parser='xmlschema') + ome = from_xml('test/data/transfer.xml') _, src_img_map, filelist = self.transfer._create_image_map(ome) correct_map = {"root_0/2022-01/14/" "18-30-55.264/combined_result.tiff": [1678, 1679]}