Skip to content

Commit

Permalink
Merge pull request #457 from DiamondLightSource/update-pipelines-imag…
Browse files Browse the repository at this point in the history 
…e-saver-and-rescaler

Update pipelines to match changes to image saver and rescaler methods
  • Loading branch information
@yousefmoazzam
yousefmoazzam authored Oct 2, 2024
2 parents 0852f3b + e20c553 commit 6af2e2e
Show file tree
Hide file tree
Showing 16 changed files with 181 additions and 87 deletions.
21 changes: 16 additions & 5 deletions httomo/transform_layer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,28 @@ def insert_data_reducer(self, pipeline: Pipeline) -> Pipeline:
return Pipeline(loader, methods)

def insert_save_images_after_sweep(self, pipeline: Pipeline) -> Pipeline:
"""For sweep methods we add image saving method after.
In addition we also add saving the results of the reconstruction,
if the module is present"""
"""For sweep methods we add image saving method after, and also a rescaling method to
rescale the data passed to the image saver. In addition we also add saving the results
of the reconstruction, if the module is present"""
loader = pipeline.loader
methods = []
sweep_before = False
for m in pipeline:
methods.append(m)
if m.sweep or "recon" in m.module_path and sweep_before:
methods.append(
make_method_wrapper(
self._repo,
"httomolibgpu.misc.rescale",
"rescale_to_int",
comm=self._comm,
save_result=False,
task_id=f"rescale_sweep_{m.task_id}",
perc_range_min=0.0,
perc_range_max=100.0,
bits=8,
)
)
methods.append(
make_method_wrapper(
self._repo,
Expand All @@ -92,8 +105,6 @@ def insert_save_images_after_sweep(self, pipeline: Pipeline) -> Pipeline:
task_id=f"saveimage_sweep_{m.task_id}",
subfolder_name="images_sweep_" + str(m.method_name),
axis=1,
perc_range_min=5,
perc_range_max=95,
),
)
sweep_before = True
Expand Down
11 changes: 7 additions & 4 deletions tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,17 @@
id: statistics
side_outputs:
glob_stats: glob_stats
- method: rescale_to_int
module_path: httomolibgpu.misc.rescale
parameters:
perc_range_min: 0.0
perc_range_max: 100.0
bits: 8
glob_stats: ${{statistics.side_outputs.glob_stats}}
- method: save_to_images
module_path: httomolib.misc.images
parameters:
subfolder_name: images
axis: auto
file_format: tif
bits: 8
perc_range_min: 0.0
perc_range_max: 95.0
jpeg_quality: 95
glob_stats: ${{statistics.side_outputs.glob_stats}}
11 changes: 7 additions & 4 deletions tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,17 @@
id: statistics
side_outputs:
glob_stats: glob_stats
- method: rescale_to_int
module_path: httomolibgpu.misc.rescale
parameters:
perc_range_min: 0.0
perc_range_max: 100.0
bits: 8
glob_stats: ${{statistics.side_outputs.glob_stats}}
- method: save_to_images
module_path: httomolib.misc.images
parameters:
subfolder_name: images
axis: auto
file_format: tif
bits: 8
perc_range_min: 0.0
perc_range_max: 95.0
jpeg_quality: 95
glob_stats: ${{statistics.side_outputs.glob_stats}}
9 changes: 6 additions & 3 deletions tests/samples/pipeline_template_examples/pipeline_cpu1.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,16 @@
sinogram_order: false
algorithm: 'gridrec'
init_recon: null
- method: rescale_to_int
module_path: httomolibgpu.misc.rescale
parameters:
perc_range_min: 0.0
perc_range_max: 100.0
bits: 8
- method: save_to_images
module_path: httomolib.misc.images
parameters:
subfolder_name: images
axis: auto
file_format: tif
bits: 8
perc_range_min: 0.0
perc_range_max: 100.0
jpeg_quality: 95
9 changes: 6 additions & 3 deletions tests/samples/pipeline_template_examples/pipeline_cpu2.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,16 @@
parameters:
size: 3
axis: auto
- method: rescale_to_int
module_path: httomolibgpu.misc.rescale
parameters:
perc_range_min: 0.0
perc_range_max: 100.0
bits: 8
- method: save_to_images
module_path: httomolib.misc.images
parameters:
subfolder_name: images
axis: auto
file_format: tif
bits: 8
perc_range_min: 0.0
perc_range_max: 100.0
jpeg_quality: 95
12 changes: 7 additions & 5 deletions tests/samples/pipeline_template_examples/pipeline_cpu3.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,17 @@
id: statistics
side_outputs:
glob_stats: glob_stats
- method: rescale_to_int
module_path: httomolibgpu.misc.rescale
parameters:
perc_range_min: 0.0
perc_range_max: 100.0
bits: 8
glob_stats: ${{statistics.side_outputs.glob_stats}}
- method: save_to_images
module_path: httomolib.misc.images
parameters:
subfolder_name: images
axis: auto
file_format: tif
bits: 8
perc_range_min: 0.0
perc_range_max: 100.0
jpeg_quality: 95
glob_stats: ${{statistics.side_outputs.glob_stats}}

9 changes: 6 additions & 3 deletions tests/samples/pipeline_template_examples/pipeline_cpu4.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,16 @@
sinogram_order: false
algorithm: 'gridrec'
init_recon: null
- method: rescale_to_int
module_path: httomolibgpu.misc.rescale
parameters:
perc_range_min: 0.0
perc_range_max: 100.0
bits: 8
- method: save_to_images
module_path: httomolib.misc.images
parameters:
subfolder_name: images
axis: auto
file_format: tif
bits: 8
perc_range_min: 0.0
perc_range_max: 100.0
jpeg_quality: 95
9 changes: 6 additions & 3 deletions tests/samples/pipeline_template_examples/pipeline_gpu1.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,16 @@
recon_size: null
recon_mask_radius: null
save_result: true
- method: rescale_to_int
module_path: httomolibgpu.misc.rescale
parameters:
perc_range_min: 0.0
perc_range_max: 100.0
bits: 8
- method: save_to_images
module_path: httomolib.misc.images
parameters:
subfolder_name: images
axis: auto
file_format: tif
bits: 8
perc_range_min: 0.0
perc_range_max: 100.0
jpeg_quality: 95
9 changes: 6 additions & 3 deletions tests/samples/pipeline_template_examples/testing/testing_pipeline.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,16 @@
algorithm: gridrec
init_recon: null
save_result: true
- method: rescale_to_int
module_path: httomolibgpu.misc.rescale
parameters:
perc_range_min: 0.0
perc_range_max: 100.0
bits: 8
- method: save_to_images
module_path: httomolib.misc.images
parameters:
subfolder_name: images
axis: 1
file_format: tif
bits: 8
perc_range_min: 0.0
perc_range_max: 100.0
jpeg_quality: 95
15 changes: 11 additions & 4 deletions tests/samples/python_templates/pipeline_cpu1.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,18 +73,25 @@ def methods_to_list() -> PipelineConfig:
}
full_pipeline_list.append(method4)
method5 = {
"method": "rescale_to_int",
"module_path": "httomolibgpu.misc.rescale",
"parameters": {
"perc_range_min": 0.0,
"perc_range_max": 100.0,
"bits": 8,
},
}
full_pipeline_list.append(method5)
method6 = {
"method": "save_to_images",
"module_path": "httomolib.misc.images",
"parameters": {
"subfolder_name": "images",
"axis": 1,
"file_format": "tif",
"bits": 8,
"perc_range_min": 0.0,
"perc_range_max": 100.0,
"jpeg_quality": 95,
},
}
full_pipeline_list.append(method5)
full_pipeline_list.append(method6)

return full_pipeline_list
15 changes: 11 additions & 4 deletions tests/samples/python_templates/pipeline_cpu2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,18 +105,25 @@ def methods_to_list() -> PipelineConfig:
}
full_pipeline_list.append(method7)
method8 = {
"method": "rescale_to_int",
"module_path": "httomolibgpu.misc.rescale",
"parameters": {
"perc_range_min": 0.0,
"perc_range_max": 100.0,
"bits": 8,
},
}
full_pipeline_list.append(method8)
method9 = {
"method": "save_to_images",
"module_path": "httomolib.misc.images",
"parameters": {
"subfolder_name": "images",
"axis": 1,
"file_format": "tif",
"bits": 8,
"perc_range_min": 0.0,
"perc_range_max": 100.0,
"jpeg_quality": 95,
},
}
full_pipeline_list.append(method8)
full_pipeline_list.append(method9)

return full_pipeline_list
17 changes: 12 additions & 5 deletions tests/samples/python_templates/pipeline_cpu3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,19 +91,26 @@ def methods_to_list() -> PipelineConfig:
}
full_pipeline_list.append(method6)
method7 = {
"method": "rescale_to_int",
"module_path": "httomolibgpu.misc.rescale",
"parameters": {
"perc_range_min": 0.0,
"perc_range_max": 100.0,
"bits": 8,
"glob_stats": "${{statistics.side_outputs.glob_stats}}",
},
}
full_pipeline_list.append(method7)
method8 = {
"method": "save_to_images",
"module_path": "httomolib.misc.images",
"parameters": {
"subfolder_name": "images",
"axis": 1,
"file_format": "tif",
"bits": 8,
"perc_range_min": 0.0,
"perc_range_max": 100.0,
"jpeg_quality": 95,
"glob_stats": "${{statistics.side_outputs.glob_stats}}",
},
}
full_pipeline_list.append(method7)
full_pipeline_list.append(method8)

return full_pipeline_list
15 changes: 11 additions & 4 deletions tests/samples/python_templates/pipeline_gpu1.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,18 +88,25 @@ def methods_to_list() -> PipelineConfig:
}
full_pipeline_list.append(method5)
method6 = {
"method": "rescale_to_int",
"module_path": "httomolibgpu.misc.rescale",
"parameters": {
"perc_range_min": 0.0,
"perc_range_max": 100.0,
"bits": 8,
},
}
full_pipeline_list.append(method6)
method7 = {
"method": "save_to_images",
"module_path": "httomolib.misc.images",
"parameters": {
"subfolder_name": "images",
"axis": 1,
"file_format": "tif",
"bits": 8,
"perc_range_min": 0.0,
"perc_range_max": 100.0,
"jpeg_quality": 95,
},
}
full_pipeline_list.append(method6)
full_pipeline_list.append(method7)

return full_pipeline_list
12 changes: 6 additions & 6 deletions tests/test_pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -454,14 +454,14 @@ def test_tomo_standard_testing_pipeline_output_with_save_all(
subprocess.check_output(cmd)

files = get_files("output_dir/")
assert len(files) == 10
assert len(files) == 11

_check_yaml(files, "temp.yaml")
_check_tif(files, 3, (160, 160))

#: check the generated h5 files
h5_files = list(filter(lambda x: ".h5" in x, files))
assert len(h5_files) == 4
assert len(h5_files) == 5

for file_to_open in h5_files:
if "tomopy-recon-tomo-gridrec.h5" in file_to_open:
Expand All @@ -488,7 +488,7 @@ def test_i12_testing_pipeline_output(
subprocess.check_output(cmd)

files = get_files("output_dir/")
assert len(files) == 17
assert len(files) == 18

_check_yaml(files, "temp.yaml")

Expand All @@ -499,7 +499,7 @@ def test_i12_testing_pipeline_output(
assert len(tif_files) == 10

h5_files = list(filter(lambda x: ".h5" in x, files))
assert len(h5_files) == 4
assert len(h5_files) == 5

gridrec_recon = list(filter(lambda x: "recon-gridrec.h5" in x, h5_files))[0]
minus_log_tomo = list(filter(lambda x: "minus_log.h5" in x, h5_files))[0]
Expand Down Expand Up @@ -600,14 +600,14 @@ def test_diad_testing_pipeline_output(
subprocess.check_output(cmd)

files = get_files("output_dir/")
assert len(files) == 9
assert len(files) == 10

_check_yaml(files, "temp.yaml")
_check_tif(files, 2, (26, 26))

#: check the generated h5 files
h5_files = list(filter(lambda x: ".h5" in x, files))
assert len(h5_files) == 4
assert len(h5_files) == 5

for file_to_open in h5_files:
if "tomopy-normalize-tomo.h5" in file_to_open:
Expand Down
Loading

0 comments on commit 6af2e2e

Please sign in to comment.