From 1f118e854c37f9a55cb5624250d426df3dce92dd Mon Sep 17 00:00:00 2001
From: Christopher Rhodes <christopher.rhodes@embl.de>
Date: Fri, 20 Oct 2023 10:48:39 +0200
Subject: [PATCH] Lifted z-stack assertion, made util.loop_workflow more
generic e.g. for on-the-fly data conversion tasks
---
.../actual_runs/20230805_kristineberg_PA.py | 44 ++++++++++---------
extensions/chaeo/util.py | 40 +++++++++--------
extensions/chaeo/workflows.py | 34 +++++++-------
3 files changed, 64 insertions(+), 54 deletions(-)
diff --git a/extensions/chaeo/actual_runs/20230805_kristineberg_PA.py b/extensions/chaeo/actual_runs/20230805_kristineberg_PA.py
index 09cd9b87..2a59dfe7 100644
--- a/extensions/chaeo/actual_runs/20230805_kristineberg_PA.py
+++ b/extensions/chaeo/actual_runs/20230805_kristineberg_PA.py
@@ -3,18 +3,21 @@ from pathlib import Path
from extensions.chaeo.util import autonumber_new_directory, get_matching_files, loop_workflow
from extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.accessors import CziImageFileAccessor, write_accessor_data_to_file
+from model_server.accessors import CziImageFileAccessor, write_accessor_data_to_file, InMemoryDataAccessor
+from model_server.process import rescale
-def export_single_channel_tif_from_multichannel_czi(files, where_output, channel):
- for czif in files:
- in_acc = CziImageFileAccessor(czif)
- outf = Path(where_output) / (Path(czif).stem + '.tif')
- write_accessor_data_to_file(
- outf,
- in_acc.get_one_channel_data(channel),
- )
- print(f'Wrote file: {outf}')
+def export_single_channel_tif_from_multichannel_czi(input_file_path, output_folder_path, channel, **kwargs):
+ in_acc = CziImageFileAccessor(input_file_path)
+ data = in_acc.get_one_channel_data(channel).data
+ if 'rescale_zmask_clip' in kwargs:
+ data = rescale(data, clip=kwargs['rescale_zmask_clip'])
+ outf = Path(output_folder_path) / (Path(input_file_path).stem + '.tif')
+ write_accessor_data_to_file(
+ outf,
+ InMemoryDataAccessor(data),
+ )
+ print(f'Wrote file: {outf}')
if __name__ == '__main__':
where_czi = 'c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/TREC_STOP_15_Kristineberg/230805_automic_AI_PA/20230805-122525_AI_PA_successfulrun_recognitiononPLL405cilindionas/Selection'
@@ -45,17 +48,18 @@ if __name__ == '__main__':
input_files = get_matching_files(where_czi, 'czi')
tif_export_params = {
- 'channel': 0
+ 'channel': 0,
+ 'rescale_zmask_clip': 0.01,
}
- export_single_channel_tif_from_multichannel_czi(input_files, where_output, 0)
-
- # loop_workflow(
- # input_files,
- # where_output,
- # export_single_channel_tif_from_multichannel_czi,
- # tif_export_params,
- # catch_and_continue=False,
- # )
+ loop_workflow(
+ input_files,
+ where_output,
+ export_single_channel_tif_from_multichannel_czi,
+ tif_export_params,
+ catch_and_continue=False,
+ export_batch_csvs=False,
+ write_intermediate_products=False,
+ )
print('Finished')
\ No newline at end of file
diff --git a/extensions/chaeo/util.py b/extensions/chaeo/util.py
index 88f522f2..b127d937 100644
--- a/extensions/chaeo/util.py
+++ b/extensions/chaeo/util.py
@@ -49,13 +49,14 @@ def get_matching_files(where: str, ext: str, coord_filter: dict={}) -> list:
return files
-def loop_workflow(files, where_output, workflow_func, params,
+def loop_workflow(files, output_folder_path, workflow_func, params,
+ export_batch_csvs=True,
write_intermediate_products=True, catch_and_continue=True):
failures = []
for ii, ff in enumerate(files):
export_kwargs = {
- 'input_zstack_path': ff, # TODO: use a more generic name for implied args
- 'where_output': where_output,
+ 'input_file_path': ff,
+ 'output_folder_path': output_folder_path,
**params,
}
@@ -74,25 +75,26 @@ def loop_workflow(files, where_output, workflow_func, params,
raise e
# record dataframes associated with workflow results
- batch_csv = {
- 'workflow_data': result['dataframe'],
- 'timer_results': pd.DataFrame(result['timer_results'], index=[0]),
- 'workflow_parameters': pd.json_normalize(export_kwargs),
- }
- for k in batch_csv.keys():
- df = batch_csv[k]
- df['input_file'] = ff
- if ii == 0:
- csv_args = {'mode': 'w', 'header': True}
- else: # append to existing file
- csv_args = {'mode': 'a', 'header': False}
- csv_path = Path(where_output) / f'{k}.csv'
- df.to_csv(csv_path, index=False, **csv_args)
+ if export_batch_csvs:
+ batch_csv = {
+ 'workflow_data': result['dataframe'],
+ 'timer_results': pd.DataFrame(result['timer_results'], index=[0]),
+ 'workflow_parameters': pd.json_normalize(export_kwargs),
+ }
+ for k in batch_csv.keys():
+ df = batch_csv[k]
+ df['input_file'] = ff
+ if ii == 0:
+ csv_args = {'mode': 'w', 'header': True}
+ else: # append to existing file
+ csv_args = {'mode': 'a', 'header': False}
+ csv_path = Path(output_folder_path) / f'{k}.csv'
+ df.to_csv(csv_path, index=False, **csv_args)
# export intermediate data if flagged
if write_intermediate_products:
for k in result['interm'].keys():
- path = Path(where_output) / k / (Path(ff).stem + '.tif')
+ path = Path(output_folder_path) / k / (Path(ff).stem + '.tif')
path.parent.mkdir(parents=True, exist_ok=True)
write_accessor_data_to_file(
path,
@@ -100,4 +102,4 @@ def loop_workflow(files, where_output, workflow_func, params,
)
if len(failures) > 0:
- pd.DataFrame(failures).to_csv(Path(where_output) / 'failures.csv')
\ No newline at end of file
+ pd.DataFrame(failures).to_csv(Path(output_folder_path) / 'failures.csv')
\ No newline at end of file
diff --git a/extensions/chaeo/workflows.py b/extensions/chaeo/workflows.py
index f84cf2eb..a5cf3f6a 100644
--- a/extensions/chaeo/workflows.py
+++ b/extensions/chaeo/workflows.py
@@ -12,18 +12,20 @@ from extensions.chaeo.annotators import draw_boxes_on_3d_image
from extensions.chaeo.products import export_patches_from_zstack, export_patch_masks_from_zstack, export_multichannel_patches_from_zstack
from extensions.chaeo.zmask import build_zmask_from_object_mask, project_stack_from_focal_points
from model_server.accessors import generate_file_accessor, InMemoryDataAccessor, write_accessor_data_to_file
+from model_server.process import rescale
from model_server.workflows import Timer
# TODO: unpack and validate inputs
# TODO: expose channel indices and color balance vectors to caller
def export_patches_from_multichannel_zstack(
- input_zstack_path: str,
+ input_file_path: str,
+ output_folder_path: str,
ilastik_project_file: str,
pxmap_threshold: float,
pixel_class: int,
zmask_channel: int,
patches_channel: int,
- where_output: str,
+ rescale_zmask_clip: int = None,
mask_type: str = 'boxes',
zmask_filters: Dict = None,
zmask_expand_box_by: int = None,
@@ -35,14 +37,16 @@ def export_patches_from_multichannel_zstack(
export_patch_masks=True,
) -> Dict:
ti = Timer()
- stack = generate_file_accessor(Path(input_zstack_path))
- fstem = Path(input_zstack_path).stem
+ stack = generate_file_accessor(Path(input_file_path))
+ fstem = Path(input_file_path).stem
ti.click('file_input')
- # assert stack.nz > 1, 'Expecting z-stack'
# MIP and classify pixels
+ zmask_data = stack.get_one_channel_data(channel=zmask_channel).data.max(axis=-1, keepdims=True)
+ if rescale_zmask_clip:
+ zmask_data = rescale(zmask_data, rescale_zmask_clip)
mip = InMemoryDataAccessor(
- stack.get_one_channel_data(channel=0).data.max(axis=-1, keepdims=True)
+ zmask_data,
)
px_model = IlastikPixelClassifierModel(
params={'project_file': Path(ilastik_project_file)}
@@ -52,7 +56,7 @@ def export_patches_from_multichannel_zstack(
if export_pixel_probabilities:
write_accessor_data_to_file(
- Path(where_output) / 'pixel_probabilities' / (fstem + '.tif'),
+ Path(output_folder_path) / 'pixel_probabilities' / (fstem + '.tif'),
pxmap
)
ti.click('export_pixel_probability')
@@ -75,7 +79,7 @@ def export_patches_from_multichannel_zstack(
if export_3d_patches:
files = export_patches_from_zstack(
- Path(where_output) / '3d_patches',
+ Path(output_folder_path) / '3d_patches',
stack.get_one_channel_data(patches_channel),
zmask_meta,
prefix=fstem,
@@ -87,14 +91,14 @@ def export_patches_from_multichannel_zstack(
if export_2d_patches_for_annotation:
files = export_multichannel_patches_from_zstack(
- Path(where_output) / '2d_patches_annotation',
+ Path(output_folder_path) / '2d_patches_annotation',
stack,
zmask_meta,
prefix=fstem,
rescale_clip=0.001,
make_3d=False,
focus_metric='max_sobel',
- ch_white=4,
+ ch_white=patches_channel,
ch_rgb_overlay=(3, None, None),
draw_bounding_box=False,
bounding_box_channel=1,
@@ -112,7 +116,7 @@ def export_patches_from_multichannel_zstack(
if export_2d_patches_for_training:
files = export_multichannel_patches_from_zstack(
- Path(where_output) / '2d_patches_training',
+ Path(output_folder_path) / '2d_patches_training',
stack.get_one_channel_data(4),
zmask_meta,
prefix=fstem,
@@ -130,8 +134,8 @@ def export_patches_from_multichannel_zstack(
if export_patch_masks:
files = export_patch_masks_from_zstack(
- Path(where_output) / 'patch_masks',
- stack.get_one_channel_data(4),
+ Path(output_folder_path) / 'patch_masks',
+ stack.get_one_channel_data(patches_channel),
zmask_meta,
prefix=fstem,
)
@@ -144,7 +148,7 @@ def export_patches_from_multichannel_zstack(
)
)
write_accessor_data_to_file(
- Path(where_output) / 'annotated_zstacks' / (fstem + '.tif'),
+ Path(output_folder_path) / 'annotated_zstacks' / (fstem + '.tif'),
annotated
)
ti.click('export_annotated_zstack')
@@ -161,7 +165,7 @@ def export_patches_from_multichannel_zstack(
return {
'pixel_model_id': px_model.model_id,
- 'input_filepath': input_zstack_path,
+ 'input_filepath': input_file_path,
'number_of_objects': len(zmask_meta),
'success': True,
'timer_results': ti.events,
--
GitLab