Skip to content
Snippets Groups Projects
Commit 1f118e85 authored by Christopher Randolph Rhodes's avatar Christopher Randolph Rhodes
Browse files

Lifted z-stack assertion, made util.loop_workflow more generic e.g. for... 

Lifted z-stack assertion, made util.loop_workflow more generic e.g. for on-the-fly data conversion tasks
parent 18bfccbd
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
extensions/chaeo/actual_runs/20230805_kristineberg_PA.py
+ 24
− 20
View file @ 1f118e85
......@@ -3,18 +3,21 @@ from pathlib import Path
from extensions.chaeo.util import autonumber_new_directory, get_matching_files, loop_workflow
from extensions.chaeo.workflows import export_patches_from_multichannel_zstack
from model_server.accessors import CziImageFileAccessor, write_accessor_data_to_file
from model_server.accessors import CziImageFileAccessor, write_accessor_data_to_file, InMemoryDataAccessor
from model_server.process import rescale
def export_single_channel_tif_from_multichannel_czi(files, where_output, channel):
for czif in files:
in_acc = CziImageFileAccessor(czif)
outf = Path(where_output) / (Path(czif).stem + '.tif')
write_accessor_data_to_file(
outf,
in_acc.get_one_channel_data(channel),
)
print(f'Wrote file: {outf}')
def export_single_channel_tif_from_multichannel_czi(input_file_path, output_folder_path, channel, **kwargs):
in_acc = CziImageFileAccessor(input_file_path)
data = in_acc.get_one_channel_data(channel).data
if 'rescale_zmask_clip' in kwargs:
data = rescale(data, clip=kwargs['rescale_zmask_clip'])
outf = Path(output_folder_path) / (Path(input_file_path).stem + '.tif')
write_accessor_data_to_file(
outf,
InMemoryDataAccessor(data),
)
print(f'Wrote file: {outf}')
if __name__ == '__main__':
where_czi = 'c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/TREC_STOP_15_Kristineberg/230805_automic_AI_PA/20230805-122525_AI_PA_successfulrun_recognitiononPLL405cilindionas/Selection'
......@@ -45,17 +48,18 @@ if __name__ == '__main__':
input_files = get_matching_files(where_czi, 'czi')
tif_export_params = {
'channel': 0
'channel': 0,
'rescale_zmask_clip': 0.01,
}
export_single_channel_tif_from_multichannel_czi(input_files, where_output, 0)
# loop_workflow(
# input_files,
# where_output,
# export_single_channel_tif_from_multichannel_czi,
# tif_export_params,
# catch_and_continue=False,
# )
loop_workflow(
input_files,
where_output,
export_single_channel_tif_from_multichannel_czi,
tif_export_params,
catch_and_continue=False,
export_batch_csvs=False,
write_intermediate_products=False,
)
print('Finished')
\ No newline at end of file
extensions/chaeo/util.py
+ 21
− 19
View file @ 1f118e85
......@@ -49,13 +49,14 @@ def get_matching_files(where: str, ext: str, coord_filter: dict={}) -> list:
return files
def loop_workflow(files, where_output, workflow_func, params,
def loop_workflow(files, output_folder_path, workflow_func, params,
export_batch_csvs=True,
write_intermediate_products=True, catch_and_continue=True):
failures = []
for ii, ff in enumerate(files):
export_kwargs = {
'input_zstack_path': ff, # TODO: use a more generic name for implied args
'where_output': where_output,
'input_file_path': ff,
'output_folder_path': output_folder_path,
**params,
}
......@@ -74,25 +75,26 @@ def loop_workflow(files, where_output, workflow_func, params,
raise e
# record dataframes associated with workflow results
batch_csv = {
'workflow_data': result['dataframe'],
'timer_results': pd.DataFrame(result['timer_results'], index=[0]),
'workflow_parameters': pd.json_normalize(export_kwargs),
}
for k in batch_csv.keys():
df = batch_csv[k]
df['input_file'] = ff
if ii == 0:
csv_args = {'mode': 'w', 'header': True}
else: # append to existing file
csv_args = {'mode': 'a', 'header': False}
csv_path = Path(where_output) / f'{k}.csv'
df.to_csv(csv_path, index=False, **csv_args)
if export_batch_csvs:
batch_csv = {
'workflow_data': result['dataframe'],
'timer_results': pd.DataFrame(result['timer_results'], index=[0]),
'workflow_parameters': pd.json_normalize(export_kwargs),
}
for k in batch_csv.keys():
df = batch_csv[k]
df['input_file'] = ff
if ii == 0:
csv_args = {'mode': 'w', 'header': True}
else: # append to existing file
csv_args = {'mode': 'a', 'header': False}
csv_path = Path(output_folder_path) / f'{k}.csv'
df.to_csv(csv_path, index=False, **csv_args)
# export intermediate data if flagged
if write_intermediate_products:
for k in result['interm'].keys():
path = Path(where_output) / k / (Path(ff).stem + '.tif')
path = Path(output_folder_path) / k / (Path(ff).stem + '.tif')
path.parent.mkdir(parents=True, exist_ok=True)
write_accessor_data_to_file(
path,
......@@ -100,4 +102,4 @@ def loop_workflow(files, where_output, workflow_func, params,
)
if len(failures) > 0:
pd.DataFrame(failures).to_csv(Path(where_output) / 'failures.csv')
\ No newline at end of file
pd.DataFrame(failures).to_csv(Path(output_folder_path) / 'failures.csv')
\ No newline at end of file
extensions/chaeo/workflows.py
+ 19
− 15
View file @ 1f118e85
......@@ -12,18 +12,20 @@ from extensions.chaeo.annotators import draw_boxes_on_3d_image
from extensions.chaeo.products import export_patches_from_zstack, export_patch_masks_from_zstack, export_multichannel_patches_from_zstack
from extensions.chaeo.zmask import build_zmask_from_object_mask, project_stack_from_focal_points
from model_server.accessors import generate_file_accessor, InMemoryDataAccessor, write_accessor_data_to_file
from model_server.process import rescale
from model_server.workflows import Timer
# TODO: unpack and validate inputs
# TODO: expose channel indices and color balance vectors to caller
def export_patches_from_multichannel_zstack(
input_zstack_path: str,
input_file_path: str,
output_folder_path: str,
ilastik_project_file: str,
pxmap_threshold: float,
pixel_class: int,
zmask_channel: int,
patches_channel: int,
where_output: str,
rescale_zmask_clip: int = None,
mask_type: str = 'boxes',
zmask_filters: Dict = None,
zmask_expand_box_by: int = None,
......@@ -35,14 +37,16 @@ def export_patches_from_multichannel_zstack(
export_patch_masks=True,
) -> Dict:
ti = Timer()
stack = generate_file_accessor(Path(input_zstack_path))
fstem = Path(input_zstack_path).stem
stack = generate_file_accessor(Path(input_file_path))
fstem = Path(input_file_path).stem
ti.click('file_input')
# assert stack.nz > 1, 'Expecting z-stack'
# MIP and classify pixels
zmask_data = stack.get_one_channel_data(channel=zmask_channel).data.max(axis=-1, keepdims=True)
if rescale_zmask_clip:
zmask_data = rescale(zmask_data, rescale_zmask_clip)
mip = InMemoryDataAccessor(
stack.get_one_channel_data(channel=0).data.max(axis=-1, keepdims=True)
zmask_data,
)
px_model = IlastikPixelClassifierModel(
params={'project_file': Path(ilastik_project_file)}
......@@ -52,7 +56,7 @@ def export_patches_from_multichannel_zstack(
if export_pixel_probabilities:
write_accessor_data_to_file(
Path(where_output) / 'pixel_probabilities' / (fstem + '.tif'),
Path(output_folder_path) / 'pixel_probabilities' / (fstem + '.tif'),
pxmap
)
ti.click('export_pixel_probability')
......@@ -75,7 +79,7 @@ def export_patches_from_multichannel_zstack(
if export_3d_patches:
files = export_patches_from_zstack(
Path(where_output) / '3d_patches',
Path(output_folder_path) / '3d_patches',
stack.get_one_channel_data(patches_channel),
zmask_meta,
prefix=fstem,
......@@ -87,14 +91,14 @@ def export_patches_from_multichannel_zstack(
if export_2d_patches_for_annotation:
files = export_multichannel_patches_from_zstack(
Path(where_output) / '2d_patches_annotation',
Path(output_folder_path) / '2d_patches_annotation',
stack,
zmask_meta,
prefix=fstem,
rescale_clip=0.001,
make_3d=False,
focus_metric='max_sobel',
ch_white=4,
ch_white=patches_channel,
ch_rgb_overlay=(3, None, None),
draw_bounding_box=False,
bounding_box_channel=1,
......@@ -112,7 +116,7 @@ def export_patches_from_multichannel_zstack(
if export_2d_patches_for_training:
files = export_multichannel_patches_from_zstack(
Path(where_output) / '2d_patches_training',
Path(output_folder_path) / '2d_patches_training',
stack.get_one_channel_data(4),
zmask_meta,
prefix=fstem,
......@@ -130,8 +134,8 @@ def export_patches_from_multichannel_zstack(
if export_patch_masks:
files = export_patch_masks_from_zstack(
Path(where_output) / 'patch_masks',
stack.get_one_channel_data(4),
Path(output_folder_path) / 'patch_masks',
stack.get_one_channel_data(patches_channel),
zmask_meta,
prefix=fstem,
)
......@@ -144,7 +148,7 @@ def export_patches_from_multichannel_zstack(
)
)
write_accessor_data_to_file(
Path(where_output) / 'annotated_zstacks' / (fstem + '.tif'),
Path(output_folder_path) / 'annotated_zstacks' / (fstem + '.tif'),
annotated
)
ti.click('export_annotated_zstack')
......@@ -161,7 +165,7 @@ def export_patches_from_multichannel_zstack(
return {
'pixel_model_id': px_model.model_id,
'input_filepath': input_zstack_path,
'input_filepath': input_file_path,
'number_of_objects': len(zmask_meta),
'success': True,
'timer_results': ti.events,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment