From c78eca27abb64c39a90b9ba02cb5393b012817dd Mon Sep 17 00:00:00 2001
From: Christopher Rhodes <christopher.rhodes@embl.de>
Date: Wed, 28 Feb 2024 14:00:53 +0100
Subject: [PATCH] Moved all of chaeo extension to trec-adaptive-feedback
project
---
model_server/extensions/chaeo/__init__.py | 0
.../chaeo/batch_jobs/20231026_Porto.py | 82 --------
.../extensions/chaeo/batch_jobs/__init__.py | 0
.../extensions/chaeo/conf/__init__.py | 0
model_server/extensions/chaeo/ecotaxa.py | 45 -----
.../extensions/chaeo/examples/__init__.py | 0
.../chaeo/examples/batch_obj_cla.py | 45 -----
.../chaeo/examples/label_patches.py | 17 --
...fer_labels_to_ilastik_object_classifier.py | 97 ----------
model_server/extensions/chaeo/h5util.py | 34 ----
model_server/extensions/chaeo/models.py | 112 -----------
.../20230805_kristineberg_PA.py | 78 --------
.../20230807_kristineberg_spiked.py | 52 -----
.../old_batch_jobs/20231008_Bilbao_PA.py | 91 ---------
.../old_batch_jobs/20231023_Porto_4ch.py | 78 --------
.../20231023_Porto_EtOHfixed.py | 55 ------
.../old_batch_jobs/20231023_Porto_Live.py | 56 ------
.../chaeo/old_batch_jobs/__init__.py | 0
.../chaeo/old_batch_jobs/coloring_book.py | 52 -----
.../int_test_20231028_Porto_PA.py | 81 --------
.../old_batch_jobs/proj0004-exp0038-fixed.py | 43 -----
model_server/extensions/chaeo/router.py | 42 -----
.../extensions/chaeo/tests/__init__.py | 0
.../chaeo/tests/test_roiset_workflow.py | 67 -------
model_server/extensions/chaeo/workflows.py | 177 ------------------
25 files changed, 1304 deletions(-)
delete mode 100644 model_server/extensions/chaeo/__init__.py
delete mode 100644 model_server/extensions/chaeo/batch_jobs/20231026_Porto.py
delete mode 100644 model_server/extensions/chaeo/batch_jobs/__init__.py
delete mode 100644 model_server/extensions/chaeo/conf/__init__.py
delete mode 100644 model_server/extensions/chaeo/ecotaxa.py
delete mode 100644 model_server/extensions/chaeo/examples/__init__.py
delete mode 100644 model_server/extensions/chaeo/examples/batch_obj_cla.py
delete mode 100644 model_server/extensions/chaeo/examples/label_patches.py
delete mode 100644 model_server/extensions/chaeo/examples/transfer_labels_to_ilastik_object_classifier.py
delete mode 100644 model_server/extensions/chaeo/h5util.py
delete mode 100644 model_server/extensions/chaeo/models.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/20230805_kristineberg_PA.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/20230807_kristineberg_spiked.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/20231008_Bilbao_PA.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_4ch.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_EtOHfixed.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_Live.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/__init__.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/coloring_book.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/int_test_20231028_Porto_PA.py
delete mode 100644 model_server/extensions/chaeo/old_batch_jobs/proj0004-exp0038-fixed.py
delete mode 100644 model_server/extensions/chaeo/router.py
delete mode 100644 model_server/extensions/chaeo/tests/__init__.py
delete mode 100644 model_server/extensions/chaeo/tests/test_roiset_workflow.py
delete mode 100644 model_server/extensions/chaeo/workflows.py
diff --git a/model_server/extensions/chaeo/__init__.py b/model_server/extensions/chaeo/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/model_server/extensions/chaeo/batch_jobs/20231026_Porto.py b/model_server/extensions/chaeo/batch_jobs/20231026_Porto.py
deleted file mode 100644
index 52ec5788..00000000
--- a/model_server/extensions/chaeo/batch_jobs/20231026_Porto.py
+++ /dev/null
@@ -1,82 +0,0 @@
-from pathlib import Path
-
-from model_server.base.roiset import RoiSetMetaParams, RoiSetExportParams
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv
-from model_server.extensions.chaeo.workflows import export_zstack_roiset
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-
-if __name__ == '__main__':
- sample_id = '20231026-porto'
- root = Path('y:/TREC_STOP_26_Porto/MobileLab/LSM900')
- where_czi = (root / '231026_automic/20231026-152512_lowzoom_data/LowZoom').__str__()
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0022/output',
- 'batch-output'
- )
-
- # px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
- px_ilp = Path(
- 'y:/TREC_Heidelberg/LSM900/Tina/20240109_automic/ilastik/pxAF405-8bit-embedded.ilp'
- ).__str__()
-
- params = {
- 'segmentation_channel': 0,
- 'patches_channel': 2,
- }
-
- roi_params = RoiSetMetaParams(**{
- 'mask_type': 'boxes',
- 'filters': {
- 'area': {'min': 1e3, 'max': 1e8}
- },
- 'expand_box_by': [128, 2]
- })
-
- export_params = RoiSetExportParams(**{
- 'pixel_probabilities': True,
- 'patches_3d': {},
- 'annotated_patches_2d': {
- 'draw_bounding_box': True,
- 'rgb_overlay_channels': [1, None, None],
- 'rgb_overlay_weights': [0.2, 1.0, 1.0],
- 'pad_to': 512,
- },
- 'patches_2d': {
- 'draw_bounding_box': False,
- },
- 'annotated_zstacks': {},
- 'object_classes': True,
- 'dataframe': True,
- })
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
- models = {
- 'pixel_classifier': {
- 'model': IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)}),
- 'params': {
- 'px_class': 0,
- 'px_prob_threshold': 0.25,
- }
- },
- }
-
- loop_workflow(
- input_files,
- where_output,
- export_zstack_roiset,
- models,
- # params,
- segmentation_channel=params['segmentation_channel'],
- patches_channel=params['patches_channel'],
- export_params=export_params,
- roi_params=roi_params,
- catch_and_continue=False,
- )
-
- csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
- write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900')
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/batch_jobs/__init__.py b/model_server/extensions/chaeo/batch_jobs/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/model_server/extensions/chaeo/conf/__init__.py b/model_server/extensions/chaeo/conf/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/model_server/extensions/chaeo/ecotaxa.py b/model_server/extensions/chaeo/ecotaxa.py
deleted file mode 100644
index 255552f3..00000000
--- a/model_server/extensions/chaeo/ecotaxa.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from pathlib import Path
-
-import pandas as pd
-
-def write_ecotaxa_tsv(patches_csv_path: str, where: str, sample_id: str, scope_id: str):
- # import patch output table
- df_patches = pd.read_csv(patches_csv_path)
- df_patches['img_file_name'] = df_patches['patch_filename'].apply(lambda x: '2d_patches_annotation/' + x)
-
- # make second column index level to comply w/ EcoTaxa schema
- df_patches.columns = pd.MultiIndex.from_frame(
- pd.DataFrame([
- df_patches.columns,
- df_patches.dtypes.apply(
- lambda x: '[f]' if x in ['float64'] else '[t]'
- )
- ]).T,
- names=['variable', 'data type']
- )
-
- # add new columns for ecotaxa
- df_patches.loc[:, ('acq_instrument', '[t]')] = 'Other microscope'
- df_patches.loc[:, ('acq_instrument_microscope', '[t]')] = scope_id
- df_patches.loc[:, ('sample_id', '[t]')] = sample_id
-
- df_patches.loc[:, ('acq_id', '[t]')] = df_patches.loc[:, ('input_file', '[t]')]
- df_patches.loc[:, ('object_id', '[t]')] = df_patches.loc[:, ('patch_id', '[t]')]
- df_patches.loc[:, ('process_id', '[t]')] = df_patches.loc[:, ('patch_id', '[t]')]
- df_patches.loc[:, ('acq_pixel_scale_in_micrometers', '[f]')] = df_patches.loc[:, ('pixel_scale_in_micrometers', '[f]')]
-
- cols_to_transfer = [
- 'img_file_name',
- 'object_id',
- 'acq_id',
- 'acq_instrument',
- 'acq_instrument_microscope',
- 'sample_id',
- 'process_id'
- ]
- df_export = df_patches.loc[:, pd.IndexSlice[cols_to_transfer, :]]
- df_export.to_csv(Path(where) / 'ecotaxa.tsv', sep='\t', index=False)
-
-def write_ecotaxa_tsv_chunked_subdirectories(top_dir: str, csv_filename: str, sample_id: str, scope_id: str):
- for sd in Path(top_dir).iterdir():
- write_ecotaxa_tsv((sd / csv_filename).__str__(), sd.__str__(), sample_id, scope_id)
diff --git a/model_server/extensions/chaeo/examples/__init__.py b/model_server/extensions/chaeo/examples/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/model_server/extensions/chaeo/examples/batch_obj_cla.py b/model_server/extensions/chaeo/examples/batch_obj_cla.py
deleted file mode 100644
index 752c50c5..00000000
--- a/model_server/extensions/chaeo/examples/batch_obj_cla.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from pathlib import Path
-
-from model_server.conf.testing import output_path
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from extensions.ilastik.models import PatchStackObjectClassifier
-from model_server.extensions.chaeo.workflows import export_zstack_roiset
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-
-
-if __name__ == '__main__':
- where_czi = 'c:/Users/rhodes/projects/proj0004-marine-photoactivation/data/exp0038/AutoMic/20230906-163415/Selection'
- where_output = autonumber_new_directory(
- output_path,
- 'zmask_to_automic',
- )
-
- px_ilp = Path.home() / 'model_server' / 'testing' / 'zmask' / 'AF405-bodies_boundaries.ilp'
- ob_ilp = Path.home() / 'model_server' / 'testing' / 'zmask' / 'auto_obj_after.ilp'
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'patches_channel': 4,
- 'zmask_filters': {'area': (1e3, 1e8)},
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={'P': (0, 10)}, )
-
- models = [
- IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)}),
- PatchStackObjectClassifier(params={'project_file': Path(ob_ilp)})
- ]
-
- loop_workflow(
- input_files,
- where_output,
- export_zstack_roiset,
- models,
- params,
- catch_and_continue=False,
- )
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/examples/label_patches.py b/model_server/extensions/chaeo/examples/label_patches.py
deleted file mode 100644
index 54b242ee..00000000
--- a/model_server/extensions/chaeo/examples/label_patches.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory
-from model_server.extensions.chaeo.workflows import transfer_ecotaxa_labels_to_patch_stacks
-
-if __name__ == '__main__':
- root = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0009/output')
- transfer_ecotaxa_labels_to_patch_stacks(
- where_masks=(root / 'batch-output-20231012-0016/patch_masks').__str__(),
- where_patches=(root / 'batch-output-20231012-0016/2d_patches_training').__str__(),
- object_csv=(root / 'batch-output-20231011-0003/df_objects.csv').__str__(),
- ecotaxa_tsv='c:/Users/rhodes/projects/proj0011-plankton-seg/exp0013/ecotaxa_export_10468_20231012_0930.tsv',
- where_output=autonumber_new_directory(root, 'labeled_patches'),
- )
- print('Finished')
-
-
diff --git a/model_server/extensions/chaeo/examples/transfer_labels_to_ilastik_object_classifier.py b/model_server/extensions/chaeo/examples/transfer_labels_to_ilastik_object_classifier.py
deleted file mode 100644
index 04e0c20e..00000000
--- a/model_server/extensions/chaeo/examples/transfer_labels_to_ilastik_object_classifier.py
+++ /dev/null
@@ -1,97 +0,0 @@
-from pathlib import Path
-import numpy as np
-import pandas as pd
-import skimage
-
-from model_server.base.accessors import make_patch_stack_from_file
-from model_server.extensions.chaeo.models import generate_ilastik_object_classifier
-from extensions.ilastik.models import PatchStackObjectClassifier
-from model_server.base.accessors import GenericImageDataAccessor, write_accessor_data_to_file
-
-
-def compare_object_maps(truth: GenericImageDataAccessor, inferred: GenericImageDataAccessor) -> pd.DataFrame:
- """
- Compare two object maps to assess classification results
- :param truth: t-stack of truth objects
- :param inferred: t-stack of inferred objects, presumably with same segmentation boundaries as truth
- :return: DataFrame comparing results for each frame in truth and inferred stacks
- """
- assert truth.shape == inferred.shape
- assert np.all((truth.data == 0) == (inferred.data == 0))
- assert inferred.chroma == 1
-
- labels = []
- for zi in range(0, inferred.nz):
- inf_img = inferred.data[:, :, :, zi]
-
- unique = np.unique(inf_img)
- assert unique[0] == 0
-
- dd = {'zi': zi, 'truth_label': np.unique(truth.data[:, :, :, zi])[1], 'multiples': False}
-
- if len(unique) == 1: # no object in frame
- dd['inferred_label'] = unique[0]
- elif len(unique) > 2: # multiple objects in frame, so mask out all but largest
- print(f'Found multiple nonzero unique values in label {zi}: {unique}')
- ob_id = skimage.measure.label(inf_img)
- pr = skimage.measure.regionprops_table(ob_id, properties=['label', 'area'])
- mask = inf_img == pr['label'][pr['area'].argmax()]
- dd['inferred_label'] = np.unique(mask * inf_img)[-1] # occasionally no object in frame
- dd['multiples'] = True
- else: # exactly one unique object class in frame
- dd['inferred_label'] = unique[1]
- labels.append(dd)
- return pd.DataFrame(labels)
-
-
-def infer_and_compare(classifier: PatchStackObjectClassifier, prefix, raw, mask, labels):
- result_acc, _ = classifier.infer(raw, mask)
- write_accessor_data_to_file(root / f'zstack_train_result.tif', result_acc)
-
- # write comparison tables
- df_comp = compare_object_maps(labels, result_acc)
- df_comp.to_csv(root / f'compare_{prefix}_result.csv', index=False)
- print(f'Generated ilastik project {classifier_file.name}')
- print('Truth and inferred labels match?')
- print(pd.value_counts(df_comp['truth_label'] == df_comp['inferred_label']))
-
-
-if __name__ == '__main__':
- root = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0009/output/labeled_patches-20231030-0007')
- template_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0014/template_obj.ilp')
-
- df_labels = pd.read_csv(root / 'labels_key.csv')
- label_names = list(df_labels.sort_values('annotation_class_id').annotation_class.unique())
- label_names[0] = 'none'
- assert len(label_names) >= 2
-
- # auto-populate an object classifier
- classifier_file = generate_ilastik_object_classifier(
- template_ilp,
- root / 'new_auto_obj.ilp',
- make_patch_stack_from_file(root / 'zstack_train_raw.tif'),
- make_patch_stack_from_file(root / 'zstack_train_mask.tif'),
- make_patch_stack_from_file(root / 'zstack_train_label.tif'),
- label_names,
- allow_multiple_objects=False
- )
- classifier = PatchStackObjectClassifier({'project_file': classifier_file})
-
- # verify self-consistency of training set
- infer_and_compare(
- classifier,
- 'train',
- make_patch_stack_from_file(root / 'zstack_train_raw.tif'),
- make_patch_stack_from_file(root / 'zstack_train_mask.tif'),
- make_patch_stack_from_file(root / 'zstack_train_label.tif')
- )
-
- # run test set
- infer_and_compare(
- classifier,
- 'test',
- make_patch_stack_from_file(root / 'zstack_test_raw.tif'),
- make_patch_stack_from_file(root / 'zstack_test_mask.tif'),
- make_patch_stack_from_file(root / 'zstack_test_label.tif'),
- )
-
diff --git a/model_server/extensions/chaeo/h5util.py b/model_server/extensions/chaeo/h5util.py
deleted file mode 100644
index 0b508132..00000000
--- a/model_server/extensions/chaeo/h5util.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import json
-import uuid
-
-import h5py
-
-def get_dataset_info(h5: h5py.File, lane : int = 0):
- """
- Report out specific datasets in ilastik project file HDF5
- :param h5: handle to ilastik project file, as h5py.File object
- :param lane: ilastik lane identifier
- :return: (dict) selected data values from project file
- """
- lns = f'{lane:04d}'
- lane = f'Input Data/infos/lane{lns}'
- info = {}
- for gk in ['Raw Data', 'Segmentation Image']:
- info[gk] = {}
- for dk in ['location', 'filePath', 'shape', 'nickname']:
- try:
- info[gk][dk] = h5[f'{lane}/{gk}/{dk}'][()]
- except Exception as e:
- print(e)
- try:
- info[gk]['id'] = uuid.UUID(h5[f'{lane}/{gk}/datasetId'][()].decode())
- except ValueError as e:
- info[gk]['id'] = '<invalid UUID>'
- info[gk]['axistags'] = json.loads(h5[f'{lane}/{gk}/axistags'][()].decode())
- info[gk]['axes'] = [ax['key'] for ax in info[gk]['axistags']['axes']]
-
- obj_cl_group = h5[f'ObjectClassification/LabelInputs/{lns}']
- info['misc'] = {
- 'number_of_label_inputs': len(obj_cl_group.items())
- }
- return info
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/models.py b/model_server/extensions/chaeo/models.py
deleted file mode 100644
index 865537e6..00000000
--- a/model_server/extensions/chaeo/models.py
+++ /dev/null
@@ -1,112 +0,0 @@
-from pathlib import Path
-import shutil
-
-import h5py
-import numpy as np
-import skimage
-
-from model_server.base.accessors import PatchStack
-
-
-def generate_ilastik_object_classifier(
- template_ilp: Path,
- target_ilp: Path,
- raw_stack: PatchStack,
- mask_stack: PatchStack,
- label_stack: PatchStack,
- label_names: list,
- lane: int = 0,
- allow_multiple_objects=True,
-) -> Path:
- """
- Starting with a template project file, transfer input data and labels to a new project file.
- :param template_ilp: path to existing ilastik object classifier to use as a template
- :param target_ilp: path to new classifier
- :param raw_stack: stack of patches containing raw data
- :param mask_stack: stack of patches containing object masks
- :param label_stack: stack of patches containing object labels
- :param label_names: list of label names
- :param lane: ilastik lane identifier
- :param allow_multiple_objects: skip check for multiple objects in each mask image
- :return: path to generated object classifier
- """
- assert mask_stack.shape == raw_stack.shape
- assert label_stack.shape == raw_stack.shape
-
- new_ilp = shutil.copy(template_ilp, target_ilp)
-
- accessors = {
- 'Raw Data': raw_stack,
- 'Segmentation Image': mask_stack,
- }
-
- # get labels from label image
- labels = []
- for ii in range(0, label_stack.count):
- unique = np.unique(label_stack.iat(ii))
- assert len(unique) <= 2, 'Label image contains more than one non-zero value'
- assert unique[0] == 0, 'Label image does not contain unlabeled background'
- assert unique[-1] < len(label_names) + 1, f'Label ID {unique[-1]} exceeds number of label names: {len(label_names)}'
- labels.append(unique[-1])
-
- if not allow_multiple_objects:
- ob_id = skimage.measure.label(mask_stack.iat(ii))
- num_obj = len(np.unique(ob_id)) - 1
- if num_obj > 1:
- raise MoreThanOneObjectError(f'Found {num_obj} connected objects in mask {ii}')
-
- # write to new project file
- with h5py.File(new_ilp, 'r+') as h5:
-
- for gk in ['Raw Data', 'Segmentation Image']:
- group = f'Input Data/infos/lane{lane:04d}/{gk}'
-
- # set path to input image files
- del h5[f'{group}/filePath']
- h5[f'{group}/filePath'] = accessors[gk].fpath.name
- assert not Path(h5[f'{group}/filePath'][()].decode()).is_absolute()
- assert h5[f'{group}/filePath'][()] == accessors[gk].fpath.name.encode()
- assert h5[f'{group}/location'][()] == 'FileSystem'.encode()
-
- # set input nickname
- del h5[f'{group}/nickname']
- h5[f'{group}/nickname'] = accessors[gk].fpath.stem
-
- # set input shape
- del h5[f'{group}/shape']
- shape_zyx = [accessors[gk].shape_dict[ax] for ax in ['Z', 'Y', 'X']]
- h5[f'{group}/shape'] = np.array(shape_zyx)
-
- # change key of label names
- if (k := 'ObjectClassification/LabelNames') in h5.keys():
- del h5[k]
- ln = np.array(label_names)
- h5.create_dataset(k, data=ln.astype('O'))
-
- if (k := 'ObjectClassification/MaxNumObj') in h5.keys():
- del h5[k]
- h5[k] = len(label_names) - 1
-
- del h5['currentApplet']
- h5['currentApplet'] = 1
-
- # change object labels
- if (k := f'ObjectClassification/LabelInputs/{lane:04d}') in h5.keys():
- del h5[k]
- lag = h5.create_group(k)
- for zi, la in enumerate(labels):
- lag[f'{zi}'] = np.array([0., float(la)])
-
- # delete existing classification weights
- if (k := f'ObjectExtraction/RegionFeatures/{lane:04d}') in h5.keys():
- del h5[k]
- if (k := 'ObjectClassification/ClassifierForests') in h5.keys():
- del h5[k]
-
- return Path(new_ilp)
-
-class Error(Exception):
- pass
-
-class MoreThanOneObjectError(Error):
- pass
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/20230805_kristineberg_PA.py b/model_server/extensions/chaeo/old_batch_jobs/20230805_kristineberg_PA.py
deleted file mode 100644
index 9e4e8016..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/20230805_kristineberg_PA.py
+++ /dev/null
@@ -1,78 +0,0 @@
-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-from model_server.base.accessors import CziImageFileAccessor, write_accessor_data_to_file, InMemoryDataAccessor
-from model_server.base.process import rescale
-
-
-def export_single_channel_tif_from_multichannel_czi(input_file_path, output_folder_path, channel, **kwargs):
- in_acc = CziImageFileAccessor(input_file_path)
- data = in_acc.get_one_channel_data(channel).data
- if 'rescale_zmask_clip' in kwargs:
- data = rescale(data, clip=kwargs['rescale_zmask_clip'])
- outf = Path(output_folder_path) / (Path(input_file_path).stem + '.tif')
- write_accessor_data_to_file(
- outf,
- InMemoryDataAccessor(data),
- )
- print(f'Wrote file: {outf}')
-
-if __name__ == '__main__':
- where_czi = 'c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/TREC_STOP_15_Kristineberg/230805_automic_AI_PA/20230805-122525_AI_PA_successfulrun_recognitiononPLL405cilindionas/Selection'
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0016/output',
- 'batch-output'
- )
-
- px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0016/pxAF405_dim8bit.ilp')
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'patches_channel': 2,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (1e3, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'export_pixel_probabilities': True,
- 'export_2d_patches_for_training': True,
- 'draw_bounding_box_on_2d_patch': True,
- 'export_2d_patches_for_annotation': True,
- 'export_3d_patches': False,
- 'export_annotated_zstack': True,
- 'export_patch_masks': True,
- 'zmask_clip': 0.01,
- 'rgb_overlay_channels': (3, None, None),
- 'rgb_overlay_weights': (0.5, 1.0, 1.0)
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
-
- tif_export_params = {
- 'channel': 0,
- 'rescale_zmask_clip': 0.01,
- }
-
- # loop_workflow(
- # input_files,
- # where_output,
- # export_single_channel_tif_from_multichannel_czi,
- # tif_export_params,
- # catch_and_continue=False,
- # export_batch_csvs=False,
- # write_intermediate_products=False,
- # )
-
- loop_workflow(
- input_files,
- where_output,
- export_patches_from_multichannel_zstack,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- )
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/20230807_kristineberg_spiked.py b/model_server/extensions/chaeo/old_batch_jobs/20230807_kristineberg_spiked.py
deleted file mode 100644
index cfdd0531..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/20230807_kristineberg_spiked.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-# TODO: support list-comp of single image sequence in multiple locations
-
-# TODO: split multi-pos CZI into sequence of accessors, append to list
-
-if __name__ == '__main__':
- root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/')
- where_czi = (root / 'TREC_STOP_24_Bilbao/231008_automic/20231008-162336/Selection').__str__()
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017',
- 'batch-output'
- )
-
- px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'patches_channel': 2,
- 'zmask_clip': 0.01,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (1e3, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'export_pixel_probabilities': True,
- 'export_2d_patches_for_training': True,
- 'draw_bounding_box_on_2d_patch': True,
- 'export_2d_patches_for_annotation': True,
- 'export_3d_patches': False,
- 'export_annotated_zstack': True,
- 'export_patch_masks': True,
- 'rgb_overlay_channels': (1, None, None),
- 'rgb_overlay_weights': (0.5, 1.0, 1.0)
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
- loop_workflow(
- input_files,
- where_output,
- export_patches_from_multichannel_zstack,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- )
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/20231008_Bilbao_PA.py b/model_server/extensions/chaeo/old_batch_jobs/20231008_Bilbao_PA.py
deleted file mode 100644
index e0929874..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/20231008_Bilbao_PA.py
+++ /dev/null
@@ -1,91 +0,0 @@
-from pathlib import Path
-
-import pandas as pd
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-def write_ecotaxa_tsv(patches_csv_path, where):
- # import patch output table
- df_patches = pd.read_csv(patches_csv_path)
- df_patches['img_file_name'] = df_patches['patch_filename'].apply(lambda x: '2d_patches_annotation/' + x)
-
- # make second column index level to comply w/ EcoTaxa schema
- df_patches.columns = pd.MultiIndex.from_frame(
- pd.DataFrame([
- df_patches.columns,
- df_patches.dtypes.apply(
- lambda x: '[f]' if x in ['float64'] else '[t]'
- )
- ]).T,
- names=['variable', 'data type']
- )
-
- # add new columns for ecotaxa
- df_patches.loc[:, ('acq_instrument', '[t]')] = 'Other microscope'
- df_patches.loc[:, ('acq_instrument_microscope', '[t]')] = 'EMBL-MS-Zeiss-LSM900'
- df_patches.loc[:, ('sample_id', '[t]')] = '20231008-bilbao'
-
- df_patches.loc[:, ('acq_id', '[t]')] = df_patches.loc[:, ('input_file', '[t]')]
- df_patches.loc[:, ('object_id', '[t]')] = df_patches.loc[:, ('patch_id', '[t]')]
- df_patches.loc[:, ('process_id', '[t]')] = df_patches.loc[:, ('patch_id', '[t]')]
-
- cols_to_transfer = [
- 'img_file_name',
- 'object_id',
- 'acq_id',
- 'acq_instrument',
- 'acq_instrument_microscope',
- 'sample_id',
- 'process_id'
- ]
- df_export = df_patches.loc[:, pd.IndexSlice[cols_to_transfer, :]]
- df_export.to_csv(Path(where) / 'ecotaxa.tsv', sep='\t', index=False)
-
-if __name__ == '__main__':
- root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/')
- where_czi = (root / 'TREC_STOP_24_Bilbao/231008_automic/20231008-162336/Selection').__str__()
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/output',
- 'batch-output'
- )
-
- px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'zmask_zindex': 3,
- 'patches_channel': 2,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (1e3, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'export_pixel_probabilities': True,
- 'export_2d_patches_for_training': True,
- 'draw_bounding_box_on_2d_patch': True,
- 'export_2d_patches_for_annotation': True,
- 'export_3d_patches': False,
- 'export_annotated_zstack': True,
- 'export_patch_masks': True,
- 'zmask_clip': 0.01,
- 'rgb_overlay_channels': (1, None, None),
- 'rgb_overlay_weights': (0.2, 1.0, 1.0)
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
- loop_workflow(
- input_files,
- where_output,
- export_patches_from_multichannel_zstack,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- )
-
- csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
- write_ecotaxa_tsv(csv_path, where_output)
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_4ch.py b/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_4ch.py
deleted file mode 100644
index a421c791..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_4ch.py
+++ /dev/null
@@ -1,78 +0,0 @@
-from pathlib import Path
-
-import czifile
-
-from model_server.base.accessors import write_accessor_data_to_file
-from model_server.base.czi_util import get_accessor_from_multiposition_czi
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv_chunked_subdirectories
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-
-if __name__ == '__main__':
- sample_id = '20231023-porto-4ch'
- root = Path('y:/TREC_STOP_26_Porto/MobileLab/LSM900/231023')
- czi_filepath = root / 'PK2_LM_BioO5D_231023_am_bottle_8_5_1_2_7.czi'
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0017-ehcfm-analysis/exp0001/output',
- 'batch-output'
- )
- cf = czifile.CziFile(czi_filepath.__str__())
-
- # write single-position TIF color stacks
- where_proc = Path(where_output) / 'proc'
-
- # for pos in range(0, cf.shape[cf.axes.index('S')]):
- for pos in [0, 50, 100, 150, 200]:
- fname = f'{czi_filepath.stem}_p{pos:04d}.tif'
- write_accessor_data_to_file(
- where_proc / fname,
- get_accessor_from_multiposition_czi(cf, pos)
- )
- print(f'Wrote file {fname}')
-
- px_ilp = Path('c:/Users/rhodes/projects/proj0017-ehcfm-analysis/exp0001/ilastik/px_TL_boundaries.ilp').__str__()
-
- params = {
- 'pxmap_threshold': 0.4,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 2,
- 'zmask_zindex': None,
- 'patches_channel': 2,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (2e2, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'export_pixel_probabilities': True,
- 'export_2d_patches_for_training': True,
- 'draw_bounding_box_on_2d_patch': True,
- 'export_2d_patches_for_annotation': True,
- 'export_3d_patches': False,
- 'export_annotated_zstack': True,
- 'export_patch_masks': True,
- 'zmask_clip': 0.01,
- 'rgb_overlay_channels': (1, None, None),
- 'rgb_overlay_weights': (0.2, 1.0, 1.0),
- 'draw_label_on_zstack': True,
- 'pxmap_use_all_channels': False,
- }
-
- input_files = get_matching_files(where_proc, 'tif', coord_filter={})
-
- loop_workflow(
- input_files,
- where_output,
- export_patches_from_multichannel_zstack,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- chunk_size=50,
- )
-
- write_ecotaxa_tsv_chunked_subdirectories(
- where_output,
- 'workflow_data.csv',
- sample_id=sample_id,
- scope_id='EMBL-MS-Zeiss-LSM900',
- )
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_EtOHfixed.py b/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_EtOHfixed.py
deleted file mode 100644
index abc8eeec..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_EtOHfixed.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-
-if __name__ == '__main__':
- root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/')
- where_czi = (root / 'TREC_STOP_26_Porto/231023_automic/20231023-175838_EtOHfixed/LowZoom').__str__()
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0020/output',
- 'batch-output'
- )
-
- px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'zmask_zindex': None,
- 'patches_channel': 2,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (1e3, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'export_pixel_probabilities': True,
- 'export_2d_patches_for_training': True,
- 'draw_bounding_box_on_2d_patch': True,
- 'export_2d_patches_for_annotation': True,
- 'export_3d_patches': False,
- 'export_annotated_zstack': True,
- 'export_patch_masks': True,
- 'zmask_clip': 0.01,
- 'rgb_overlay_channels': (1, None, None),
- 'rgb_overlay_weights': (0.2, 1.0, 1.0),
- 'draw_label_on_zstack': True,
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
- loop_workflow(
- input_files,
- where_output,
- export_patches_from_multichannel_zstack,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- )
-
- csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
- write_ecotaxa_tsv(csv_path, where_output, sample_id='20231023-porto', scope_id='EMBL-MS-Zeiss-LSM900')
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_Live.py b/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_Live.py
deleted file mode 100644
index e458f7a3..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_Live.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-
-if __name__ == '__main__':
- sample_id = '20231023-porto-live'
- root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/')
- where_czi = (root / 'TREC_STOP_26_Porto/231023_automic/20231023-182725_Live/LowZoom').__str__()
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0021/output',
- 'batch-output'
- )
-
- px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'zmask_zindex': None,
- 'patches_channel': 2,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (1e3, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'export_pixel_probabilities': True,
- 'export_2d_patches_for_training': True,
- 'draw_bounding_box_on_2d_patch': True,
- 'export_2d_patches_for_annotation': True,
- 'export_3d_patches': False,
- 'export_annotated_zstack': True,
- 'export_patch_masks': True,
- 'zmask_clip': 0.01,
- 'rgb_overlay_channels': (1, None, None),
- 'rgb_overlay_weights': (0.2, 1.0, 1.0),
- 'draw_label_on_zstack': True,
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
- loop_workflow(
- input_files,
- where_output,
- export_patches_from_multichannel_zstack,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- )
-
- csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
- write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900')
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/__init__.py b/model_server/extensions/chaeo/old_batch_jobs/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/model_server/extensions/chaeo/old_batch_jobs/coloring_book.py b/model_server/extensions/chaeo/old_batch_jobs/coloring_book.py
deleted file mode 100644
index e65dd1d2..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/coloring_book.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-from skimage.filters import gaussian
-from skimage.measure import label, regionprops_table
-
-import tifffile
-
-from model_server.base.accessors import PatchStack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-from model_server.base.accessors import write_accessor_data_to_file, InMemoryDataAccessor
-
-if __name__ == '__main__':
- root = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0024')
- px_ilp = root / 'px02.ilp'
- sigma = 1.0
- thresh = 0.15
- min_area = 400
-
- tf = tifffile.imread(root / '20231008-162336-z04-TL.tif')
- instack = PatchStack(np.moveaxis(tf, 0, -1))
- px_mod = IlastikPixelClassifierModel(params={'project_file': px_ilp})
- pxmaps = []
-
- def pipeline(yx_img):
- pxmap, _ = px_mod.infer(InMemoryDataAccessor(yx_img))
- smoothed = gaussian(pxmap.get_one_channel_data(0).data, sigma=sigma)
- mask = smoothed > thresh
- proc_mask = mask
- lamap = label(proc_mask).astype('uint16')
- df = (
- pd.DataFrame(
- regionprops_table(
- lamap[:, :, 0, 0],
- properties=('label', 'area')
- )
- )
- )
- la_keep = df.loc[df['area'] > min_area, 'label'].unique()
- fil_mask = np.isin(lamap, la_keep)
- inv = np.invert(fil_mask)
- return InMemoryDataAccessor(inv)
-
- results = []
- for i in range(0, instack.count):
- res = pipeline(instack.iat(i))
- results.append(res)
- outfile = root / 'output' / f'res_{i:04d}.tif'
- outfile.parent.mkdir(parents=True, exist_ok=True)
- write_accessor_data_to_file(outfile, res)
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/int_test_20231028_Porto_PA.py b/model_server/extensions/chaeo/old_batch_jobs/int_test_20231028_Porto_PA.py
deleted file mode 100644
index 4248d80e..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/int_test_20231028_Porto_PA.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv_chunked_subdirectories
-from extensions.chaeo import ZMaskExportParams
-from model_server.extensions.chaeo.workflows import export_zstack_roiset
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-
-if __name__ == '__main__':
- sample_id = '20231028-porto-PA'
- root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/')
- where_czi = (root / 'TREC_STOP_26_Porto/231028_automic/20231028-134649_successfulrun/Selection').__str__()
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0023/output',
- 'batch-output'
- )
-
- px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
-
- export_params = {
- 'pixel_probabilities': True,
- # 'patches_3d': {},
- 'patches_2d_for_training': {
- # 'draw_bounding_box': False,
- },
- 'patches_2d_for_annotation': {
- 'draw_bounding_box': True,
- 'rgb_overlay_channels': (1, None, None),
- 'rgb_overlay_weights': (0.2, 1.0, 1.0),
- },
- 'annotated_z_stack': {
- 'draw_label': True
- }
- }
- ZMaskExportParams(**export_params)
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'zmask_zindex': None,
- 'patches_channel': 2,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (1e3, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'zmask_clip': 0.01,
- # 'export_pixel_probabilities': True,
- # 'export_2d_patches_for_training': True,
- # 'draw_bounding_box_on_2d_patch': True,
- # 'export_2d_patches_for_annotation': True,
- # 'export_3d_patches': False,
- # 'export_annotated_zstack': True,
- # 'export_patch_masks': True,
-
- # 'rgb_overlay_channels': (1, None, None),
- # 'rgb_overlay_weights': (0.2, 1.0, 1.0),
- # 'draw_label_on_zstack': True,
- 'exports': ZMaskExportParams(**export_params),
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
- loop_workflow(
- input_files,
- where_output,
- export_zstack_roiset,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- chunk_size=25,
- )
-
- write_ecotaxa_tsv_chunked_subdirectories(
- where_output,
- 'workflow_data.csv',
- sample_id=sample_id,
- scope_id='EMBL-MS-Zeiss-LSM900'
- )
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/proj0004-exp0038-fixed.py b/model_server/extensions/chaeo/old_batch_jobs/proj0004-exp0038-fixed.py
deleted file mode 100644
index efa8a8e7..00000000
--- a/model_server/extensions/chaeo/old_batch_jobs/proj0004-exp0038-fixed.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-if __name__ == '__main__':
- where_czi = 'z:/rhodes/projects/proj0004-marine-photoactivation/data/exp0038/AutoMic/20230906-163415/Selection'
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0009/output',
- 'batch-output'
- )
-
- px_ilp = Path.home() / 'model_server' / 'ilastik' / 'AF405-bodies_boundaries.ilp'
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'patches_channel': 4,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (1e3, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'export_pixel_probabilities': False,
- 'export_2d_patches_for_training': True,
- 'export_2d_patches_for_annotation': True,
- 'export_3d_patches': False,
- 'export_annotated_zstack': False,
- 'export_patch_masks': True,
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={'P': (0, 10)}, )
-
- loop_workflow(
- input_files,
- where_output,
- export_patches_from_multichannel_zstack,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- )
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/router.py b/model_server/extensions/chaeo/router.py
deleted file mode 100644
index 87d75074..00000000
--- a/model_server/extensions/chaeo/router.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from fastapi import APIRouter
-
-from model_server.extensions.chaeo.workflows import export_zstack_roiset
-from model_server.base.session import Session
-from model_server.base.validators import validate_workflow_inputs
-
-router = APIRouter(
- prefix='/chaeo',
- tags=['chaeo'],
-)
-
-session = Session()
-
-@router.put('/classify_zstack/infer')
-def infer_px_then_ob_maps(
- px_model_id: str,
- ob_model_id: str,
- input_filename: str,
- pxmap_threshold: float,
- pxmap_foreground_channel: int,
- segmentation_channel: int,
- patches_channel: int,
- zmask_filters: dict = {'area': (1e3, 1e8)},
-) -> dict:
- inpath = session.paths['inbound_images'] / input_filename
- validate_workflow_inputs([px_model_id, ob_model_id], [inpath])
-
- record = export_zstack_roiset(
- inpath,
- session.paths['outbound_images'],
- [
- session.models[px_model_id]['object'],
- session.models[px_model_id]['object']
- ],
- pxmap_threshold=pxmap_threshold,
- pxmap_foreground_channel=pxmap_foreground_channel,
- segmentation_channel=segmentation_channel,
- patches_channel=patches_channel,
- zmask_filters=zmask_filters,
- )
-
- return record
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/tests/__init__.py b/model_server/extensions/chaeo/tests/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/model_server/extensions/chaeo/tests/test_roiset_workflow.py b/model_server/extensions/chaeo/tests/test_roiset_workflow.py
deleted file mode 100644
index a52fd11f..00000000
--- a/model_server/extensions/chaeo/tests/test_roiset_workflow.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import unittest
-
-from model_server.base.models import DummyInstanceSegmentationModel
-from model_server.base.roiset import RoiSetMetaParams, RoiSetExportParams
-from model_server.conf.testing import output_path, roiset_test_data
-from model_server.extensions.chaeo.workflows import export_zstack_roiset
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-from tests.test_roiset import BaseTestRoiSetMonoProducts
-
-class TestRoiSetWorkflow(BaseTestRoiSetMonoProducts, unittest.TestCase):
-
- def test_object_map_workflow(self):
- pp = roiset_test_data['pipeline_params']
-
- models = {
- 'pixel_classifier': {
- 'model': IlastikPixelClassifierModel(params={'project_file': roiset_test_data['pixel_classifier']}),
- 'params': {
- 'px_class': 0,
- 'px_prob_threshold': 0.6,
- }
- },
- 'object_classifier': {
- 'name': 'dummy',
- 'model': DummyInstanceSegmentationModel(),
- }
- }
-
- roi_params = RoiSetMetaParams(**{
- 'mask_type': 'boxes',
- 'filters': {
- 'area': {'min': 1e3, 'max': 1e8}
- },
- 'expand_box_by': [128, 2]
- })
-
- export_params = RoiSetExportParams(**{
- 'pixel_probabilities': True,
- 'patches_3d': {},
- 'annotated_patches_2d': {
- 'draw_bounding_box': True,
- 'rgb_overlay_channels': [3, None, None],
- 'rgb_overlay_weights': [0.2, 1.0, 1.0],
- 'pad_to': 512,
- },
- 'patches_2d': {
- 'draw_bounding_box': False,
- 'draw_mask': False,
- },
- 'patch_masks': {
- 'pad_to': 256,
- },
- 'annotated_zstacks': {},
- 'object_classes': True,
- 'dataframe': True,
- })
-
- export_zstack_roiset(
- roiset_test_data['multichannel_zstack']['path'],
- output_path / 'roiset' / 'workflow',
- models,
- segmentation_channel=pp['segmentation_channel'],
- patches_channel=pp['patches_channel'],
- export_params=export_params,
- roi_params=roi_params,
- )
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/workflows.py b/model_server/extensions/chaeo/workflows.py
deleted file mode 100644
index b6f362e1..00000000
--- a/model_server/extensions/chaeo/workflows.py
+++ /dev/null
@@ -1,177 +0,0 @@
-from pathlib import Path
-from typing import Dict, List
-
-import numpy as np
-import pandas as pd
-
-from skimage.measure import label
-from skimage.morphology import dilation
-from sklearn.model_selection import train_test_split
-
-from base.roiset import RoiSetMetaParams, RoiSetExportParams
-from base.process import mask_largest_object
-from base.roiset import _get_label_ids, RoiSet
-
-from model_server.base.accessors import generate_file_accessor, InMemoryDataAccessor, write_accessor_data_to_file
-from model_server.base.models import Model, InstanceSegmentationModel, SemanticSegmentationModel
-from model_server.base.workflows import Timer
-
-
-def export_zstack_roiset(
- input_file_path: str,
- output_folder_path: str,
- models: List[Model],
- segmentation_channel: int,
- patches_channel: int,
- zmask_zindex: int = None, # None for MIP,
- roi_params: RoiSetMetaParams = RoiSetMetaParams(),
- export_params: RoiSetExportParams = RoiSetExportParams(),
-) -> Dict:
- assert isinstance(models['pixel_classifier']['model'], SemanticSegmentationModel)
-
- ti = Timer()
- stack = generate_file_accessor(Path(input_file_path))
- fstem = Path(input_file_path).stem
- ti.click('file_input')
-
- # MIP if no zmask z-index is given, then classify pixels
- if isinstance(zmask_zindex, int):
- assert 0 < zmask_zindex < stack.nz
- zmask_data = stack.get_one_channel_data(channel=segmentation_channel).data[:, :, :, zmask_zindex]
- else:
- zmask_data = stack.get_one_channel_data(channel=segmentation_channel).data.max(axis=-1, keepdims=True)
- mip = InMemoryDataAccessor(zmask_data)
-
- mip_mask = models['pixel_classifier']['model'].label_pixel_class(mip, **models['pixel_classifier']['params'])
- ti.click('classify_pixels')
-
- # make zmask
- rois = RoiSet(stack, _get_label_ids(mip_mask), params=roi_params)
- ti.click('generate_zmasks')
-
- # optionally classify if an object classifier is passed
- if 'object_classifier' in models.keys():
- assert isinstance(models['object_classifier']['model'], InstanceSegmentationModel)
- rois.classify_by(
- models['object_classifier']['name'],
- patches_channel,
- models['object_classifier']['model']
- )
- ti.click('classify_objects')
-
- rois.run_exports(Path(output_folder_path), patches_channel, fstem, export_params)
- ti.click('export_roi_products')
-
- return {
- 'timer_results': ti.events,
- 'dataframe': rois.get_df(),
- 'interm': {},
- 'output_path': output_folder_path,
- }
-
-
-
-def transfer_ecotaxa_labels_to_patch_stacks(
- where_masks: str,
- where_patches: str,
- object_csv: str,
- ecotaxa_tsv: str,
- where_output: str,
- patch_size: tuple = (256, 256),
- tr_split=0.6,
- dilate_label_mask: bool = True, # to mitigate connected components error in ilastik
- allow_multiple_objects: bool = False,
-) -> Dict:
- assert tr_split > 0.5 # reduce chance that low-probability objects are omitted from training
-
- # read patch metadata
- df_obj = pd.read_csv(
- object_csv,
- )
- df_ecotaxa = pd.read_csv(
- ecotaxa_tsv,
- sep='\t',
- header=[0],
- dtype={
- ('object_annotation_date', '[t]'): str,
- ('object_annotation_time', '[t]'): str,
- ('object_annotation_category_id', '[t]'): str,
- }
- )
- df_merge = pd.merge(df_obj, df_ecotaxa, left_on='patch_id', right_on='object_id')
-
- # assign each unique lowest-level annotation to a class index
- se_unique = pd.Series(
- df_merge.object_annotation_hierarchy.unique()
- )
- df_split = (
- se_unique.str.rsplit(
- pat='>', n=1, expand=True
- )
- )
- df_labels = pd.DataFrame({
- 'annotation_class_id': df_split.index + 1,
- 'hierarchy': se_unique,
- 'annotation_class': df_split.loc[:, 1].str.lower()
- })
-
- # join patch filenames and annotation classes
- df_pf = pd.merge(
- df_merge[['patch_filename', 'object_annotation_hierarchy']],
- df_labels,
- left_on='object_annotation_hierarchy',
- right_on='hierarchy',
- )
- df_pl = df_pf[df_pf['object_annotation_hierarchy'].notnull()]
-
- # export annotation classes and their summary stats
- df_tr, df_te = train_test_split(df_pl, train_size=tr_split)
- # df_labels['counts'] = df_pl['annotation_class_id'].value_counts()
- df_labels = pd.merge(
- df_labels,
- pd.DataFrame(
- [df_pl.annotation_class_id.value_counts(), df_tr.annotation_class_id.value_counts(), df_te.annotation_class_id.value_counts()],
- index=['total', 'to_train', 'to_test']
- ).T,
- left_on='annotation_class_id',
- right_index=True,
- how='outer'
- )
- df_labels.loc[df_labels.to_train.isna(), 'to_train'] = 0
- df_labels.loc[df_labels.to_test.isna(), 'to_test'] = 0
- for col in ['total', 'to_train', 'to_test']:
- df_labels.loc[df_labels[col].isna(), col] = 0
- df_labels.to_csv(Path(where_output) / 'labels_key.csv', index=False)
-
- # export patches as z-stacks
- for (dfk, dfv) in {'train': df_tr, 'test': df_te}.items():
- zstack_keys = ['mask', 'label', 'raw']
- zstacks = {f'{dfk}_{zsk}': np.zeros((*patch_size, 1, len(dfv)), dtype='uint8') for zsk in zstack_keys}
- stack_meta = []
- for fi, pl in enumerate(dfv.itertuples(name='PatchFile')):
- fn = pl._asdict()['patch_filename']
- ac = pl._asdict()['annotation_class']
- aci = pl._asdict()['annotation_class_id']
-
- stack_meta.append({'zi': fi, 'patch_filename': fn, 'annotation_class': ac, 'annotation_class_id': aci})
- acc_bm = generate_file_accessor(Path(where_masks) / fn)
- assert acc_bm.is_mask()
- assert acc_bm.hw == patch_size, f'Unexpected patch size {patch_size}'
- assert acc_bm.chroma == 1
- assert acc_bm.nz == 1
- mask = acc_bm.data[:, :, 0, 0]
- if dilate_label_mask:
- mask = dilation(mask)
- if not allow_multiple_objects:
- ob_id = label(acc_bm.data[:, :, 0, 0])
- mask = mask_largest_object(ob_id)
- zstacks[dfk + '_mask'][:, :, 0, fi] = mask
- zstacks[dfk + '_label'][:, :, 0, fi] = (mask == 255) * aci
-
- acc_pa = generate_file_accessor(Path(where_patches) / fn)
- zstacks[dfk + '_raw'][:, :, :, fi] = acc_pa.data[:, :, :, 0]
-
- for k in zstacks.keys():
- write_accessor_data_to_file(Path(where_output) / f'zstack_{k}.tif', InMemoryDataAccessor(zstacks[k]))
-
- pd.DataFrame(stack_meta).to_csv(Path(where_output) / f'{dfk}_stack.csv', index=False)
\ No newline at end of file
--
GitLab