Renamed RoiSet export workflow in chaeo extension; optionally classify if an...

Renamed RoiSet export workflow in chaeo extension; optionally classify if an object classifier is part of models; loop_workflow now simply passes parameters through to workflow

model_server/base/roiset.py

@@ -203,7 +203,7 @@ class RoiSet(object):
                 query_str = query_str + f' & {k} > {vmin} & {k} < {vmax}'
         return df.loc[df.query(query_str).index, :]
 
-    def get_df(self) -> pd.DataFrame:  # TODO: exclude columns that refer to objects
+    def get_df(self) -> pd.DataFrame:
         return self._df
 
     def get_slices(self) -> pd.Series:
@@ -212,7 +212,7 @@ class RoiSet(object):
     def add_df_col(self, name, se: pd.Series) -> None:
         self._df[name] = se
 
-    def get_multichannel_projection(self):  # TODO: document and test
+    def get_multichannel_projection(self):
         if self.count:
             projected = project_stack_from_focal_points(
                 self._df['centroid-0'].to_numpy(),
@@ -225,8 +225,6 @@ class RoiSet(object):
             projected = self.acc_raw.data.max(axis=-1)
         return projected
 
-    # TODO: remove, since padding is implicit in PatchStack
-    # TODO: test case where patch channel is restricted
     def get_raw_patches(self, channel=None, pad_to=256, make_3d=False):  # padded, un-annotated 2d patches
         if channel:
             patches_df = self.get_patches(white_channel=channel, pad_to=pad_to)

model_server/base/util.py

@@ -79,11 +79,12 @@ def loop_workflow(
         output_folder_path: str,
         workflow_func: callable,
         models: List[Model],
-        params: dict,
+        # params: dict,
         export_batch_csvs: bool = True,
         write_intermediate_products: bool = True,
         catch_and_continue: bool = True,
         chunk_size: int = None,
+        **params,
 ):
     """
     Iteratively call the specified workflow function on each of a list of input files

model_server/extensions/chaeo/batch_jobs/20231026_Porto.py

-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-
-if __name__ == '__main__':
-    sample_id = '20231026-porto'
-    root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/')
-    where_czi = (root / 'TREC_STOP_26_Porto/231026_automic/20231026-152512_lowzoom_data/LowZoom').__str__()
-    where_output = autonumber_new_directory(
-        'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0022/output',
-        'batch-output'
-    )
-
-    px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
-
-    params = {
-        'pxmap_threshold': 0.25,
-        'pxmap_foreground_channel': 0,
-        'segmentation_channel': 0,
-        'zmask_zindex': None,
-        'patches_channel': 2,
-        'zmask_type': 'boxes',
-        'zmask_filters': {'area': (1e3, 1e8)},
-        'zmask_expand_box_by': (128, 3),
-        'export_pixel_probabilities': True,
-        'export_2d_patches_for_training': True,
-        'draw_bounding_box_on_2d_patch': True,
-        'export_2d_patches_for_annotation': True,
-        'export_3d_patches': False,
-        'export_annotated_zstack': True,
-        'export_patch_masks': True,
-        'zmask_clip': 0.01,
-        'rgb_overlay_channels': (1, None, None),
-        'rgb_overlay_weights': (0.2, 1.0, 1.0),
-        'draw_label_on_zstack': True,
-    }
-
-    input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
-    loop_workflow(
-        input_files,
-        where_output,
-        export_patches_from_multichannel_zstack,
-        [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
-        params,
-        catch_and_continue=False,
-    )
-
-    csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
-    write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900')
-
-    print('Finished')
\ No newline at end of file

model_server/extensions/chaeo/examples/batch_obj_cla.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from model_server.conf.testing import output_path
 from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
 from extensions.ilastik.models import PatchStackObjectClassifier
-from model_server.extensions.chaeo.workflows import infer_object_map_from_zstack
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
 from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
 
 
@@ -36,7 +36,7 @@ if __name__ == '__main__':
     loop_workflow(
         input_files,
         where_output,
-        infer_object_map_from_zstack,
+        export_zstack_roiset,
         models,
         params,
         catch_and_continue=False,

model_server/extensions/chaeo/old_batch_jobs/20231026_Porto.py

+from pathlib import Path
+
+from model_server.base.roiset import RoiSetMetaParams, RoiSetExportParams
+from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
+from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
+from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
+
+
+if __name__ == '__main__':
+    sample_id = '20231026-porto'
+    root = Path('y:/TREC_STOP_26_Porto/MobileLab/LSM900')
+    where_czi = (root / '231026_automic/20231026-152512_lowzoom_data/LowZoom').__str__()
+    where_output = autonumber_new_directory(
+        'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0022/output',
+        'batch-output'
+    )
+
+    # px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
+    px_ilp = Path(
+        'y:/TREC_Heidelberg/LSM900/Tina/20240109_automic/ilastik/pxAF405-8bit-embedded.ilp'
+    ).__str__()
+
+    params = {
+        # 'pxmap_threshold': 0.25,
+        # 'pxmap_foreground_channel': 0,
+        'segmentation_channel': 0,
+        # 'zmask_zindex': None,
+        'patches_channel': 2,
+        # 'zmask_type': 'boxes',
+        # 'zmask_filters': {'area': (1e3, 1e8)},
+        # 'zmask_expand_box_by': (128, 3),
+        # 'export_pixel_probabilities': True,
+        # 'export_2d_patches_for_training': True,
+        # 'draw_bounding_box_on_2d_patch': True,
+        # 'export_2d_patches_for_annotation': True,
+        # 'export_3d_patches': False,
+        # 'export_annotated_zstack': True,
+        # 'export_patch_masks': True,
+        # 'zmask_clip': 0.01,
+        # 'rgb_overlay_channels': (1, None, None),
+        # 'rgb_overlay_weights': (0.2, 1.0, 1.0),
+        # 'draw_label_on_zstack': True,
+    }
+
+    roi_params = RoiSetMetaParams(**{
+        'mask_type': 'boxes',
+        'filters': {
+            'area': {'min': 1e3, 'max': 1e8}
+        },
+        'expand_box_by': [128, 2]
+    })
+
+    export_params = RoiSetExportParams(**{
+        'pixel_probabilities': True,
+        'patches_3d': {},
+        'annotated_patches_2d': {
+            'draw_bounding_box': True,
+            'rgb_overlay_channels': [1, None, None],
+            'rgb_overlay_weights': [0.2, 1.0, 1.0],
+            'pad_to': 512,
+        },
+        'patches_2d': {
+            'draw_bounding_box': False,
+            # 'draw_mask': False,
+        },
+        # 'patch_masks': {
+        #     'pad_to': 256,
+        # },
+        'annotated_zstacks': {},
+        'object_classes': True,
+        'dataframe': True,
+    })
+
+    input_files = get_matching_files(where_czi, 'czi', coord_filter={})
+
+    models = {
+        'pixel_classifier': {
+            'model': IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)}),
+            'params': {
+                'px_class': 0,
+                'px_prob_threshold': 0.25,
+            }
+        },
+    }
+
+    loop_workflow(
+        input_files,
+        where_output,
+        export_zstack_roiset,
+        models,
+        # params,
+        segmentation_channel=params['segmentation_channel'],
+        patches_channel=params['patches_channel'],
+        export_params=export_params,
+        roi_params=roi_params,
+        catch_and_continue=False,
+    )
+
+    csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
+    write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900')
+
+    print('Finished')
\ No newline at end of file

model_server/extensions/chaeo/batch_jobs/int_test_20231028_Porto_PA.py → model_server/extensions/chaeo/old_batch_jobs/int_test_20231028_Porto_PA.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
 from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv_chunked_subdirectories
 from extensions.chaeo import ZMaskExportParams
-from model_server.extensions.chaeo.workflows import infer_object_map_from_zstack
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
 from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
 
 
@@ -64,7 +64,7 @@ if __name__ == '__main__':
     loop_workflow(
         input_files,
         where_output,
-        infer_object_map_from_zstack,
+        export_zstack_roiset,
         [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
         params,
         catch_and_continue=False,

model_server/extensions/chaeo/router.py

 from fastapi import APIRouter
 
-from model_server.extensions.chaeo.workflows import infer_object_map_from_zstack
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
 from model_server.base.session import Session
 from model_server.base.validators import validate_workflow_inputs
 
@@ -25,7 +25,7 @@ def infer_px_then_ob_maps(
     inpath = session.paths['inbound_images'] / input_filename
     validate_workflow_inputs([px_model_id, ob_model_id], [inpath])
 
-    record = infer_object_map_from_zstack(
+    record = export_zstack_roiset(
         inpath,
         session.paths['outbound_images'],
         [

model_server/extensions/chaeo/tests/test_roiset_workflow.py

@@ -3,7 +3,7 @@ import unittest
 from model_server.base.models import DummyInstanceSegmentationModel
 from model_server.base.roiset import RoiSetMetaParams, RoiSetExportParams
 from model_server.conf.testing import output_path, roiset_test_data
-from model_server.extensions.chaeo.workflows import infer_object_map_from_zstack
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
 from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
 
 from tests.test_roiset import BaseTestRoiSetMonoProducts
@@ -56,7 +56,7 @@ class TestRoiSetWorkflow(BaseTestRoiSetMonoProducts, unittest.TestCase):
             'dataframe': True,
         })
 
-        infer_object_map_from_zstack(
+        export_zstack_roiset(
             roiset_test_data['multichannel_zstack']['path'],
             output_path / 'roiset' / 'workflow',
             models,

model_server/extensions/chaeo/workflows.py

@@ -17,7 +17,7 @@ from model_server.base.models import Model, InstanceSegmentationModel, SemanticS
 from model_server.base.workflows import Timer
 
 
-def infer_object_map_from_zstack(
+def export_zstack_roiset(
         input_file_path: str,
         output_folder_path: str,
         models: List[Model],
@@ -27,9 +27,7 @@ def infer_object_map_from_zstack(
         roi_params: RoiSetMetaParams = RoiSetMetaParams(),
         export_params: RoiSetExportParams = RoiSetExportParams(),
 ) -> Dict:
-    assert len(models) == 2
     assert isinstance(models['pixel_classifier']['model'], SemanticSegmentationModel)
-    assert isinstance(models['object_classifier']['model'], InstanceSegmentationModel)
 
     ti = Timer()
     stack = generate_file_accessor(Path(input_file_path))
@@ -51,12 +49,15 @@ def infer_object_map_from_zstack(
     rois = RoiSet(stack, _get_label_ids(mip_mask), params=roi_params)
     ti.click('generate_zmasks')
 
-    rois.classify_by(
-        models['object_classifier']['name'],
-        patches_channel,
-        models['object_classifier']['model']
-    )
-    ti.click('classify_objects')
+    # optionally classify if an object classifier is passed
+    if 'object_classifier' in models.keys():
+        assert isinstance(models['object_classifier']['model'], InstanceSegmentationModel)
+        rois.classify_by(
+            models['object_classifier']['name'],
+            patches_channel,
+            models['object_classifier']['model']
+        )
+        ti.click('classify_objects')
 
     rois.run_exports(Path(output_folder_path), patches_channel, fstem, export_params)
     ti.click('export_roi_products')