diff --git a/model_server/base/roiset.py b/model_server/base/roiset.py
index a66f97414c61b163403abfb75d96c119f616400b..7c3b3ae907a0ae8ccc1e709cfa2ae8f5b8e58ed8 100644
--- a/model_server/base/roiset.py
+++ b/model_server/base/roiset.py
@@ -203,7 +203,7 @@ class RoiSet(object):
query_str = query_str + f' & {k} > {vmin} & {k} < {vmax}'
return df.loc[df.query(query_str).index, :]
- def get_df(self) -> pd.DataFrame: # TODO: exclude columns that refer to objects
+ def get_df(self) -> pd.DataFrame:
return self._df
def get_slices(self) -> pd.Series:
@@ -212,7 +212,7 @@ class RoiSet(object):
def add_df_col(self, name, se: pd.Series) -> None:
self._df[name] = se
- def get_multichannel_projection(self): # TODO: document and test
+ def get_multichannel_projection(self):
if self.count:
projected = project_stack_from_focal_points(
self._df['centroid-0'].to_numpy(),
@@ -225,8 +225,6 @@ class RoiSet(object):
projected = self.acc_raw.data.max(axis=-1)
return projected
- # TODO: remove, since padding is implicit in PatchStack
- # TODO: test case where patch channel is restricted
def get_raw_patches(self, channel=None, pad_to=256, make_3d=False): # padded, un-annotated 2d patches
if channel:
patches_df = self.get_patches(white_channel=channel, pad_to=pad_to)
diff --git a/model_server/base/util.py b/model_server/base/util.py
index 0d35d3c4bf267c95bc8e9527db473407f75f9c1b..112118832acb0d15caed1ef29118c3bcc43ea7df 100644
--- a/model_server/base/util.py
+++ b/model_server/base/util.py
@@ -79,11 +79,12 @@ def loop_workflow(
output_folder_path: str,
workflow_func: callable,
models: List[Model],
- params: dict,
+ # params: dict,
export_batch_csvs: bool = True,
write_intermediate_products: bool = True,
catch_and_continue: bool = True,
chunk_size: int = None,
+ **params,
):
"""
Iteratively call the specified workflow function on each of a list of input files
diff --git a/model_server/extensions/chaeo/batch_jobs/20231026_Porto.py b/model_server/extensions/chaeo/batch_jobs/20231026_Porto.py
deleted file mode 100644
index 17b83355d9855d32182dc20c2b30c48523a40ecd..0000000000000000000000000000000000000000
--- a/model_server/extensions/chaeo/batch_jobs/20231026_Porto.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from pathlib import Path
-
-from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
-from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv
-from model_server.extensions.chaeo.workflows import export_patches_from_multichannel_zstack
-from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
-
-
-if __name__ == '__main__':
- sample_id = '20231026-porto'
- root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/')
- where_czi = (root / 'TREC_STOP_26_Porto/231026_automic/20231026-152512_lowzoom_data/LowZoom').__str__()
- where_output = autonumber_new_directory(
- 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0022/output',
- 'batch-output'
- )
-
- px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
-
- params = {
- 'pxmap_threshold': 0.25,
- 'pxmap_foreground_channel': 0,
- 'segmentation_channel': 0,
- 'zmask_zindex': None,
- 'patches_channel': 2,
- 'zmask_type': 'boxes',
- 'zmask_filters': {'area': (1e3, 1e8)},
- 'zmask_expand_box_by': (128, 3),
- 'export_pixel_probabilities': True,
- 'export_2d_patches_for_training': True,
- 'draw_bounding_box_on_2d_patch': True,
- 'export_2d_patches_for_annotation': True,
- 'export_3d_patches': False,
- 'export_annotated_zstack': True,
- 'export_patch_masks': True,
- 'zmask_clip': 0.01,
- 'rgb_overlay_channels': (1, None, None),
- 'rgb_overlay_weights': (0.2, 1.0, 1.0),
- 'draw_label_on_zstack': True,
- }
-
- input_files = get_matching_files(where_czi, 'czi', coord_filter={})
-
- loop_workflow(
- input_files,
- where_output,
- export_patches_from_multichannel_zstack,
- [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
- params,
- catch_and_continue=False,
- )
-
- csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
- write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900')
-
- print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/examples/batch_obj_cla.py b/model_server/extensions/chaeo/examples/batch_obj_cla.py
index eef5e70343061d6a47d09aac236395df141d1c17..752c50c56fb2054a5b6d947d753e1edfa1c97534 100644
--- a/model_server/extensions/chaeo/examples/batch_obj_cla.py
+++ b/model_server/extensions/chaeo/examples/batch_obj_cla.py
@@ -3,7 +3,7 @@ from pathlib import Path
from model_server.conf.testing import output_path
from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
from extensions.ilastik.models import PatchStackObjectClassifier
-from model_server.extensions.chaeo.workflows import infer_object_map_from_zstack
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
@@ -36,7 +36,7 @@ if __name__ == '__main__':
loop_workflow(
input_files,
where_output,
- infer_object_map_from_zstack,
+ export_zstack_roiset,
models,
params,
catch_and_continue=False,
diff --git a/model_server/extensions/chaeo/batch_jobs/20230807_kristineberg_spiked.py b/model_server/extensions/chaeo/old_batch_jobs/20230807_kristineberg_spiked.py
similarity index 100%
rename from model_server/extensions/chaeo/batch_jobs/20230807_kristineberg_spiked.py
rename to model_server/extensions/chaeo/old_batch_jobs/20230807_kristineberg_spiked.py
diff --git a/model_server/extensions/chaeo/batch_jobs/20231008_Bilbao_PA.py b/model_server/extensions/chaeo/old_batch_jobs/20231008_Bilbao_PA.py
similarity index 100%
rename from model_server/extensions/chaeo/batch_jobs/20231008_Bilbao_PA.py
rename to model_server/extensions/chaeo/old_batch_jobs/20231008_Bilbao_PA.py
diff --git a/model_server/extensions/chaeo/batch_jobs/20231023_Porto_4ch.py b/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_4ch.py
similarity index 100%
rename from model_server/extensions/chaeo/batch_jobs/20231023_Porto_4ch.py
rename to model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_4ch.py
diff --git a/model_server/extensions/chaeo/batch_jobs/20231023_Porto_EtOHfixed.py b/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_EtOHfixed.py
similarity index 100%
rename from model_server/extensions/chaeo/batch_jobs/20231023_Porto_EtOHfixed.py
rename to model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_EtOHfixed.py
diff --git a/model_server/extensions/chaeo/batch_jobs/20231023_Porto_Live.py b/model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_Live.py
similarity index 100%
rename from model_server/extensions/chaeo/batch_jobs/20231023_Porto_Live.py
rename to model_server/extensions/chaeo/old_batch_jobs/20231023_Porto_Live.py
diff --git a/model_server/extensions/chaeo/old_batch_jobs/20231026_Porto.py b/model_server/extensions/chaeo/old_batch_jobs/20231026_Porto.py
new file mode 100644
index 0000000000000000000000000000000000000000..2448034658889dfe5fa105a8c43257fbfb55e308
--- /dev/null
+++ b/model_server/extensions/chaeo/old_batch_jobs/20231026_Porto.py
@@ -0,0 +1,103 @@
+from pathlib import Path
+
+from model_server.base.roiset import RoiSetMetaParams, RoiSetExportParams
+from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
+from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
+from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
+
+
+if __name__ == '__main__':
+ sample_id = '20231026-porto'
+ root = Path('y:/TREC_STOP_26_Porto/MobileLab/LSM900')
+ where_czi = (root / '231026_automic/20231026-152512_lowzoom_data/LowZoom').__str__()
+ where_output = autonumber_new_directory(
+ 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0022/output',
+ 'batch-output'
+ )
+
+ # px_ilp = Path('c:/Users/rhodes/projects/proj0011-plankton-seg/exp0017/pxAF405_dim8bit.ilp').__str__()
+ px_ilp = Path(
+ 'y:/TREC_Heidelberg/LSM900/Tina/20240109_automic/ilastik/pxAF405-8bit-embedded.ilp'
+ ).__str__()
+
+ params = {
+ # 'pxmap_threshold': 0.25,
+ # 'pxmap_foreground_channel': 0,
+ 'segmentation_channel': 0,
+ # 'zmask_zindex': None,
+ 'patches_channel': 2,
+ # 'zmask_type': 'boxes',
+ # 'zmask_filters': {'area': (1e3, 1e8)},
+ # 'zmask_expand_box_by': (128, 3),
+ # 'export_pixel_probabilities': True,
+ # 'export_2d_patches_for_training': True,
+ # 'draw_bounding_box_on_2d_patch': True,
+ # 'export_2d_patches_for_annotation': True,
+ # 'export_3d_patches': False,
+ # 'export_annotated_zstack': True,
+ # 'export_patch_masks': True,
+ # 'zmask_clip': 0.01,
+ # 'rgb_overlay_channels': (1, None, None),
+ # 'rgb_overlay_weights': (0.2, 1.0, 1.0),
+ # 'draw_label_on_zstack': True,
+ }
+
+ roi_params = RoiSetMetaParams(**{
+ 'mask_type': 'boxes',
+ 'filters': {
+ 'area': {'min': 1e3, 'max': 1e8}
+ },
+ 'expand_box_by': [128, 2]
+ })
+
+ export_params = RoiSetExportParams(**{
+ 'pixel_probabilities': True,
+ 'patches_3d': {},
+ 'annotated_patches_2d': {
+ 'draw_bounding_box': True,
+ 'rgb_overlay_channels': [1, None, None],
+ 'rgb_overlay_weights': [0.2, 1.0, 1.0],
+ 'pad_to': 512,
+ },
+ 'patches_2d': {
+ 'draw_bounding_box': False,
+ # 'draw_mask': False,
+ },
+ # 'patch_masks': {
+ # 'pad_to': 256,
+ # },
+ 'annotated_zstacks': {},
+ 'object_classes': True,
+ 'dataframe': True,
+ })
+
+ input_files = get_matching_files(where_czi, 'czi', coord_filter={})
+
+ models = {
+ 'pixel_classifier': {
+ 'model': IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)}),
+ 'params': {
+ 'px_class': 0,
+ 'px_prob_threshold': 0.25,
+ }
+ },
+ }
+
+ loop_workflow(
+ input_files,
+ where_output,
+ export_zstack_roiset,
+ models,
+ # params,
+ segmentation_channel=params['segmentation_channel'],
+ patches_channel=params['patches_channel'],
+ export_params=export_params,
+ roi_params=roi_params,
+ catch_and_continue=False,
+ )
+
+ csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
+ write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900')
+
+ print('Finished')
\ No newline at end of file
diff --git a/model_server/extensions/chaeo/old_batch_jobs/__init__.py b/model_server/extensions/chaeo/old_batch_jobs/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/model_server/extensions/chaeo/batch_jobs/coloring_book.py b/model_server/extensions/chaeo/old_batch_jobs/coloring_book.py
similarity index 100%
rename from model_server/extensions/chaeo/batch_jobs/coloring_book.py
rename to model_server/extensions/chaeo/old_batch_jobs/coloring_book.py
diff --git a/model_server/extensions/chaeo/batch_jobs/int_test_20231028_Porto_PA.py b/model_server/extensions/chaeo/old_batch_jobs/int_test_20231028_Porto_PA.py
similarity index 95%
rename from model_server/extensions/chaeo/batch_jobs/int_test_20231028_Porto_PA.py
rename to model_server/extensions/chaeo/old_batch_jobs/int_test_20231028_Porto_PA.py
index 6007943bc394d0d17a751621e2273ef0260de3e6..4248d80e509257d325efbc0675c479026a94b9a1 100644
--- a/model_server/extensions/chaeo/batch_jobs/int_test_20231028_Porto_PA.py
+++ b/model_server/extensions/chaeo/old_batch_jobs/int_test_20231028_Porto_PA.py
@@ -3,7 +3,7 @@ from pathlib import Path
from model_server.base.util import autonumber_new_directory, get_matching_files, loop_workflow
from model_server.extensions.chaeo.ecotaxa import write_ecotaxa_tsv_chunked_subdirectories
from extensions.chaeo import ZMaskExportParams
-from model_server.extensions.chaeo.workflows import infer_object_map_from_zstack
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
@@ -64,7 +64,7 @@ if __name__ == '__main__':
loop_workflow(
input_files,
where_output,
- infer_object_map_from_zstack,
+ export_zstack_roiset,
[IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
params,
catch_and_continue=False,
diff --git a/model_server/extensions/chaeo/batch_jobs/proj0004-exp0038-fixed.py b/model_server/extensions/chaeo/old_batch_jobs/proj0004-exp0038-fixed.py
similarity index 100%
rename from model_server/extensions/chaeo/batch_jobs/proj0004-exp0038-fixed.py
rename to model_server/extensions/chaeo/old_batch_jobs/proj0004-exp0038-fixed.py
diff --git a/model_server/extensions/chaeo/router.py b/model_server/extensions/chaeo/router.py
index decf71c6852b41fe2295776f14ba43d2ca4559b8..87d75074f2819d24069146080d9321b9985207a8 100644
--- a/model_server/extensions/chaeo/router.py
+++ b/model_server/extensions/chaeo/router.py
@@ -1,6 +1,6 @@
from fastapi import APIRouter
-from model_server.extensions.chaeo.workflows import infer_object_map_from_zstack
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
from model_server.base.session import Session
from model_server.base.validators import validate_workflow_inputs
@@ -25,7 +25,7 @@ def infer_px_then_ob_maps(
inpath = session.paths['inbound_images'] / input_filename
validate_workflow_inputs([px_model_id, ob_model_id], [inpath])
- record = infer_object_map_from_zstack(
+ record = export_zstack_roiset(
inpath,
session.paths['outbound_images'],
[
diff --git a/model_server/extensions/chaeo/tests/test_roiset_workflow.py b/model_server/extensions/chaeo/tests/test_roiset_workflow.py
index 53c830b830d3519d9a17ff798a84c691f6724ea0..a52fd11f650a2fd5d5ee0656b9b03ad2aa7b1aab 100644
--- a/model_server/extensions/chaeo/tests/test_roiset_workflow.py
+++ b/model_server/extensions/chaeo/tests/test_roiset_workflow.py
@@ -3,7 +3,7 @@ import unittest
from model_server.base.models import DummyInstanceSegmentationModel
from model_server.base.roiset import RoiSetMetaParams, RoiSetExportParams
from model_server.conf.testing import output_path, roiset_test_data
-from model_server.extensions.chaeo.workflows import infer_object_map_from_zstack
+from model_server.extensions.chaeo.workflows import export_zstack_roiset
from model_server.extensions.ilastik.models import IlastikPixelClassifierModel
from tests.test_roiset import BaseTestRoiSetMonoProducts
@@ -56,7 +56,7 @@ class TestRoiSetWorkflow(BaseTestRoiSetMonoProducts, unittest.TestCase):
'dataframe': True,
})
- infer_object_map_from_zstack(
+ export_zstack_roiset(
roiset_test_data['multichannel_zstack']['path'],
output_path / 'roiset' / 'workflow',
models,
diff --git a/model_server/extensions/chaeo/workflows.py b/model_server/extensions/chaeo/workflows.py
index c651f95ecdf68f4b8f3aa431e6d4e512a2467aa4..b6f362e1ba54418876acd6cde310232c2a7b286c 100644
--- a/model_server/extensions/chaeo/workflows.py
+++ b/model_server/extensions/chaeo/workflows.py
@@ -17,7 +17,7 @@ from model_server.base.models import Model, InstanceSegmentationModel, SemanticS
from model_server.base.workflows import Timer
-def infer_object_map_from_zstack(
+def export_zstack_roiset(
input_file_path: str,
output_folder_path: str,
models: List[Model],
@@ -27,9 +27,7 @@ def infer_object_map_from_zstack(
roi_params: RoiSetMetaParams = RoiSetMetaParams(),
export_params: RoiSetExportParams = RoiSetExportParams(),
) -> Dict:
- assert len(models) == 2
assert isinstance(models['pixel_classifier']['model'], SemanticSegmentationModel)
- assert isinstance(models['object_classifier']['model'], InstanceSegmentationModel)
ti = Timer()
stack = generate_file_accessor(Path(input_file_path))
@@ -51,12 +49,15 @@ def infer_object_map_from_zstack(
rois = RoiSet(stack, _get_label_ids(mip_mask), params=roi_params)
ti.click('generate_zmasks')
- rois.classify_by(
- models['object_classifier']['name'],
- patches_channel,
- models['object_classifier']['model']
- )
- ti.click('classify_objects')
+ # optionally classify if an object classifier is passed
+ if 'object_classifier' in models.keys():
+ assert isinstance(models['object_classifier']['model'], InstanceSegmentationModel)
+ rois.classify_by(
+ models['object_classifier']['name'],
+ patches_channel,
+ models['object_classifier']['model']
+ )
+ ti.click('classify_objects')
rois.run_exports(Path(output_folder_path), patches_channel, fstem, export_params)
ti.click('export_roi_products')