diff --git a/model_server/base/accessors.py b/model_server/base/accessors.py
index bc74f919f3d2cc294398e921cffcdb7673754232..702388158c8b083f8c49898c8dc67dc1599d629e 100644
--- a/model_server/base/accessors.py
+++ b/model_server/base/accessors.py
@@ -73,6 +73,9 @@ class GenericImageDataAccessor(ABC):
def dtype(self):
return self.data.dtype
+ def write(self, fp: Path, mkdir=True):
+ write_accessor_data_to_file(fp, self, mkdir=mkdir)
+
def get_axis(self, ch):
return self.axes.index(ch.upper())
@@ -122,6 +125,10 @@ class GenericImageFileAccessor(GenericImageDataAccessor): # image data is loaded
raise FileAccessorError(f'Could not find file at {fpath}')
self.fpath = fpath
+ @staticmethod
+ def read(fp: Path):
+ return generate_file_accessor(fp)
+
class TifSingleSeriesFileAccessor(GenericImageFileAccessor):
def __init__(self, fpath: Path):
super().__init__(fpath)
diff --git a/model_server/base/roiset.py b/model_server/base/roiset.py
index 28c1ce2d03d31695e91de1f98d1c428ed2cd7a19..2e47c7c58bcdde1a0ed4a0e0aa4e0511c363d36c 100644
--- a/model_server/base/roiset.py
+++ b/model_server/base/roiset.py
@@ -73,14 +73,18 @@ def _get_label_ids(acc_seg_mask: GenericImageDataAccessor, allow_3d=False, conne
"""
if allow_3d and connect_3d:
nda_la = label(
- acc_seg_mask.data[:, :, 0, :]
+ acc_seg_mask.data[:, :, 0, :],
+ connectivity=3,
).astype('uint16')
return InMemoryDataAccessor(np.expand_dims(nda_la, 2))
elif allow_3d and not connect_3d:
nla = 0
la_3d = np.zeros((*acc_seg_mask.hw, 1, acc_seg_mask.nz), dtype='uint16')
for zi in range(0, acc_seg_mask.nz):
- la_2d = label(acc_seg_mask.data[:, :, 0, zi]).astype('uint16')
+ la_2d = label(
+ acc_seg_mask.data[:, :, 0, zi],
+ connectivity=2,
+ ).astype('uint16')
la_2d[la_2d > 0] = la_2d[la_2d > 0] + nla
nla = la_2d.max()
la_3d[:, :, 0, zi] = la_2d
@@ -88,7 +92,8 @@ def _get_label_ids(acc_seg_mask: GenericImageDataAccessor, allow_3d=False, conne
else:
return InMemoryDataAccessor(
label(
- acc_seg_mask.data[:, :, 0, :].max(axis=-1)
+ acc_seg_mask.data[:, :, 0, :].max(axis=-1),
+ connectivity=1,
).astype('uint16')
)
diff --git a/model_server/clients/ilastik_map_objects.py b/model_server/clients/ilastik_map_objects.py
index 4fe5179171d0c70b00396f14e3b5cd5efbbecf7c..48ae65953e6388e6b9ff04dd681c9b763c94c078 100644
--- a/model_server/clients/ilastik_map_objects.py
+++ b/model_server/clients/ilastik_map_objects.py
@@ -44,7 +44,7 @@ def main(request_func, in_abspath, params):
resp = request_func(
'PUT',
'/ilastik/seg/load/',
- {
+ body={
'project_file': px_ilp,
'duplicate': False,
},
@@ -55,7 +55,7 @@ def main(request_func, in_abspath, params):
# load object classifier
resp = request_func(
'PUT', '/ilastik/pxmap_to_obj/load/',
- {
+ body={
'project_file': ob_ilp,
'duplicate': False,
},
diff --git a/model_server/extensions/ilastik/models.py b/model_server/extensions/ilastik/models.py
index 4a1c4c0ab785e7b2eac3b481447390d2ad28749d..e8afb2381fe0f950113492721a3d7a8c899f2e2e 100644
--- a/model_server/extensions/ilastik/models.py
+++ b/model_server/extensions/ilastik/models.py
@@ -15,6 +15,7 @@ from model_server.base.models import Model, ImageToImageModel, InstanceSegmentat
class IlastikParams(BaseModel):
project_file: str
+ duplicate: bool = True
class IlastikModel(Model):
diff --git a/model_server/extensions/ilastik/router.py b/model_server/extensions/ilastik/router.py
index 9ad2733e8cbeba14674a57e25df5d2b8cd61874d..491e7e40ab678c34426ef0ae8a846c2fe065ea2f 100644
--- a/model_server/extensions/ilastik/router.py
+++ b/model_server/extensions/ilastik/router.py
@@ -15,7 +15,7 @@ router = APIRouter(
session = Session()
-def load_ilastik_model(model_class: ilm.IlastikModel, params: dict, duplicate=True) -> dict:
+def load_ilastik_model(model_class: ilm.IlastikModel, params: ilm.IlastikParams) -> dict:
"""
Load an ilastik model of a given class and project filename.
:param model_class:
@@ -24,7 +24,7 @@ def load_ilastik_model(model_class: ilm.IlastikModel, params: dict, duplicate=Tr
:return: dict containing model's ID
"""
project_file = params.project_file
- if not duplicate:
+ if not params.duplicate:
existing_model_id = session.find_param_in_loaded_models('project_file', project_file, is_path=True)
if existing_model_id is not None:
session.log_info(f'An ilastik model from {project_file} already existing exists; did not load a duplicate')
@@ -34,27 +34,24 @@ def load_ilastik_model(model_class: ilm.IlastikModel, params: dict, duplicate=Tr
return {'model_id': result}
@router.put('/seg/load/')
-def load_px_model(params: ilm.IlastikPixelClassifierParams, duplicate: bool = True) -> dict:
+def load_px_model(params: ilm.IlastikPixelClassifierParams) -> dict:
return load_ilastik_model(
ilm.IlastikPixelClassifierModel,
params,
- duplicate=duplicate
)
@router.put('/pxmap_to_obj/load/')
-def load_pxmap_to_obj_model(params: ilm.IlastikParams, duplicate: bool = True) -> dict:
+def load_pxmap_to_obj_model(params: ilm.IlastikParams) -> dict:
return load_ilastik_model(
ilm.IlastikObjectClassifierFromPixelPredictionsModel,
params,
- duplicate=duplicate
)
@router.put('/seg_to_obj/load/')
-def load_seg_to_obj_model(params: ilm.IlastikParams, duplicate: bool = True) -> dict:
+def load_seg_to_obj_model(params: ilm.IlastikParams) -> dict:
return load_ilastik_model(
ilm.IlastikObjectClassifierFromSegmentationModel,
params,
- duplicate=duplicate
)
@router.put('/pixel_then_object_classification/infer')
diff --git a/model_server/extensions/ilastik/tests/test_ilastik.py b/model_server/extensions/ilastik/tests/test_ilastik.py
index a48867c0e689aa4eb26589aa4a2d2e040ae7f81a..1907459d2d5d3a87be5da49e291fe15e4e1b7ca0 100644
--- a/model_server/extensions/ilastik/tests/test_ilastik.py
+++ b/model_server/extensions/ilastik/tests/test_ilastik.py
@@ -226,15 +226,13 @@ class TestIlastikOverApi(TestServerBaseClass):
self.assertEqual(len(resp_list_1st), 1, resp_list_1st)
resp_load_2nd = self._put(
'ilastik/seg/load/',
- body={'project_file': str(ilastik_classifiers['px'])},
- query={'duplicate': True},
+ body={'project_file': str(ilastik_classifiers['px']), 'duplicate': True},
)
resp_list_2nd = self._get('models').json()
self.assertEqual(len(resp_list_2nd), 2, resp_list_2nd)
resp_load_3rd = self._put(
'ilastik/seg/load/',
- body={'project_file': str(ilastik_classifiers['px'])},
- query={'duplicate': False},
+ body={'project_file': str(ilastik_classifiers['px']), 'duplicate': False},
)
resp_list_3rd = self._get('models').json()
self.assertEqual(len(resp_list_3rd), 2, resp_list_3rd)
@@ -273,13 +271,11 @@ class TestIlastikOverApi(TestServerBaseClass):
# load models with these paths
resp1 = self._put(
'ilastik/seg/load/',
- body={'project_file': ilp_win},
- query={'duplicate': False},
+ body={'project_file': ilp_win, 'duplicate': False},
)
resp2 = self._put(
'ilastik/seg/load/',
- body={'project_file': ilp_posx},
- query={'duplicate': False},
+ body={'project_file': ilp_posx, 'duplicate': False},
)
self.assertEqual(resp1.json(), resp2.json())
diff --git a/model_server/scripts/correct_ilastik_abspath.py b/model_server/scripts/correct_ilastik_abspath.py
new file mode 100644
index 0000000000000000000000000000000000000000..f915bf3f54bcbe01cb097ee7ea9e7b2915eb18c5
--- /dev/null
+++ b/model_server/scripts/correct_ilastik_abspath.py
@@ -0,0 +1,48 @@
+from os.path import relpath
+from pathlib import Path
+import shutil
+
+import h5py
+
+def make_abspath_to_relpath(ilp_filename: str, my_root: Path, their_root: Path):
+ pa_ilp_old = my_root / ilp_filename
+ assert pa_ilp_old.exists()
+ pa_ilp_new = pa_ilp_old.parent / f'relpath_{pa_ilp_old.name}'
+ with h5py.File(shutil.copy(pa_ilp_old, pa_ilp_new), 'r+') as h5:
+ infos = h5['Input Data/infos']
+ for lane in infos.keys():
+ for role in infos[lane].keys():
+ if len(infos[lane][role]) == 0:
+ continue
+ pa_img_abs = Path(infos[lane][role]['filePath'][()].decode())
+ my_ilp_dir = (my_root / ilp_filename).parent
+ their_ilp_dir = (their_root / ilp_filename).parent
+ pa_img_rel = Path(relpath(pa_img_abs, their_ilp_dir))
+ if pa_img_rel.parts[-2].upper().endswith('.H5'):
+ assert (my_ilp_dir / Path(*pa_img_rel.parts[0:-1])).exists()
+ else:
+ assert (my_ilp_dir / pa_img_rel).exists()
+ del infos[lane][role]['filePath']
+ infos[lane][role]['filePath'] = str(pa_img_rel)
+ return pa_ilp_new
+
+
+
+
+if __name__ == '__main__':
+ files = [
+ '01_ilastik_files/240301_LSM900_DNA_PC.ilp',
+ '01_ilastik_files/240320_LSM900_DNA_OC_new.ilp',
+ '01_ilastik_files/240301_LSM900_TM_PC.ilp',
+ '01_ilastik_files/240320_LSM900_TM_OC_new.ilp'
+ ]
+ for f in files:
+ new_ilp = make_abspath_to_relpath(
+ f,
+ Path('w:/03_analysis/Trial3_LSM900'),
+ Path('/g/cuylen/01_Share/Filemaker/01_Experiments/Experiments_1100/1156/03_analysis/Trial3_LSM900')
+ )
+ print(f'Finished converting {new_ilp}')
+
+
+
diff --git a/model_server/scripts/verify_multichannel_ilastik_inputs.py b/model_server/scripts/verify_multichannel_ilastik_inputs.py
new file mode 100644
index 0000000000000000000000000000000000000000..1159b3f1ae42d9902474a854c7f85f1134748fe5
--- /dev/null
+++ b/model_server/scripts/verify_multichannel_ilastik_inputs.py
@@ -0,0 +1,111 @@
+from pathlib import Path
+
+import h5py
+import numpy as np
+import pandas as pd
+
+from model_server.base.accessors import generate_file_accessor, write_accessor_data_to_file, InMemoryDataAccessor
+from model_server.extensions.ilastik.models import IlastikPixelClassifierModel, IlastikObjectClassifierFromPixelPredictionsModel
+
+def get_input_files(where_ilp: Path) -> list:
+ files = []
+ with h5py.File(where_ilp, 'r') as h5:
+ infos = h5['Input Data/infos']
+ for lane in infos.keys():
+ lane_dict = {}
+ for role in infos[lane].keys():
+ if len(infos[lane][role]) == 0:
+ continue
+ rel_path = Path(infos[lane][role]['filePath'][()].decode())
+ lane_dict[role] = where_ilp.parent / rel_path
+ files.append(lane_dict)
+ return files
+
+if __name__ == '__main__':
+ where_out = Path('c:/Users/rhodes/projects/proj0015-model-server/issues/0032_multiple_input_channels/output')
+ root = Path('w:/03_analysis/Trial3_LSM900')
+ max_files = 1
+ ilps = [
+ '01_ilastik_files/relpath_240301_LSM900_DNA_PC.ilp',
+ '01_ilastik_files/relpath_240320_LSM900_DNA_OC_new.ilp',
+ '01_ilastik_files/relpath_240301_LSM900_TM_PC.ilp',
+ '01_ilastik_files/relpath_240320_LSM900_TM_OC_new.ilp'
+ ]
+ records = []
+ for f in ilps:
+ ilp = root / f
+ assert ilp.exists()
+ outdir = where_out / ilp.stem
+ outdir.mkdir(parents=True, exist_ok=True)
+
+ if ilp.stem.upper().endswith('_PC'):
+ mod = IlastikPixelClassifierModel(
+ params={'project_file': str(ilp)},
+ enforce_embedded=False
+ )
+ infiles = get_input_files(ilp)
+ for ln in infiles[0:max_files]:
+ acc_raw = generate_file_accessor(root / ln['Raw Data'])
+ pxmap = mod.infer(acc_raw)[0]
+ pxmap_fn = 'pxmap_' + ln['Raw Data'].stem + '.tif'
+ write_accessor_data_to_file(outdir / pxmap_fn, pxmap)
+ record = {
+ 'classifier': str(ilp.relative_to(root)),
+ 'input_raw_data': str(ln['Raw Data'].relative_to(root)),
+ 'input_raw_data_chroma': acc_raw.chroma,
+ 'input_raw_data_dtype': acc_raw.dtype,
+ 'input_raw_data_shape_dict': acc_raw.shape_dict,
+ 'output_file': pxmap_fn,
+ 'output_dtype': pxmap.dtype,
+ 'output_chroma': pxmap.chroma,
+ 'output_shape_dict': pxmap.shape_dict,
+ }
+ records.append(record)
+
+ elif ilp.stem.upper().endswith('_OC_NEW'):
+ mod = IlastikObjectClassifierFromPixelPredictionsModel(
+ params={'project_file': str(ilp)},
+ enforce_embedded=False
+ )
+ infiles = get_input_files(ilp)
+ for ln in infiles[0:max_files]:
+ acc_raw = generate_file_accessor(root / ln['Raw Data'])
+ pa_pxmap = root / ln['Prediction Maps']
+
+ if pa_pxmap.parts[-2].upper().endswith('.H5'):
+ pa_h5f = root / Path(*pa_pxmap.parts[0:-1])
+ h5_key = pa_pxmap.parts[-1]
+ pxmap_data = h5py.File(pa_h5f)[h5_key][()] # C x Y x X ?
+ pxmap_yxc = np.moveaxis(
+ pxmap_data,
+ [1, 2, 0],
+ [0, 1, 2]
+ )
+ acc_pxmap = InMemoryDataAccessor(np.expand_dims(pxmap_yxc, -1))
+ else:
+ acc_pxmap = generate_file_accessor(pa_pxmap)
+ obmap = mod.infer(acc_raw, acc_pxmap)[0]
+ obmap_fn = 'obmap_' + ln['Raw Data'].stem + '.tif'
+ write_accessor_data_to_file(outdir / obmap_fn, obmap)
+ record = {
+ 'classifier': str(ilp.relative_to(root)),
+ 'input_raw_data': str(ln['Raw Data'].relative_to(root)),
+ 'input_raw_data_chroma': acc_raw.chroma,
+ 'input_raw_data_dtype': acc_raw.dtype,
+ 'input_raw_data_shape_dict': acc_raw.shape_dict,
+ 'input_pxmap': str(ln['Prediction Maps'].relative_to(root)),
+ 'input_pxmap_chroma': acc_pxmap.chroma,
+ 'input_pxmap_dtype': acc_pxmap.dtype,
+ 'input_pxmap_shape_dict': acc_pxmap.shape_dict,
+ 'output_file': obmap_fn,
+ 'output_dtype': obmap.dtype,
+ 'output_chroma': obmap.chroma,
+ 'output_shape_dict': obmap.shape_dict,
+ }
+ records.append(record)
+
+ else:
+ raise Exception(f'unidentified project file {ilp}')
+
+ pd.DataFrame(records).to_csv(where_out / 'record.csv', index=False)
+ print('Finished')
diff --git a/tests/test_roiset.py b/tests/test_roiset.py
index c1d773c6d5ac975ecf0102e5bf349bb3e385e98b..d45c6132ea6dd52447f3f2614d0d9f35aef2a355 100644
--- a/tests/test_roiset.py
+++ b/tests/test_roiset.py
@@ -455,7 +455,6 @@ class TestRoiSetMultichannelProducts(BaseTestRoiSetMonoProducts, unittest.TestCa
self.assertTrue(pa.exists())
pacc = generate_file_accessor(pa)
self.assertEqual(pacc.hw, (256, 256))
- print('res')
def test_run_export_mono_2d_patch(self):
p = RoiSetExportParams(**{