Skip to content
Snippets Groups Projects
Commit 3f116376 authored by Christopher Randolph Rhodes's avatar Christopher Randolph Rhodes
Browse files

Merge branch 'use_relative_patch_paths' into 'staging' 

Use relative patch paths

See merge request !22
parents a44d0f11 2deca4a1
No related branches found
No related tags found
3 merge requests!37Release 2024.04.19,!34Revert "Temporary error-handling for debug...",!22Use relative patch paths
Hide whitespace changes
Inline Side-by-side
model_server/base/roiset.py
+ 17
− 14
View file @ 3f116376
......@@ -264,11 +264,11 @@ class RoiSet(object):
patches_df = self.get_patches(**kwargs)
return PatchStack(list(patches_df.patch))
def export_annotated_zstack(self, where, prefix='zstack', **kwargs) -> str(Path):
def export_annotated_zstack(self, where, prefix='zstack', **kwargs) -> str:
annotated = InMemoryDataAccessor(draw_boxes_on_3d_image(self, **kwargs))
fp = where / (prefix + '.tif')
write_accessor_data_to_file(fp, annotated)
return str(fp)
return (prefix + '.tif')
def get_zmask(self, mask_type='boxes'):
"""
......@@ -330,13 +330,14 @@ class RoiSet(object):
om[self.acc_obj_ids.data == roi.label] = oc
self.object_class_maps[name] = InMemoryDataAccessor(om)
def export_dataframe(self, csv_path: Path):
def export_dataframe(self, csv_path: Path) -> str:
csv_path.parent.mkdir(parents=True, exist_ok=True)
self._df.drop(['expanded_slice', 'slice', 'relative_slice', 'binary_mask'], axis=1).to_csv(csv_path, index=False)
return csv_path
return csv_path.name
def export_patch_masks(self, where: Path, pad_to: int = None, prefix='mask', expanded=False) -> list:
def export_patch_masks(self, where: Path, pad_to: int = None, prefix='mask', expanded=False) -> pd.DataFrame:
patches_df = self.get_patch_masks(pad_to=pad_to, expanded=expanded).copy()
def _export_patch_mask(roi):
......@@ -344,7 +345,7 @@ class RoiSet(object):
ext = 'png'
fname = f'{prefix}-la{roi.label:04d}-zi{roi.zi:04d}.{ext}'
write_accessor_data_to_file(where / fname, patch)
return str(where / fname)
return fname
patches_df['patch_mask_path'] = patches_df.apply(_export_patch_mask, axis=1)
return patches_df
......@@ -364,7 +365,7 @@ class RoiSet(object):
write_accessor_data_to_file(where / fname, resampled)
else:
write_accessor_data_to_file(where / fname, patch)
return str(where / fname)
return fname
patches_df['patch_path'] = patches_df.apply(_export_patch, axis=1)
return patches_df
......@@ -549,22 +550,22 @@ class RoiSet(object):
df_exp = self.export_patches(
subdir, white_channel=channel, prefix=pr, make_3d=True, expanded=True, **kp
)
record[k] = list(df_exp.patch_path)
record[k] = [str(Path(k) / fn) for fn in df_exp.patch_path]
if k == 'annotated_patches_2d':
df_exp = self.export_patches(
subdir, prefix=pr, make_3d=False, white_channel=channel,
bounding_box_channel=1, bounding_box_linewidth=2, **kp,
)
record[k] = list(df_exp.patch_path)
record[k] = [str(Path(k) / fn) for fn in df_exp.patch_path]
if k == 'patches_2d':
df_exp = self.export_patches(
subdir, white_channel=channel, prefix=pr, make_3d=False, **kp
)
self._df = self._df.join(df_exp.patch_path)
self._df['patch_id'] = self._df.apply(lambda _: uuid4(), axis=1)
record[k] = list(df_exp.patch_path)
record[k] = [str(Path(k) / fn) for fn in df_exp.patch_path]
if k == 'annotated_zstacks':
record[k] = self.export_annotated_zstack(subdir, prefix=pr, **kp)
record[k] = str(Path(k) / self.export_annotated_zstack(subdir, prefix=pr, **kp))
if k == 'object_classes':
for kc, acc in self.object_class_maps.items():
fp = subdir / kc / (pr + '.tif')
......@@ -587,9 +588,11 @@ class RoiSet(object):
pad_to=None,
expanded=False
)
self._df = self._df.join(df_exp.patch_mask_path)
record['dataframe'] = str(self.export_dataframe(where / 'dataframe' / (prefix + '.csv')))
record['tight_patch_masks'] = list(df_exp.patch_mask_path)
se_pa = df_exp.patch_mask_path.apply(lambda x: str(Path('tight_patch_masks') / x)).rename('tight_patch_masks')
self._df = self._df.join(se_pa)
df_fn = self.export_dataframe(where / 'dataframe' / (prefix + '.csv'))
record['dataframe'] = str(Path('dataframe') / df_fn)
record['tight_patch_masks'] = list(se_pa)
return record
@staticmethod
......
tests/test_roiset.py
+ 51
− 27
View file @ 3f116376
......@@ -102,15 +102,16 @@ class TestRoiSetMonoProducts(BaseTestRoiSetMonoProducts, unittest.TestCase):
def test_make_expanded_2d_patches(self):
roiset = self._make_roi_set()
where = output_path / 'expanded_2d_patches'
df_res = roiset.export_patches(
output_path / 'expanded_2d_patches',
where,
draw_bounding_box=True,
expanded=True,
pad_to=256,
)
df = roiset.get_df()
for f in df_res.patch_path:
acc = generate_file_accessor(f)
acc = generate_file_accessor(where / f)
la = int(re.search(r'la([\d]+)', str(f)).group(1))
roi_q = df.loc[df.label == la, :]
self.assertEqual(len(roi_q), 1)
......@@ -118,14 +119,15 @@ class TestRoiSetMonoProducts(BaseTestRoiSetMonoProducts, unittest.TestCase):
def test_make_tight_2d_patches(self):
roiset = self._make_roi_set()
where = output_path / 'tight_2d_patches'
df_res = roiset.export_patches(
output_path / 'tight_2d_patches',
where,
draw_bounding_box=True,
expanded=False
)
df = roiset.get_df()
for f in df_res.patch_path: # all exported files are same shape as bounding boxes in RoiSet's datatable
acc = generate_file_accessor(f)
acc = generate_file_accessor(where / f)
la = int(re.search(r'la([\d]+)', str(f)).group(1))
roi_q = df.loc[df.label == la, :]
self.assertEqual(len(roi_q), 1)
......@@ -134,23 +136,25 @@ class TestRoiSetMonoProducts(BaseTestRoiSetMonoProducts, unittest.TestCase):
def test_make_expanded_3d_patches(self):
roiset = self._make_roi_set()
where = output_path / '3d_patches'
df_res = roiset.export_patches(
output_path / '3d_patches',
where,
make_3d=True,
expanded=True
)
self.assertGreaterEqual(len(df_res), 1)
for f in df_res.patch_path:
acc = generate_file_accessor(f)
acc = generate_file_accessor(where / f)
self.assertGreater(acc.nz, 1)
def test_export_annotated_zstack(self):
roiset = self._make_roi_set()
where = output_path / 'annotated_zstack'
file = roiset.export_annotated_zstack(
output_path / 'annotated_zstack',
where,
)
result = generate_file_accessor(file)
result = generate_file_accessor(where / file)
self.assertEqual(result.shape, roiset.acc_raw.shape)
def test_flatten_image(self):
......@@ -244,31 +248,34 @@ class TestRoiSetMultichannelProducts(BaseTestRoiSetMonoProducts, unittest.TestCa
)
def test_multichannel_to_mono_2d_patches(self):
where = output_path / 'multichannel' / 'mono_2d_patches'
df_res = self.roiset.export_patches(
output_path / 'multichannel' / 'mono_2d_patches',
where,
white_channel=3,
draw_bounding_box=True,
expanded=True,
pad_to=256,
)
result = generate_file_accessor(df_res.patch_path.iloc[0])
result = generate_file_accessor(where / df_res.patch_path.iloc[0])
self.assertEqual(result.chroma, 1)
def test_multichannnel_to_mono_2d_patches_rgb_bbox(self):
where = output_path / 'multichannel' / 'mono_2d_patches_rgb_bbox'
df_res = self.roiset.export_patches(
output_path / 'multichannel' / 'mono_2d_patches_rgb_bbox',
where,
white_channel=3,
draw_bounding_box=True,
bounding_box_channel=1,
expanded=True,
pad_to=256,
)
result = generate_file_accessor(df_res.patch_path.iloc[0])
result = generate_file_accessor(where / df_res.patch_path.iloc[0])
self.assertEqual(result.chroma, 3)
def test_multichannnel_to_rgb_2d_patches_bbox(self):
where = output_path / 'multichannel' / 'rgb_2d_patches_bbox'
df_res = self.roiset.export_patches(
output_path / 'multichannel' / 'rgb_2d_patches_bbox',
where,
white_channel=4,
rgb_overlay_channels=(3, None, None),
draw_mask=False,
......@@ -278,12 +285,13 @@ class TestRoiSetMultichannelProducts(BaseTestRoiSetMonoProducts, unittest.TestCa
expanded=True,
pad_to=256,
)
result = generate_file_accessor(df_res.patch_path.iloc[0])
result = generate_file_accessor(where / df_res.patch_path.iloc[0])
self.assertEqual(result.chroma, 3)
def test_multichannnel_to_rgb_2d_patches_mask(self):
where = output_path / 'multichannel' / 'rgb_2d_patches_mask'
df_res = self.roiset.export_patches(
output_path / 'multichannel' / 'rgb_2d_patches_mask',
where,
white_channel=4,
rgb_overlay_channels=(3, None, None),
draw_mask=True,
......@@ -292,12 +300,13 @@ class TestRoiSetMultichannelProducts(BaseTestRoiSetMonoProducts, unittest.TestCa
expanded=True,
pad_to=256,
)
result = generate_file_accessor(df_res.patch_path.iloc[0])
result = generate_file_accessor(where / df_res.patch_path.iloc[0])
self.assertEqual(result.chroma, 3)
def test_multichannnel_to_rgb_2d_patches_contour(self):
where = output_path / 'multichannel' / 'rgb_2d_patches_contour'
df_res = self.roiset.export_patches(
output_path / 'multichannel' / 'rgb_2d_patches_contour',
where,
rgb_overlay_channels=(3, None, None),
draw_contour=True,
contour_channel=1,
......@@ -305,39 +314,42 @@ class TestRoiSetMultichannelProducts(BaseTestRoiSetMonoProducts, unittest.TestCa
expanded=True,
pad_to=256,
)
result = generate_file_accessor(df_res.patch_path.iloc[0])
result = generate_file_accessor(where / df_res.patch_path.iloc[0])
self.assertEqual(result.chroma, 3)
self.assertEqual(result.get_one_channel_data(2).data.max(), 0) # blue channel is black
def test_multichannel_to_multichannel_tif_patches(self):
where = output_path / 'multichannel' / 'multichannel_tif_patches'
df_res = self.roiset.export_patches(
output_path / 'multichannel' / 'multichannel_tif_patches',
where,
expanded=True,
pad_to=256,
)
result = generate_file_accessor(df_res.patch_path.iloc[0])
result = generate_file_accessor(where / df_res.patch_path.iloc[0])
self.assertEqual(result.chroma, 5)
self.assertEqual(result.nz, 1)
def test_multichannel_annotated_zstack(self):
where = output_path / 'multichannel' / 'annotated_zstack'
file = self.roiset.export_annotated_zstack(
output_path / 'multichannel' / 'annotated_zstack',
where,
'test_multichannel_annotated_zstack',
expanded=True,
pad_to=256,
)
result = generate_file_accessor(file)
result = generate_file_accessor(where / file)
self.assertEqual(result.chroma, self.stack.chroma)
self.assertEqual(result.nz, self.stack.nz)
def test_export_single_channel_annotated_zstack(self):
where = output_path / 'annotated_zstack'
file = self.roiset.export_annotated_zstack(
output_path / 'annotated_zstack',
where,
channel=3,
expanded=True,
pad_to=256,
)
result = generate_file_accessor(file)
result = generate_file_accessor(where / file)
self.assertEqual(result.hw, self.roiset.acc_raw.hw)
self.assertEqual(result.nz, self.roiset.acc_raw.nz)
self.assertEqual(result.chroma, 1)
......@@ -363,19 +375,31 @@ class TestRoiSetMultichannelProducts(BaseTestRoiSetMonoProducts, unittest.TestCa
'dataframe': True,
})
where = output_path / 'run_exports'
res = self.roiset.run_exports(
output_path / 'run_exports',
where,
channel=3,
prefix='test',
params=p
)
# test on return paths
for k, v in res.items():
if isinstance(v, list):
for f in v:
self.assertTrue(Path(f).exists())
self.assertFalse(Path(f).is_absolute())
self.assertTrue((where / f).exists())
else:
self.assertTrue(Path(v).exists())
self.assertFalse(Path(v).is_absolute())
self.assertTrue((where / v).exists())
# test on paths in CSV
test_df = pd.read_csv(where / res['dataframe'])
for c in test_df.columns:
if '_path' in c:
for f in test_df[c]:
self.assertTrue((where / f).exists(), where / f)
class TestRoiSetFromZmask(unittest.TestCase):
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment