From 66bdc9bbbff432feb67c507cfc37633b84be5873 Mon Sep 17 00:00:00 2001
From: Constantin Pape <constantin.pape@iwr.uni-heidelberg.de>
Date: Fri, 20 Sep 2019 14:56:15 +0200
Subject: [PATCH] Use proper scale factors in export_segmentation
---
.gitignore | 1 +
scripts/export/export_segmentation.py | 36 +++++++++++++++++----------
scripts/export/to_bdv.py | 2 +-
3 files changed, 25 insertions(+), 14 deletions(-)
diff --git a/.gitignore b/.gitignore
index a42e734..7a15357 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ tmp*
*.DS_Store
data/dev-*
*.tif
+*.swp
diff --git a/scripts/export/export_segmentation.py b/scripts/export/export_segmentation.py
index d7756df..e9a1d8f 100644
--- a/scripts/export/export_segmentation.py
+++ b/scripts/export/export_segmentation.py
@@ -5,23 +5,30 @@ import z5py
from cluster_tools.downscaling import DownscalingWorkflow
from paintera_tools import serialize_from_commit, postprocess
+from paintera_tools import set_default_shebang as set_ptools_shebang
from .to_bdv import to_bdv
from .map_segmentation_ids import map_segmentation_ids
-from ..default_config import write_default_global_config
+from ..default_config import write_default_global_config, get_default_shebang
from ..files import get_postprocess_dict
-def get_n_scales(paintera_path, paintera_key):
+def get_scale_factors(paintera_path, paintera_key):
f = z5py.File(paintera_path)
g = f[paintera_key]['data']
- keys = list(g.keys())
- scales = [key for key in keys
- if os.path.isdir(os.path.join(g.path, key)) and key.startswith('s')]
- return len(scales)
+ keys = [int(k[1:]) for k in g.keys()]
+ keys = sorted(keys)
+ scale_factors = [[1, 1, 1]]
+ rel_scales = [[1, 1, 1]]
+ for k in keys[1:]:
+ factor = g['s%i' % k].attrs['downsamplingFactors']
+ rel_factor = [int(sf // prev) for sf, prev in zip(factor, scale_factors[-1])]
+ scale_factors.append(factor)
+ rel_scales.append(rel_factor[::-1])
+ return rel_scales
def downscale(path, in_key, out_key,
- n_scales, tmp_folder, max_jobs, target):
+ scale_factors, tmp_folder, max_jobs, target):
task = DownscalingWorkflow
config_folder = os.path.join(tmp_folder, 'configs')
@@ -34,15 +41,14 @@ def downscale(path, in_key, out_key,
with open(os.path.join(config_folder, 'downscaling.config'), 'w') as f:
json.dump(config, f)
- # for now we hard-code scale factors to 2, but it would be cleaner to infer this from the data
- scales = [[2, 2, 2]] * n_scales
+ n_scales = len(scale_factors)
halos = [[0, 0, 0]] * n_scales
t = task(tmp_folder=tmp_folder, config_dir=config_folder,
target=target, max_jobs=max_jobs,
input_path=path, input_key=in_key,
output_key_prefix=out_key,
- scale_factors=scales, halos=halos)
+ scale_factors=scale_factors, halos=halos)
ret = luigi.build([t], local_scheduler=True)
if not ret:
raise RuntimeError("Downscaling the segmentation failed")
@@ -85,13 +91,17 @@ def export_segmentation(paintera_path, paintera_key, folder, new_folder, name, r
n_threads=16, size_threshold=min_segment_size,
label=label_segmentation)
+ # set correct shebang for paintera tools
+ set_ptools_shebang(get_default_shebang())
+
+ # NOTE map_to_background is needed for cilia, nee some way to enable it automatically
# export segmentation from paintera commit for all scales
serialize_from_commit(paintera_path, paintera_key, tmp_path, tmp_key0, tmp_folder,
- max_jobs, target, relabel_output=True)
+ max_jobs, target, relabel_output=True) # , map_to_background=[1])
# downscale the segemntation
- n_scales = get_n_scales(paintera_path, paintera_key)
- downscale(tmp_path, tmp_key0, tmp_key, n_scales, tmp_folder, max_jobs, target)
+ scale_factors = get_scale_factors(paintera_path, paintera_key)
+ downscale(tmp_path, tmp_key0, tmp_key, scale_factors, tmp_folder, max_jobs, target)
# convert to bdv
out_path = os.path.join(new_folder, 'segmentations', '%s.h5' % name)
diff --git a/scripts/export/to_bdv.py b/scripts/export/to_bdv.py
index e31b659..479b1c3 100644
--- a/scripts/export/to_bdv.py
+++ b/scripts/export/to_bdv.py
@@ -28,7 +28,7 @@ def to_bdv(in_path, in_key, out_path, resolution, tmp_folder, target='slurm'):
configs = PainteraToBdvWorkflow.get_config()
config = configs['copy_volume']
- config.update({'threads_per_job': 8, 'mem_limit': 32, 'time_limit': 1600,
+ config.update({'threads_per_job': 32, 'mem_limit': 64, 'time_limit': 2400,
'chunks': [32, 256, 256]})
with open(os.path.join(config_folder, 'copy_volume.config'), 'w') as f:
json.dump(config, f)
--
GitLab