Implement segmentation export

make_initial_version.py

@@ -3,18 +3,17 @@
 import os
 
 from scripts.files import make_folder_structure
-from script.export import export_segmentation
+from scripts.export import export_segmentation
 
 
 def make_segmentations(old_folder, folder):
     path = '/g/kreshuk/data/arendt/platyneris_v1/data.n5'
 
     # export nucleus segemntation
-    key_nuclei = 'volumes/paintera/nuclei'
-    nuclei_name = 'em-segmented-nuclei-labels'
-    res_nuclei = [.1, .08, .08]
-    export_segmentation(path, key_nuclei, old_folder, folder, nuclei_name, res_nuclei)
-    return
+    # key_nuclei = 'volumes/paintera/nuclei'
+    # nuclei_name = 'em-segmented-nuclei-labels'
+    # res_nuclei = [.1, .08, .08]
+    # export_segmentation(path, key_nuclei, old_folder, folder, nuclei_name, res_nuclei)
 
     # export cell segemntation
     key_cells = 'volumes/paintera/proofread_cells'
@@ -29,7 +28,7 @@ def make_initial_version():
     tag = '0.0.0'
     folder = os.path.join('data', tag)
 
-    make_folder_structure(folder)
+    # make_folder_structure(folder)
 
     make_segmentations(old_folder, folder)
 

scripts/export/export_segmentation.py

@@ -2,6 +2,7 @@ import os
 import json
 import luigi
 import z5py
+from shutil import rmtree
 
 from cluster_tools.downscaling import DownscalingWorkflow
 from paintera_tools import serialize_from_commit
@@ -46,6 +47,7 @@ def downscale(path, in_key, out_key,
     t = task(tmp_folder=tmp_folder, config_dir=config_folder,
              target=target, max_jobs=max_jobs,
              input_path=path, input_key=in_key,
+             output_key_prefix=out_key,
              scale_factors=scales, halos=halos)
     ret = luigi.build([t], local_scheduler=True)
     if not ret:
@@ -82,12 +84,11 @@ def export_segmentation(paintera_path, paintera_key, folder, new_folder, name, r
     downscale(tmp_path, tmp_key0, tmp_key, n_scales, tmp_folder, max_jobs, target)
 
     # convert to bdv
-    out_path = os.path.join(new_folder, 'data', '%s.h5' % name)
-    to_bdv(tmp_path, tmp_key, out_path, resolution)
+    out_path = os.path.join(new_folder, 'segmentations', '%s.h5' % name)
+    to_bdv(tmp_path, tmp_key, out_path, resolution, tmp_folder, target)
 
     # compute mapping to old segmentation
     map_segmentation_ids(folder, new_folder, name, tmp_folder, max_jobs, target)
 
-    # TODO
     # clean up tmp
     # rmtree(tmp_folder)

scripts/export/map_segmentation_ids.py

@@ -4,12 +4,13 @@ import luigi
 import z5py
 
 from cluster_tools.node_labels import NodeLabelWorkflow
+from ..files import get_h5_path_from_xml
 
 
 def get_seg_path(folder, name):
     # check if we have a data sub folder, if we have it load
     # the segmentation from there
-    data_folder = os.path.join(folder, 'data')
+    data_folder = os.path.join(folder, 'segmentations')
     data_folder = data_folder if os.path.exists(data_folder) else folder
 
     # check if we have a h5
@@ -21,12 +22,12 @@ def get_seg_path(folder, name):
     path = os.path.join(data_folder, '%s.xml' % name)
     # read h5 path from the xml
     if os.path.exists(path):
-        # TODO
-        raise NotImplementedError("File path from xml not implemented")
-        # path = get_h5_path_from_xml(path)
-        # return path
+        path = get_h5_path_from_xml(path)
+        if not os.path.exists(path):
+            raise RuntimeError("Invalid path in xml")
+        return path
     else:
-        raise RuntimeError("The specified folder does not contain a segmentation")
+        raise RuntimeError("The specified folder does not contain segmentation")
 
 
 def map_ids(path1, path2, out_path, tmp_folder, max_jobs, target, prefix):
@@ -43,6 +44,11 @@ def map_ids(path1, path2, out_path, tmp_folder, max_jobs, target, prefix):
     with open(os.path.join(config_folder, 'global.config'), 'w') as f:
         json.dump(global_conf, f)
 
+    conf = configs['merge_node_labels']
+    conf.update({'threads_per_job': 8, 'mem_limit': 16})
+    with open(os.path.join(config_folder, 'merge_node_labels.config'), 'w') as f:
+        json.dump(conf, f)
+
     key = 't00000/s00/0/cells'
     tmp_path = os.path.join(tmp_folder, 'data.n5')
     tmp_key = prefix
@@ -58,13 +64,14 @@ def map_ids(path1, path2, out_path, tmp_folder, max_jobs, target, prefix):
 
     ds = z5py.File(tmp_path)[tmp_key]
     lut = ds[:]
-    lut = dict(zip(lut[:, 0], lut[:, 1]))
+    assert lut.ndim == 1
+    lut = dict(zip(range(len(lut)), lut.tolist()))
 
     with open(out_path, 'w') as f:
         json.dump(lut, f)
 
 
-def map_segmentation_ids(src_folder, dest_folder, out_path, name, tmp_folder, max_jobs, target):
+def map_segmentation_ids(src_folder, dest_folder, name, tmp_folder, max_jobs, target):
     src_path = get_seg_path(src_folder, name)
     dest_path = get_seg_path(dest_folder, name)
 

scripts/export/to_bdv.py

@@ -2,8 +2,8 @@ import os
 import json
 import luigi
 
+import h5py
 import z5py
-from shutil import rmtree
 from cluster_tools.downscaling import PainteraToBdvWorkflow
 
 
@@ -16,11 +16,11 @@ def check_max_id(path, key):
         raise RuntimeError("Uint16 overflow")
     else:
         print("Max-id:", max_id, "fits int16")
+    return max_id
 
 
-def to_bdv(in_path, in_key, out_path, resolution, target='slurm'):
-    check_max_id(in_path, in_key)
-    tmp_folder = 'tmp_export_bdv'
+def to_bdv(in_path, in_key, out_path, resolution, tmp_folder, target='slurm'):
+    max_id = check_max_id(in_path, in_key)
 
     config_folder = os.path.join(tmp_folder, 'configs')
     os.makedirs(config_folder, exist_ok=True)
@@ -48,4 +48,8 @@ def to_bdv(in_path, in_key, out_path, resolution, target='slurm'):
     ret = luigi.build([task], local_scheduler=True)
     if not ret:
         raise RuntimeError("Segmentation export failed")
-    rmtree(tmp_folder)
+
+    # write the max-id
+    with h5py.File(out_path) as f:
+        ds = f['t00000/s00/0/cells']
+        ds.attrs['maxId'] = max_id

scripts/files/__init__.py

 from .copy_helper import copy_tables, copy_segmentation, copy_static_files
 from .folders import make_folder_structure
-
+from .xml_utils import get_h5_path_from_xml

scripts/files/xml_utils.py

+import xml.etree.ElementTree as ET
+
+
+def get_h5_path_from_xml(xml_path):
+    # xml horror ...
+    et_root = ET.parse(xml_path).getroot()
+    et = et_root[1]
+    et = et[0]
+    et = et[0]
+    path = et.text
+    return path

update_platy_browser.py

@@ -84,7 +84,8 @@ def make_attributes(folder, new_folder,
 
 
 # TODO check for errors
-def make_release(tag, description=''):
+def make_release(tag, folder, description=''):
+    call(['git', 'add', folder])
     call(['git', 'commit', '-m', 'Automatic platybrowser update'])
     if description == '':
         call(['git', 'tag', tag])
@@ -144,7 +145,7 @@ def update_platy_browser(update_cell_segmentation=False,
     copy_static_files(folder, new_folder)
 
     # make new release
-    make_release(new_tag, description)
+    make_release(new_tag, new_folder, description)
 
 
 def str2bool(v):