Skip to content
Snippets Groups Projects
Commit 81737ef2 authored by Constantin Pape's avatar Constantin Pape
Browse files

Implement segmentation export

parent c6c759e8
No related branches found
No related tags found
No related merge requests found
......@@ -3,18 +3,17 @@
import os
from scripts.files import make_folder_structure
from script.export import export_segmentation
from scripts.export import export_segmentation
def make_segmentations(old_folder, folder):
path = '/g/kreshuk/data/arendt/platyneris_v1/data.n5'
# export nucleus segemntation
key_nuclei = 'volumes/paintera/nuclei'
nuclei_name = 'em-segmented-nuclei-labels'
res_nuclei = [.1, .08, .08]
export_segmentation(path, key_nuclei, old_folder, folder, nuclei_name, res_nuclei)
return
# key_nuclei = 'volumes/paintera/nuclei'
# nuclei_name = 'em-segmented-nuclei-labels'
# res_nuclei = [.1, .08, .08]
# export_segmentation(path, key_nuclei, old_folder, folder, nuclei_name, res_nuclei)
# export cell segemntation
key_cells = 'volumes/paintera/proofread_cells'
......@@ -29,7 +28,7 @@ def make_initial_version():
tag = '0.0.0'
folder = os.path.join('data', tag)
make_folder_structure(folder)
# make_folder_structure(folder)
make_segmentations(old_folder, folder)
......
......@@ -2,6 +2,7 @@ import os
import json
import luigi
import z5py
from shutil import rmtree
from cluster_tools.downscaling import DownscalingWorkflow
from paintera_tools import serialize_from_commit
......@@ -46,6 +47,7 @@ def downscale(path, in_key, out_key,
t = task(tmp_folder=tmp_folder, config_dir=config_folder,
target=target, max_jobs=max_jobs,
input_path=path, input_key=in_key,
output_key_prefix=out_key,
scale_factors=scales, halos=halos)
ret = luigi.build([t], local_scheduler=True)
if not ret:
......@@ -82,12 +84,11 @@ def export_segmentation(paintera_path, paintera_key, folder, new_folder, name, r
downscale(tmp_path, tmp_key0, tmp_key, n_scales, tmp_folder, max_jobs, target)
# convert to bdv
out_path = os.path.join(new_folder, 'data', '%s.h5' % name)
to_bdv(tmp_path, tmp_key, out_path, resolution)
out_path = os.path.join(new_folder, 'segmentations', '%s.h5' % name)
to_bdv(tmp_path, tmp_key, out_path, resolution, tmp_folder, target)
# compute mapping to old segmentation
map_segmentation_ids(folder, new_folder, name, tmp_folder, max_jobs, target)
# TODO
# clean up tmp
# rmtree(tmp_folder)
......@@ -4,12 +4,13 @@ import luigi
import z5py
from cluster_tools.node_labels import NodeLabelWorkflow
from ..files import get_h5_path_from_xml
def get_seg_path(folder, name):
# check if we have a data sub folder, if we have it load
# the segmentation from there
data_folder = os.path.join(folder, 'data')
data_folder = os.path.join(folder, 'segmentations')
data_folder = data_folder if os.path.exists(data_folder) else folder
# check if we have a h5
......@@ -21,12 +22,12 @@ def get_seg_path(folder, name):
path = os.path.join(data_folder, '%s.xml' % name)
# read h5 path from the xml
if os.path.exists(path):
# TODO
raise NotImplementedError("File path from xml not implemented")
# path = get_h5_path_from_xml(path)
# return path
path = get_h5_path_from_xml(path)
if not os.path.exists(path):
raise RuntimeError("Invalid path in xml")
return path
else:
raise RuntimeError("The specified folder does not contain a segmentation")
raise RuntimeError("The specified folder does not contain segmentation")
def map_ids(path1, path2, out_path, tmp_folder, max_jobs, target, prefix):
......@@ -43,6 +44,11 @@ def map_ids(path1, path2, out_path, tmp_folder, max_jobs, target, prefix):
with open(os.path.join(config_folder, 'global.config'), 'w') as f:
json.dump(global_conf, f)
conf = configs['merge_node_labels']
conf.update({'threads_per_job': 8, 'mem_limit': 16})
with open(os.path.join(config_folder, 'merge_node_labels.config'), 'w') as f:
json.dump(conf, f)
key = 't00000/s00/0/cells'
tmp_path = os.path.join(tmp_folder, 'data.n5')
tmp_key = prefix
......@@ -58,13 +64,14 @@ def map_ids(path1, path2, out_path, tmp_folder, max_jobs, target, prefix):
ds = z5py.File(tmp_path)[tmp_key]
lut = ds[:]
lut = dict(zip(lut[:, 0], lut[:, 1]))
assert lut.ndim == 1
lut = dict(zip(range(len(lut)), lut.tolist()))
with open(out_path, 'w') as f:
json.dump(lut, f)
def map_segmentation_ids(src_folder, dest_folder, out_path, name, tmp_folder, max_jobs, target):
def map_segmentation_ids(src_folder, dest_folder, name, tmp_folder, max_jobs, target):
src_path = get_seg_path(src_folder, name)
dest_path = get_seg_path(dest_folder, name)
......
......@@ -2,8 +2,8 @@ import os
import json
import luigi
import h5py
import z5py
from shutil import rmtree
from cluster_tools.downscaling import PainteraToBdvWorkflow
......@@ -16,11 +16,11 @@ def check_max_id(path, key):
raise RuntimeError("Uint16 overflow")
else:
print("Max-id:", max_id, "fits int16")
return max_id
def to_bdv(in_path, in_key, out_path, resolution, target='slurm'):
check_max_id(in_path, in_key)
tmp_folder = 'tmp_export_bdv'
def to_bdv(in_path, in_key, out_path, resolution, tmp_folder, target='slurm'):
max_id = check_max_id(in_path, in_key)
config_folder = os.path.join(tmp_folder, 'configs')
os.makedirs(config_folder, exist_ok=True)
......@@ -48,4 +48,8 @@ def to_bdv(in_path, in_key, out_path, resolution, target='slurm'):
ret = luigi.build([task], local_scheduler=True)
if not ret:
raise RuntimeError("Segmentation export failed")
rmtree(tmp_folder)
# write the max-id
with h5py.File(out_path) as f:
ds = f['t00000/s00/0/cells']
ds.attrs['maxId'] = max_id
from .copy_helper import copy_tables, copy_segmentation, copy_static_files
from .folders import make_folder_structure
from .xml_utils import get_h5_path_from_xml
import xml.etree.ElementTree as ET
def get_h5_path_from_xml(xml_path):
# xml horror ...
et_root = ET.parse(xml_path).getroot()
et = et_root[1]
et = et[0]
et = et[0]
path = et.text
return path
......@@ -84,7 +84,8 @@ def make_attributes(folder, new_folder,
# TODO check for errors
def make_release(tag, description=''):
def make_release(tag, folder, description=''):
call(['git', 'add', folder])
call(['git', 'commit', '-m', 'Automatic platybrowser update'])
if description == '':
call(['git', 'tag', tag])
......@@ -144,7 +145,7 @@ def update_platy_browser(update_cell_segmentation=False,
copy_static_files(folder, new_folder)
# make new release
make_release(new_tag, description)
make_release(new_tag, new_folder, description)
def str2bool(v):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment