Remove deprecated scripts

deprecated/add_existing_data.py

-#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
-
-import os
-from glob import glob
-from scripts.files import add_source, add_image, add_segmentation
-
-
-def add_sources():
-    # add em source
-    add_source('sbem', '6dpf')
-    # add prospr source
-    add_source('prospr', '6dpf')
-
-
-def add_images():
-    base_folder = './data/0.2.1/images'
-
-    # add sbem raw data
-    sbem_prefix = 'sbem-6dpf-1-whole'
-    sbem_raw = './data/0.2.1/images/sbem-6dpf-1-whole-raw.xml'
-    name = 'raw'
-    add_image(sbem_prefix, name, sbem_raw, copy_data=False)
-
-    # add all prospr images
-    prospr_prefix = 'prospr-6dpf-1-whole'
-    prospr_ims = glob(os.path.join(base_folder, 'prospr-6dpf-1-whole-*'))
-    for impath in prospr_ims:
-        name = os.path.split(impath)[1]
-        name, ext = os.path.splitext(name)
-        if ext != '.xml':
-            continue
-        name = name[(len(prospr_prefix) + 1):]
-        add_image(prospr_prefix, name, impath, copy_data=False)
-
-
-def add_static_segmentations():
-    source = 'sbem-6dpf-1-whole'
-
-    # chromatin segmentation
-    chromatin_tables = {'default': './data/0.2.0/tables/sbem-6dpf-1-whole-segmented-chromatin-labels/default.csv'}
-    add_segmentation(source, 'segmented-chromatin-labels',
-                     segmentation_path='./data/rawdata/sbem-6dpf-1-whole-segmented-chromatin-labels.xml',
-                     table_path_dict=chromatin_tables, copy_data=False)
-
-    # tissue segmentation
-    tissue_tables = {'default': './data/0.1.0/tables/sbem-6dpf-1-whole-segmented-tissue-labels/base.csv'}
-    add_segmentation(source, 'segmented-tissue-labels',
-                     segmentation_path='./data/rawdata/sbem-6dpf-1-whole-segmented-tissue-labels.xml',
-                     table_path_dict=tissue_tables, copy_data=False)
-
-    # muscle segmentation
-    add_segmentation(source, 'segmented-muscle',
-                     segmentation_path='./data/rawdata/sbem-6dpf-1-whole-segmented-muscle.h5', copy_data=False)
-
-
-def add_dynamic_segmentations():
-    source = 'sbem-6dpf-1-whole'
-    paintera_root = '/g/kreshuk/data/arendt/platyneris_v1/data.n5'
-
-    # cell segmentation
-    add_segmentation(source, 'segmented-cells-labels',
-                     paintera_project=(paintera_root, 'volumes/paintera/proofread_cells'),
-                     resolution=[.025, .02, .02],
-                     table_update_function='make_cell_tables')
-
-    # nucleus segmentation
-    add_segmentation(source, 'segmented-nuclei-labels',
-                     paintera_project=(paintera_root, 'volumes/paintera/nuclei'),
-                     resolution=[.1, .08, .08],
-                     table_update_function='make_nucleus_tables')
-
-    # cilia segmentation
-    add_segmentation(source, 'segmented-cilia-labels',
-                     paintera_project=(paintera_root, 'volumes/paintera/cilia'),
-                     resolution=[.025, .01, .01],
-                     table_update_function='make_cilia_tables')
-
-
-def add_segmentations():
-    add_static_segmentations()
-    add_dynamic_segmentations()
-
-
-def add_existing_data():
-    """ Add existing data to the json files that keep track of
-        sources, image data and segmentations.
-    """
-    add_sources()
-    add_images()
-    add_segmentations()
-
-
-if __name__ == '__main__':
-    add_existing_data()

deprecated/join_tables.py

-import pandas as pd
-
-def_path = '../data/0.5.1/tables/sbem-6dpf-1-whole-segmented-cilia-labels/default.csv.bkp'
-t1 = pd.read_csv(def_path, sep='\t')
-t2 = pd.read_csv('../data/0.5.1/tables/sbem-6dpf-1-whole-segmented-cilia-labels/cilia.csv', sep='\t')
-
-l1 = t1[['label_id']].values
-l2 = t2[['label_id']].values
-assert (l1 == l2).all()
-
-t = pd.concat([t1, t2['cell_id']], axis=1)
-def_path = '../data/0.5.1/tables/sbem-6dpf-1-whole-segmented-cilia-labels/default.csv'
-t.to_csv(def_path, index=False, sep='\t')

deprecated/legacy_export.py

-import os
-# from pathlib import Path
-from shutil import copyfile
-
-
-# TODO need to replace the name in xml to make this work out of the box
-def export_segmentation(folder, name, dest_folder, out_name):
-    seg_file = os.path.join(folder, 'segmentations', '%s.h5' % name)
-    xml_file = os.path.join(folder, 'segmentations', '%s.xml' % name)
-    table_file = os.path.join(folder, 'tables', name, 'default.csv')
-    assert os.path.exists(seg_file)
-    assert os.path.exists(xml_file)
-    assert os.path.exists(table_file), table_file
-
-    seg_out = os.path.join(dest_folder, '%s.h5' % out_name)
-    print("Copying segmentation from", seg_file, "to", seg_out)
-    copyfile(seg_file, seg_out)
-
-    xml_out = os.path.join(dest_folder, '%s.xml' % out_name)
-    print("Copying xml from", xml_file, "to", xml_out)
-    copyfile(xml_file, xml_out)
-
-    table_out = os.path.join(dest_folder, 'tables', '%s.csv' % out_name)
-    print("Copying table from", table_file, "to", table_out)
-    copyfile(table_file, table_out)
-
-
-def export_tag(tag, export_cells=False, export_cilia=False):
-    tag_folder = './data/%s' % tag
-    assert os.path.exists(tag_folder)
-
-    dest_folder = '/g/arendt/EM_6dpf_segmentation/EM-Prospr'
-
-    if export_cells:
-        name = 'sbem-6dpf-1-whole-segmented-cells-labels'
-        out_name = 'em-segmented-cells-new-labels'
-        export_segmentation(tag_folder, name, dest_folder, out_name)
-
-    if export_cilia:
-        name = 'sbem-6dpf-1-whole-segmented-cilia-labels'
-        out_name = 'em-segmented-cilia-labels'
-        export_segmentation(tag_folder, name, dest_folder, out_name)
-
-
-if __name__ == '__main__':
-    # export_tag('0.1.1', export_cells=True)
-    export_tag('0.1.0', export_cilia=True)

deprecated/make_data_sources.py

-#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
-
-import os
-import glob
-from shutil import copyfile
-from scripts.files import get_h5_path_from_xml, copy_xml_with_newpath, write_simple_xml
-
-
-def copy_xmls_and_symlink_h5(name_dict, src_folder, trgt_folder):
-    for n1, n2 in name_dict.items():
-        src_xml = os.path.join(src_folder, n1)
-        trgt_xml = os.path.join(trgt_folder, n2)
-
-        # we make a softlink from src to target h5
-        # NOTE eventually, we want to copy all the data, but
-        # for now, we use softlinks in order to keep the current
-        # version of the platy browser working
-        src_h5 = get_h5_path_from_xml(src_xml, return_absolute_path=True)
-        trgt_h5 = os.path.splitext(n2)[0] + '.h5'
-        os.symlink(src_h5, os.path.join(trgt_folder, trgt_h5))
-
-        copy_xml_with_newpath(src_xml, trgt_xml, trgt_h5)
-
-
-def make_initial_data_sources(copy_sbem,
-                              copy_prospr,
-                              copy_fib,
-                              copy_regions):
-    old_folder = '/g/arendt/EM_6dpf_segmentation/EM-Prospr'
-    raw_folder = './data/rawdata'
-    os.makedirs(raw_folder, exist_ok=True)
-
-    # TODO
-    # copy cellular models
-    if copy_sbem:
-        print("Copy sbem data")
-        # copy the sbem data
-        sbem_prefix = 'sbem-6dpf-1-whole'
-        name_dict = {'em-raw-full-res.xml': 'raw.xml',
-                     'em-segmented-muscles-ariadne.xml': 'segmented-muscle.xml',
-                     'em-segmented-tissue-labels.xml': 'segmented-tissue-labels.xml'}
-        name_dict = {k: '%s-%s' % (sbem_prefix, v)
-                     for k, v in name_dict.items()}
-        copy_xmls_and_symlink_h5(name_dict, old_folder, raw_folder)
-
-    if copy_prospr:
-        print("Copy prospr data")
-        prospr_prefix = 'prospr-6dpf-1-whole'
-        # copy the prospr meds
-        prospr_names = glob.glob(os.path.join(old_folder, "*-MED*"))
-        prospr_names = [os.path.split(f)[1] for f in prospr_names]
-        prospr_names = [name for name in prospr_names if os.path.splitext(name)[1] == '.xml']
-        name_dict = {n: '%s-%s' % (prospr_prefix, n) for n in prospr_names}
-        copy_xmls_and_symlink_h5(name_dict, old_folder, raw_folder)
-
-        # copy valentyna's med file
-        input_path = '/g/kreshuk/zinchenk/cell_match/data/meds_all_genes_500nm.h5'
-        output_path = os.path.join(raw_folder, '%s_meds_all_genes.h5' % prospr_prefix)
-        xml_path = os.path.join(raw_folder, '%s_meds_all_genes.xml' % prospr_prefix)
-        write_simple_xml(xml_path, os.path.split(output_path)[1],
-                         path_type='relative')
-        copyfile(input_path, output_path)
-
-    if copy_fib:
-        print("Coby fibsem data")
-        # copy the fibsem data
-        fib_prefix = 'fibsem-6dpf-1-parapod'
-        name_dict = {'em-raw-': 'em-raw',
-                     'em-segmented-': ''}
-        name_dict = {k: '%s-%s' % (fib_prefix, v)
-                     for k, v in name_dict.items()}
-        copy_xmls_and_symlink_h5(name_dict, old_folder, raw_folder)
-
-    if copy_regions:
-
-        print("Copy regions")
-        prospr_prefix = 'prospr-6dpf-1-whole-segmented'
-        prospr_names = glob.glob(os.path.join(old_folder, "BodyPart_*"))
-        prospr_names = [os.path.split(f)[1] for f in prospr_names]
-        prospr_names = [name for name in prospr_names if os.path.splitext(name)[1] == '.xml']
-        name_dict = {n: '%s-%s' % (prospr_prefix, n.split('_')[-1]) for n in prospr_names}
-        copy_xmls_and_symlink_h5(name_dict, old_folder, raw_folder)
-
-
-if __name__ == '__main__':
-    copy_sbem = False
-    copy_prospr = False
-    copy_fib = False
-    copy_regions = True
-    make_initial_data_sources(copy_sbem, copy_prospr, copy_fib,
-                              copy_regions)

deprecated/make_initial_version.py

-#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
-
-import os
-from shutil import copyfile
-from glob import glob
-
-import h5py
-from scripts.files import make_folder_structure
-from scripts.export import export_segmentation
-from scripts.files import make_bdv_server_file, copy_image_data, copy_misc_data
-from scripts.files import copy_segmentation
-from scripts.attributes import make_nucleus_tables, make_cell_tables
-from pybdv.converter import make_bdv
-
-
-def make_sbem_segmentations(old_folder, folder):
-    path = '/g/kreshuk/data/arendt/platyneris_v1/data.n5'
-
-    # export nucleus segemntation
-    tmp_nuclei = 'tmp_export_nuclei'
-    key_nuclei = 'volumes/paintera/nuclei'
-    nuclei_name = 'sbem-6dpf-1-whole-segmented-nuclei-labels'
-    res_nuclei = [.1, .08, .08]
-    export_segmentation(path, key_nuclei, old_folder, folder, nuclei_name, res_nuclei, tmp_nuclei)
-
-    # export cell segemntation
-    tmp_cells = 'tmp_export_cells'
-    key_cells = 'volumes/paintera/proofread_cells'
-    cells_name = 'sbem-6dpf-1-whole-segmented-cells-labels'
-    res_cells = [.025, .02, .02]
-    export_segmentation(path, key_cells, old_folder, folder, cells_name, res_cells, tmp_cells,
-                        target='local', max_jobs=8)
-
-
-def make_sbem_tables(folder):
-    # make cell segmentation tables
-    name_cells = 'sbem-6dpf-1-whole-segmented-cells-labels'
-    res_cells = [.025, .02, .02]
-    make_cell_tables(folder, name_cells, 'tmp_tables_cells',
-                     res_cells, target='local', max_jobs=32)
-
-    # make nucleus segmentation tables
-    name_nuclei = 'sbem-6dpf-1-whole-segmented-nuclei-labels'
-    res_nuclei = [.1, .08, .08]
-    make_nucleus_tables(folder, name_nuclei, 'tmp_tables_nuclei',
-                        res_nuclei, target='local', max_jobs=32)
-
-    old_folder = '/g/arendt/EM_6dpf_segmentation/EM-Prospr'
-    # copy tissue segmentation table
-    tissue_name_out = 'sbem-6dpf-1-whole-segmented-tissue-labels'
-    table_folder = os.path.join(folder, 'tables', tissue_name_out)
-    os.makedirs(table_folder, exist_ok=True)
-    tissue_table_in = os.path.join(old_folder, 'tables', 'em-segmented-tissue-labels.csv')
-    tissue_table_out = os.path.join(table_folder, 'default.csv')
-    copyfile(tissue_table_in, tissue_table_out)
-
-
-def make_prospr_region_segmentations():
-    in_prefix = '/g/arendt/EM_6dpf_segmentation/EM-Prospr/BodyPart_*.h5'
-    out_prefix = './data/rawdata/prospr-6dpf-1-whole-segmented-'
-    files = glob(in_prefix)
-    for p in files:
-        name = p.split('_')[-1][:-3]
-        o = out_prefix + name + '.h5'
-        print(p, "to", o)
-        with h5py.File(p) as f:
-            key = 't00000/s00/0/cells'
-            data = f[key][:]
-            data[data > 0] = 0
-            data[data < 0] = 255
-        make_bdv(data, o, 3 * [[2, 2, 2]],
-                 unit='micrometer', resolution=[0.5, 0.5, 0.5])
-
-
-def insert_chromatin():
-    src_folder = 'data/rawdata'
-    dst_folder = 'data/0.2.0'
-    name = 'sbem-6dpf-1-whole-segmented-chromatin-labels'
-    copy_segmentation(src_folder, dst_folder, name)
-    src_table = '/g/arendt/EM_6dpf_segmentation/EM-Prospr/tables/em-segmented-chromatin-labels.csv'
-    dst_table = os.path.join(dst_folder, 'tables', name)
-    os.makedirs(dst_table, exist_ok=True)
-    dst_table = os.path.join(dst_table, 'base.csv')
-    copyfile(src_table, dst_table)
-
-
-def make_initial_version():
-
-    src_folder = 'data/rawdata'
-    old_folder = '/g/arendt/EM_6dpf_segmentation/EM-Prospr'
-    tag = '0.0.0'
-    folder = os.path.join('data', tag)
-
-    make_folder_structure(folder)
-
-    # make xmls for all necessary image data
-    copy_image_data(src_folder, os.path.join(folder, 'images'))
-    copy_misc_data(src_folder, os.path.join(folder, 'misc'))
-
-    # export the initial sbem segmentations
-    make_sbem_segmentations(old_folder, folder)
-
-    # make the tables for sbem segmentations
-    make_sbem_tables(folder)
-
-    # make the bdv server file
-    make_bdv_server_file([os.path.join(folder, 'images'),
-                          os.path.join(folder, 'segmentations')],
-                         os.path.join(folder, 'misc', 'bdvserver.txt'))
-
-
-if __name__ == '__main__':
-    insert_chromatin()
-    # make_prospr_region_segmentations()
-    # make_initial_version()

deprecated/swap_links.py

-import os
-from shutil import move
-
-
-def swap_link(link, dest):
-    root_folder = '/g/arendt/EM_6dpf_segmentation/EM-Prospr'
-    print(link, '->', dest)
-
-    # 1.) remove the link
-    os.unlink(link)
-
-    # 2.) move dest to link
-    move(dest, link)
-
-    # 3.) make link with relative path from dest -> link
-    rel_path = os.path.relpath(os.path.abspath(link),
-                               root_folder)
-    # print(dest)
-    # print(rel_path)
-    os.symlink(rel_path, dest)
-
-
-def swap_links():
-    link_folder = './data/rawdata'
-    cwd = os.getcwd()
-    os.chdir(link_folder)
-    files = os.listdir('.')
-
-    for path in files:
-        is_link = os.path.islink(path)
-        if is_link:
-            dest = os.readlink(path)
-            assert os.path.exists(dest) and os.path.isfile(dest)
-            swap_link(path, dest)
-
-    os.chdir(cwd)
-
-
-if __name__ == '__main__':
-    swap_links()

deprecated/update_prospr_names.py

-import os
-import json
-from glob import glob
-
-
-def update_prospr_names(reference_folder):
-    with open('../data/images.json') as f:
-        im_names = json.load(f)
-    prefix = 'prospr-6dpf-1-whole'
-    prospr_names = [name for name in im_names if name.startswith(prefix)]
-    print(len(prospr_names))
-
-    new_prospr_names = glob(os.path.join(reference_folder, '%s-*.xml' % prefix))
-    print(len(new_prospr_names))
-    new_prospr_names = [os.path.splitext(name)[0] for name in new_prospr_names]
-    new_prospr_names = [os.path.split(name)[1] for name in new_prospr_names]
-
-    new_names = [name for name in im_names if not name.startswith(prefix)]
-    print(len(new_names))
-    new_names += new_prospr_names
-    print(len(new_names))
-
-    # check
-    for name in new_names:
-        path = os.path.join(reference_folder, name) + '.xml'
-        assert os.path.exists(path), path
-    print("Check passed")
-
-    with open('../data/images.json', 'w') as f:
-        json.dump(new_names, f)
-
-
-if __name__ == '__main__':
-    update_prospr_names('../data/0.5.4/images')

misc/check_n5_file.py

+import os
+import numpy as np
+import z5py
+
+
+def get_chunk_stats(path, key):
+    with z5py.File(path, 'r') as f:
+        ds = f[key]
+        n_chunks_tot = ds.number_of_chunks
+
+    ds_path = os.path.join(path, key)
+    chunk_sizes = []
+    for root, dirs, files in os.walk(ds_path):
+        for name in files:
+            if name == 'attributes.json':
+                continue
+            size = os.path.getsize(os.path.join(root, name))
+            chunk_sizes.append(size)
+
+    n_chunks_filled = len(chunk_sizes)
+
+    return n_chunks_tot, n_chunks_filled, chunk_sizes
+
+
+def summarise_chunk_stats(path, key):
+    n_chunks, n_filled, sizes = get_chunk_stats(path, key)
+    percent_filled = float(n_filled) / n_chunks
+    with z5py.File(path, 'r') as f:
+        ds = f[key]
+        chunk_shape = ds.chunks
+    print("Checked dataset with chunk shape", chunk_shape)
+    print("Number of existing chunks", n_filled, "/", n_chunks, "(", percent_filled, ")")
+    print("Mean chunk size in MB:", np.mean(sizes) / 1.e6, "+-", np.std(sizes) / 1.e6)
+    print("Min/max chunk size in MB:", np.min(sizes) / 1.e6, "/", np.max(sizes) / 1.e6)
+
+
+if __name__ == '__main__':
+    p = '../data/0.6.5/images/local/sbem-6dpf-1-whole-segmented-cells.n5'
+    k = 'setup0/timepoint0/s0'
+    summarise_chunk_stats(p, k)