Implement update_major and update_minor (not tested)

.gitignore

@@ -2,3 +2,4 @@
 __pycache__/
 tmp*
 *.DS_Store
+data/dev-*

make_dev_folder.py

@@ -3,7 +3,7 @@
 import argparse
 import os
 from subprocess import check_output
-from scripts.files import copy_release_folder
+from scripts.files import copy_release_folder, make_folder_structure
 
 
 def make_dev_folder(dev_name, version=''):
@@ -19,6 +19,7 @@ def make_dev_folder(dev_name, version=''):
     if not os.path.exists(folder):
         raise RuntimeError("Source folder %s does not exist" % folder)
 
+    make_folder_structure(new_folder)
     copy_release_folder(folder, new_folder)
 
 

scripts/export/export_segmentation.py

@@ -82,4 +82,6 @@ def export_segmentation(paintera_path, paintera_key, folder, new_folder, name, r
     to_bdv(tmp_path, tmp_key, out_path, resolution, tmp_bdv, target)
 
     # compute mapping to old segmentation
-    map_segmentation_ids(folder, new_folder, name, tmp_folder, max_jobs, target)
+    # this can be skipped for new segmentations by setting folder to None
+    if folder is not None:
+        map_segmentation_ids(folder, new_folder, name, tmp_folder, max_jobs, target)

scripts/files/bdv_server.py

 import os
 from .xml_utils import get_h5_path_from_xml
+from .sources import get_privates, get_image_names, get_segmentation_names
+
+
+def add_to_bdv_config(name, path, bdv_config, relative_paths, ref_dir):
+
+    # make sure that the h5path linked in the xml exists
+    h5path = get_h5_path_from_xml(path, return_absolute_path=True)
+    if not os.path.exists(h5path):
+        msg = 'Path to h5-file in xml does not exist - %s, %s' % (path, h5path)
+        return RuntimeError(msg)
+
+    if relative_paths:
+        path = os.path.relpath(path, ref_dir)
+    bdv_config[name] = path
 
 
-# TODO enable filtering for files by some pattern
 # e.g. if we don't want to expose the fib dataset to the public yet
-def make_bdv_server_file(folders, out_path, relative_paths=True):
-    """ Make the bigserver config file from all xmls in folders.
+def make_bdv_server_file(folder, out_path, relative_paths=True):
+    """ Make the bigserver config file for a given release.
     """
-    file_list = {}
+    privates = get_privates()
+    image_names = get_image_names()
+    seg_names = get_segmentation_names()
     ref_dir = os.path.split(out_path)[0]
-    for folder in folders:
-        files = os.listdir(folder)
-        for ff in files:
-            path = os.path.join(folder, ff)
-
-            # only add xmls
-            ext = os.path.splitext(path)[1]
-            if ext != '.xml':
-                continue
-
-            # make sure that the h5path linked in the xml exists
-            h5path = get_h5_path_from_xml(path, return_absolute_path=True)
-            if not os.path.exists(h5path):
-                msg = 'Path to h5-file in xml does not exist - %s, %s' % (path,
-                                                                          h5path)
-                return RuntimeError(msg)
-
-            name = os.path.splitext(ff)[0]
-            if relative_paths:
-                path = os.path.relpath(path, ref_dir)
-            file_list[name] = path
+
+    bdv_config = {}
+    for name in image_names:
+        if name in privates:
+            continue
+        path = os.path.join(folder, 'images', '%s.xml' % name)
+        add_to_bdv_config(name, path, bdv_config, relative_paths, ref_dir)
+
+    for name in seg_names:
+        if name in privates:
+            continue
+        path = os.path.join(folder, 'segmentations', '%s.xml' % name)
+        add_to_bdv_config(name, path, bdv_config, relative_paths, ref_dir)
 
     with open(out_path, 'w') as f:
-        for name, path in file_list.items():
+        for name, path in bdv_config.items():
             line = '%s\t%s\n' % (name, path)
             f.write(line)

scripts/files/copy_helper.py

 import os
 import shutil
 from .xml_utils import copy_xml_with_newpath, get_h5_path_from_xml
-from .sources import get_image_names, RAW_FOLDER
+from .sources import get_image_names, get_segmentation_names, get_segmentations
 
 
 def copy_file(xml_in, xml_out):
@@ -56,15 +56,8 @@ def copy_image_data(src_folder, dst_folder):
         name += '.xml'
         in_path = os.path.join(src_folder, name)
         out_path = os.path.join(dst_folder, name)
-        # we might have just added he image name, so it's not
-        # in the old version folder yet. It must be in the raw folder
-        # in that case
         if not os.path.exists(in_path):
-            in_path = os.path.join(RAW_FOLDER, name)
-        if not os.path.exists(in_path):
-            raise RuntimeError("Could not find %s in either the src folder %s or raw folder %s" % (name,
-                                                                                                   src_folder,
-                                                                                                   RAW_FOLDER))
+            raise RuntimeError("Could not find %s in the src folder %s or raw folder %s" % (name, src_folder))
         # copy the xml
         copy_file(in_path, out_path)
 
@@ -81,3 +74,36 @@ def copy_misc_data(src_folder, dst_folder):
     if os.path.exists(bkmrk_in):
         shutil.copyfile(bkmrk_in,
                         os.path.join(dst_folder, 'bookmarks.json'))
+
+
+def copy_segmentations(src_folder, dst_folder):
+    names = get_segmentation_names()
+    for name in names:
+        name += '.xml'
+        in_path = os.path.join(src_folder, name)
+        out_path = os.path.join(dst_folder, name)
+        if not os.path.exists(in_path):
+            raise RuntimeError("Could not find %s in the src folder %s or raw folder %s" % (name, src_folder))
+        # copy the xml
+        copy_file(in_path, out_path)
+
+
+def copy_all_tables(src_folder, dst_folder):
+    segmentations = get_segmentations()
+    for name, seg in segmentations.items():
+        has_table = seg.get('has_tables', False) or 'table_update_function' in seg
+        if not has_table:
+            continue
+        copy_tables(src_folder, dst_folder, name)
+
+
+def copy_release_folder(src_folder, dst_folder):
+    # copy static image and misc data
+    copy_image_data(os.path.join(src_folder, 'images'),
+                    os.path.join(dst_folder, 'images'))
+    copy_misc_data(os.path.join(src_folder, 'misc'),
+                   os.path.join(dst_folder, 'misc'))
+    copy_segmentations(os.path.join(src_folder, 'segmentations'),
+                       os.path.join(dst_folder, 'segmentations'))
+    copy_all_tables(os.path.join(src_folder, 'tables'),
+                    os.path.join(dst_folder, 'tables'))

scripts/files/sources.py

@@ -9,6 +9,7 @@ RAW_FOLDER = 'data/rawdata'
 SOURCE_FILE = 'data/sources.json'
 SEGMENTATION_FILE = 'data/segmentations.json'
 IMAGE_FILE = 'data/images.json'
+PRIVATE_FILE = 'data/privates.json'
 
 # TODO we need additional functionality:
 # - remove images and segmentations
@@ -89,7 +90,21 @@ def add_source(modality, stage, id=1, region='whole'):
         json.dump(sources, f)
 
 
-def add_image(source_name, name, input_path, copy_data=True):
+def get_privates():
+    if not os.path.exists(PRIVATE_FILE):
+        return []
+    with open(PRIVATE_FILE) as f:
+        return json.load(f)
+
+
+def add_to_privates(name):
+    privates = get_privates()
+    privates.append(name)
+    with open(PRIVATE_FILE, 'w') as f:
+        json.dump(privates, f)
+
+
+def add_image(source_name, name, input_path, copy_data=True, is_private=False):
     """ Add image volume to the platy browser data.
 
     Parameter:
@@ -98,7 +113,8 @@ def add_image(source_name, name, input_path, copy_data=True):
         input_path [str] - path to the data that should be added.
             Data needs to be in bdv-hdf5 format and the path needs to point to the xml.
         copy_data [bool] - whether to copy the data. This should be set to True,
-            unless adding an image volume that is already in the rawdata folder. (default: True)
+            unless adding an image volume that is already in the rawdata folder (default: True).
+        is_private [bool] - whether this data is private (default: False).
     """
     # validate the inputs
     source_names = get_source_names()
@@ -130,10 +146,15 @@ def add_image(source_name, name, input_path, copy_data=True):
     with open(IMAGE_FILE, 'w') as f:
         json.dump(names, f)
 
+    # add the name to the private list if is_private == True
+    if is_private:
+        add_to_privates(output_name)
+
 
 def add_segmentation(source_name, name, segmentation_path=None,
                      table_path_dict=None, paintera_project=None,
-                     resolution=None, table_update_function=None, copy_data=True):
+                     resolution=None, table_update_function=None,
+                     copy_data=True, is_private=False):
     """ Add segmentation volume to the platy browser data.
 
     We distinguish between static and dynamic segmentations. A dynamic segmentation is generated from
@@ -176,6 +197,7 @@ def add_segmentation(source_name, name, segmentation_path=None,
             the segmentation is updated from paintera corrections (default: None).
         copy_data [bool] - whether to copy the data. This should be set to True,
             unless adding a segmentation that is already in the rawdata folder. (default: True)
+        is_private [bool] - whether this data is private (default: False).
     """
     # validate the inputs
 
@@ -245,3 +267,6 @@ def add_segmentation(source_name, name, segmentation_path=None,
     segmentations[output_name] = segmentation
     with open(SEGMENTATION_FILE, 'w') as f:
         json.dump(segmentations, f)
+    # add the name to the private list if is_private == True
+    if is_private:
+        add_to_privates(output_name)

scripts/release_helper.py

+import os
+from . import attributes
+from .export import export_segmentation
+from .files import add_image, add_segmentation, copy_tables
+from .files.copy_helper import copy_file
+
+
+def is_image(data, check_source):
+    if check_source and 'source' not in data:
+        return False
+    if 'name' in data and 'input_path' in data:
+        return True
+    return False
+
+
+def is_static_segmentation(data, check_source):
+    if check_source and 'source' not in data:
+        return False
+    if 'name' in data and 'segmentation_path' in data:
+        return True
+    return False
+
+
+def is_dynamic_segmentation(data, check_source):
+    if check_source and 'source' not in data:
+        return False
+    if 'name' in data and 'paintera_project' in data and 'resolution' in data:
+        if len(data['paintera_project']) != 2 or len(data['resolution']) != 3:
+            return False
+        return True
+    return False
+
+
+def check_inputs(new_data, check_source=True):
+    if not all(isinstance(data, dict) for data in new_data):
+        raise ValueError("Expect list of dicts as input")
+
+    for data in new_data:
+        if not any(is_image(data, check_source), is_static_segmentation(data, check_source), is_dynamic_segmentation(data, check_source)):
+            raise ValueError("Could not parse input element %s" % str(data))
+
+
+def add_data(data, folder, target, max_jobs, source=None):
+    source = data['source'] if source is None else source
+    name = data['name']
+    full_name = '%s-%s' % (source, name)
+    file_name = '%s.xml' % full_name
+
+    is_private = data.get('is_private', False)
+    check_source = source is None
+
+    if is_image(data, check_source):
+        # register the image data
+        add_image(source, name, data['input_path'],
+                  is_private=is_private)
+
+        # copy image data from the raw folder to new release folder
+        xml_raw = os.path.join('data/raw', file_name)
+        xml_out = os.path.join(folder, file_name)
+        copy_file(xml_raw, xml_out)
+
+    elif is_static_segmentation(data, check_source):
+        # register the static segmentation
+        table_path_dict = data.get('table_path_dict', None)
+        add_segmentation(source, name,
+                         segmentation_path=data['segmentation_path'],
+                         table_path_dict=table_path_dict,
+                         is_private=is_private)
+
+        # copy segmentation data from the raw folder to new release folder
+        xml_raw = os.path.join('data/raw', file_name)
+        xml_out = os.path.join(folder, file_name)
+        copy_file(xml_raw, xml_out)
+
+        # if we have tables, copy them as well
+        if table_path_dict is not None:
+            copy_tables('data/raw', os.path.join(folder, 'tables'), full_name)
+
+    elif is_dynamic_segmentation(data, check_source):
+        # register the dynamic segmentation
+        paintera_project = data['paintera_project']
+        resolution = data['resolution']
+        table_update_function = data.get('table_update_function', None)
+        add_segmentation(source, name,
+                         paintera_project=paintera_project,
+                         resolution=resolution,
+                         table_update_function=table_update_function,
+                         is_private=is_private)
+
+        # export segmentation data to new release folder
+        paintera_root, paintera_key = paintera_project
+        tmp_folder = 'tmp_export_%s' % full_name
+        export_segmentation(paintera_root, paintera_key,
+                            None, folder, full_name,
+                            resolution=resolution,
+                            tmp_folder=tmp_folder,
+                            target=target, max_jobs=max_jobs)
+
+        # if we have a table update function, call it
+        if table_update_function is not None:
+            tmp_folder = 'tmp_tables_%s' % name
+            update_function = getattr(attributes, table_update_function)
+            update_function(folder, name, tmp_folder, resolution,
+                            target=target, max_jobs=max_jobs)

update_major.py

 #! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
+
+import os
+import json
+import argparse
+from subprocess import check_output
+
+from scripts.files import add_source, copy_release_folder, make_folder_structure
+from scripts.release_helper import (add_data, check_inputs,
+                                    is_image, is_static_segmentation, is_dynamic_segmentation)
+
+
+def get_tags():
+    tag = check_output(['git', 'describe', '--abbrev=0']).decode('utf-8').rstrip('\n')
+    new_tag = tag.split('.')
+    new_tag[-1] = '0'  # reset patch
+    new_tag[1] = '0'  # reset minor
+    # update major
+    new_tag[0] = str(int(new_tag[0]) + 1)
+    new_tag = '.'.join(new_tag)
+    return tag, new_tag
+
+
+def update_major(new_data_dict, target='slurm', max_jobs=250):
+    """ Update major version of platy browser.
+
+    TODO explain elements of input list.
+    """
+
+    for source, new_data in new_data_dict.items():
+        add_source(source)
+        check_inputs(new_data, check_source=False)
+
+    # increase the minor (middle digit) release tag
+    tag, new_tag = get_tags()
+    print("Updating platy browser from", tag, "to", new_tag)
+
+    # make new folder structure
+    folder = os.path.join('data', tag)
+    new_folder = os.path.join('data', new_tag)
+    make_folder_structure(new_folder)
+
+    # copy the release folder
+    copy_release_folder(folder, new_folder)
+
+    # add the new sources and new data
+    for source, new_data in new_data.items():
+        for data in new_data:
+            add_data(data, new_folder, target, max_jobs,
+                     source=source)
+
+    # TODO auto-release
+    # TODO clean up
+
+
+if __name__ == '__main__':
+    help_str = "Path to a json containing list of the data to add. See docstring of 'update_major' for details."
+    parser = argparse.ArgumentParser(description='Update major version of platy-browser-data.')
+    parser.add_argument('input_path', type=str, help=help_str)
+    input_path = parser.parse_args().input_path
+    with open(input_path) as f:
+        new_data_dict = json.load(f)
+    update_major(new_data_dict)

update_minor.py

@@ -5,9 +5,9 @@ import json
 import argparse
 from subprocess import check_output
 
-from scripts.files import (add_image, add_segmentation,
-                           copy_release_folder, make_folder_structure)
-from scripts.export import export_segmentation
+from scripts.files import copy_release_folder, make_folder_structure
+from scripts.release_helper import (add_data, check_inputs,
+                                    is_image, is_static_segmentation, is_dynamic_segmentation)
 
 
 def get_tags():
@@ -19,26 +19,6 @@ def get_tags():
     return tag, new_tag
 
 
-def is_image(data):
-    if 'source' in data and 'name' in data and 'input_path' in data:
-        return True
-    return False
-
-
-def is_static_segmentation(data):
-    if 'source' in data and 'name' in data and 'segmentation_path' in data:
-        return True
-    return False
-
-
-def is_dynamic_segmentation(data):
-    if 'source' in data and 'name' in data and 'paintera_project' in data and 'resolution' in data:
-        if len(data['paintera_project']) != 2 or len(data['resolution']) != 3:
-            return False
-        return True
-    return False
-
-
 def check_inputs(new_data):
     if not all(isinstance(data, dict) for data in new_data):
         raise ValueError("Expect list of dicts as input")
@@ -48,36 +28,6 @@ def check_inputs(new_data):
             raise ValueError("Could not parse input element %s" % str(data))
 
 
-def add_data(data, folder, target, max_jobs):
-    if is_image(data):
-        # register the image data
-        add_image(data['source'], data['name'], data['input_path'])
-
-        # copy image data to new release folder
-
-    elif is_static_segmentation(data):
-        # register the static segmentation
-        add_segmentation(data['source'], data['name'],
-                         segmentation_path=data['segmentation_path'],
-                         table_path_dict=data.get('table_path_dict', None))
-
-        # copy segmentation data to new release folder
-
-        # if we have tables, copy them as well
-
-    elif is_dynamic_segmentation(data):
-        # register the dynamic segmentation
-        add_segmentation(data['source'], data['name'],
-                         paintera_project=data['paintera_project'],
-                         resolution=data['resolution'],
-                         table_update_function=data.get('table_update_function', None))
-
-        # export segmentation data to new release folder
-        export_segmentation()
-
-        # if we have a table update function, call it
-
-
 def update_minor(new_data, target='slurm', max_jobs=250):
     """ Update minor version of platy browser.
 

update_patch.py

@@ -154,9 +154,7 @@ def update_patch(update_seg_names, update_table_names,
     update_tables(folder, new_folder, table_updates,
                   target=target, max_jobs=max_jobs)
 
-    make_bdv_server_file([os.path.join(new_folder, 'images'),
-                          os.path.join(new_folder, 'segmentations')],
-                         os.path.join(new_folder, 'misc', 'bdv_server.txt'),
+    make_bdv_server_file(new_folder, os.path.join(new_folder, 'misc', 'bdv_server.txt'),
                          relative_paths=True)
     # TODO add some quality control that cheks that all files are there