Add scripts to copy tables

scripts/files/copy_helper.py

@@ -22,8 +22,23 @@ def copy_files_with_pattern(src_folder, dst_folder, pattern):
         copy_file(xml_in, xml_out)
 
 
+# For now we put symlinks with relative paths, but I am not sure
+# if this is the best idea, because I don't know if it will work on windows
 def copy_tables(src_folder, dst_folder, name):
-    pass
+    table_in = os.path.join(src_folder, 'tables', name)
+    table_out = os.path.join(dst_folder, 'tables', name)
+    os.makedirs(table_out, exist_ok=True)
+
+    table_files = os.listdir(table_in)
+    table_files = [ff for ff in table_files if os.path.splitext(ff)[1] == '.csv']
+
+    for ff in table_files:
+        src_file = os.path.join(table_in, ff)
+        dst_file = os.path.join(table_out, ff)
+
+        rel_path = os.path.relpath(src_file, table_out)
+        if not os.path.exists(dst_file):
+            os.symlink(rel_path, dst_file)
 
 
 def copy_segmentation(src_folder, dst_folder, name):
@@ -59,8 +74,3 @@ def copy_misc_data(src_folder, dst_folder):
     if os.path.exists(bkmrk_in):
         shutil.copyfile(bkmrk_in,
                         os.path.join(dst_folder, 'bookmarks.json'))
-
-
-# TODO
-def copy_static_segmentations(src_folder, dst_folder):
-    pass

update_platy_browser.py

@@ -95,11 +95,10 @@ def make_attributes(folder, new_folder,
     # update or copy nucleus tables
     if update_nucleus_tables:
         make_nucleus_tables(new_folder, NAME_NUCLEI, 'tmp_tables_nuclei', RES_NUCLEI,
-                         target=target, max_jobs=max_jobs)
+                            target=target, max_jobs=max_jobs)
     else:
         copy_tables(folder, new_folder, NAME_NUCLEI)
 
-    # TODO
     # copy tables associated with static segmentations
     static_seg_names = ('sbem-6dpf-1-whole-segmented-tissue-labels',)
     for seg_name in static_seg_names:
@@ -114,7 +113,7 @@ def make_release(tag, folder, description=''):
         call(['git', 'tag', tag])
     else:
         call(['git', '-m', description, 'tag', tag])
-    # TODO autopush ???
+    # TODO use the gitlab api instead
     # call(['git', 'push', 'origin', 'master', '--tags'])
 
 
@@ -170,8 +169,10 @@ def update_platy_browser(update_cell_segmentation=False,
     new_folder = os.path.join('data', new_tag)
     make_folder_structure(new_folder)
 
-    target = 'slurm'
-    max_jobs = 250
+    # target = 'slurm'
+    # max_jobs = 250
+    target = 'local'
+    max_jobs = 48
 
     # copy static image and misc data
     copy_image_data(os.path.join(folder, 'images'),
@@ -191,15 +192,15 @@ def update_platy_browser(update_cell_segmentation=False,
                     update_nucleus_tables,
                     target=target, max_jobs=max_jobs)
 
+    # TODO implement make release properly
+    return
     # make new release
     make_release(new_tag, new_folder, description)
     print("Updated platybrowser to new release", new_tag)
     print("All changes were successfully made. Starting clean up now.")
-    print("This can take a few hours, you can already use the new data; clean up will only remove temp files.")
-
-    # TODO would be better to start this as a job on a different machine, not the login node!
-    # TODO clean up tmp folders
-    # clean_up()
+    print("This can take a few hours, you can already use the new data.")
+    print("Clean-up will only remove temp files.")
+    clean_up()
 
 
 def str2bool(v):