Skip to content
Snippets Groups Projects
Commit 72dfb368 authored by Constantin Pape's avatar Constantin Pape
Browse files

Add scripts to copy tables

parent 1c8e755e
No related branches found
No related tags found
No related merge requests found
......@@ -22,8 +22,23 @@ def copy_files_with_pattern(src_folder, dst_folder, pattern):
copy_file(xml_in, xml_out)
# For now we put symlinks with relative paths, but I am not sure
# if this is the best idea, because I don't know if it will work on windows
def copy_tables(src_folder, dst_folder, name):
pass
table_in = os.path.join(src_folder, 'tables', name)
table_out = os.path.join(dst_folder, 'tables', name)
os.makedirs(table_out, exist_ok=True)
table_files = os.listdir(table_in)
table_files = [ff for ff in table_files if os.path.splitext(ff)[1] == '.csv']
for ff in table_files:
src_file = os.path.join(table_in, ff)
dst_file = os.path.join(table_out, ff)
rel_path = os.path.relpath(src_file, table_out)
if not os.path.exists(dst_file):
os.symlink(rel_path, dst_file)
def copy_segmentation(src_folder, dst_folder, name):
......@@ -59,8 +74,3 @@ def copy_misc_data(src_folder, dst_folder):
if os.path.exists(bkmrk_in):
shutil.copyfile(bkmrk_in,
os.path.join(dst_folder, 'bookmarks.json'))
# TODO
def copy_static_segmentations(src_folder, dst_folder):
pass
......@@ -95,11 +95,10 @@ def make_attributes(folder, new_folder,
# update or copy nucleus tables
if update_nucleus_tables:
make_nucleus_tables(new_folder, NAME_NUCLEI, 'tmp_tables_nuclei', RES_NUCLEI,
target=target, max_jobs=max_jobs)
target=target, max_jobs=max_jobs)
else:
copy_tables(folder, new_folder, NAME_NUCLEI)
# TODO
# copy tables associated with static segmentations
static_seg_names = ('sbem-6dpf-1-whole-segmented-tissue-labels',)
for seg_name in static_seg_names:
......@@ -114,7 +113,7 @@ def make_release(tag, folder, description=''):
call(['git', 'tag', tag])
else:
call(['git', '-m', description, 'tag', tag])
# TODO autopush ???
# TODO use the gitlab api instead
# call(['git', 'push', 'origin', 'master', '--tags'])
......@@ -170,8 +169,10 @@ def update_platy_browser(update_cell_segmentation=False,
new_folder = os.path.join('data', new_tag)
make_folder_structure(new_folder)
target = 'slurm'
max_jobs = 250
# target = 'slurm'
# max_jobs = 250
target = 'local'
max_jobs = 48
# copy static image and misc data
copy_image_data(os.path.join(folder, 'images'),
......@@ -191,15 +192,15 @@ def update_platy_browser(update_cell_segmentation=False,
update_nucleus_tables,
target=target, max_jobs=max_jobs)
# TODO implement make release properly
return
# make new release
make_release(new_tag, new_folder, description)
print("Updated platybrowser to new release", new_tag)
print("All changes were successfully made. Starting clean up now.")
print("This can take a few hours, you can already use the new data; clean up will only remove temp files.")
# TODO would be better to start this as a job on a different machine, not the login node!
# TODO clean up tmp folders
# clean_up()
print("This can take a few hours, you can already use the new data.")
print("Clean-up will only remove temp files.")
clean_up()
def str2bool(v):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment