Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
import os
import json
import argparse
from subprocess import check_output
import luigi
from scripts.files import copy_release_folder, make_folder_structure, make_bdv_server_file
from scripts.release_helper import add_version
from scripts.extension.registration import ApplyRegistrationLocal, ApplyRegistrationSlurm
from scripts.default_config import get_default_shebang
def get_tags():
tag = check_output(['git', 'describe', '--abbrev=0']).decode('utf-8').rstrip('\n')
new_tag = tag.split('.')
new_tag[-1] = str(int(new_tag[-1]) + 1)
new_tag = '.'.join(new_tag)
return tag, new_tag
def apply_registration(input_folder, new_folder,
transformation_file, source_prefix,
target, max_jobs):
task = ApplyRegistrationSlurm if target == 'slurm' else ApplyRegistrationLocal
tmp_folder = './tmp_registration'
os.makedirs(tmp_folder, exist_ok=True)
# find all input files
names = os.listdir(input_folder)
inputs = [os.path.join(input_folder, name) for name in names]
if len(inputs) == 0:
raise RuntimeError("Did not find any files with prefix %s in %s" % (source_prefix,
input_folder))
output_folder = os.path.join(new_folder, 'images')
# TODO parse names to get output names
output_names = []
outputs = [os.path.join(output_folder, name) for name in output_names]
# update the task config
config_dir = os.path.join(tmp_folder, 'config')
os.makedirs(config_dir, exist_ok=True)
shebang = get_default_shebang()
global_config = task.default_global_config()
global_config.update({'shebang': shebang})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(global_config, f)
# TODO more time than 3 hrs?
task_config = task.default_task_config()
task_config.update({'mem_limit': 16, 'time_limit': 180})
with open(os.path.join(config_dir, 'apply_registration.config'), 'w') as f:
json.dump(task_config, f)
# write path name files to json
input_file = os.path.join(tmp_folder, 'input_files.json')
with open(input_file, 'w') as f:
json.dump(inputs, f)
output_file = os.path.joout(tmp_folder, 'output_files.json')
with open(output_file, 'w') as f:
json.dump(outputs, f)
t = task(tmp_folder=tmp_folder, config_dir=config_dir, max_jobs=max_jobs,
input_path_file=input_file, output_path_file=output_file,
transformation_file=transformation_file)
ret = luigi.build([t], local_scheduler=True)
if not ret:
raise RuntimeError("Registration failed")
def update_regestration(transformation_file, input_folder, source_prefix, target, max_jobs):
""" Update the prospr segmentation.
This is a special case of 'update_patch', that applies a new prospr registration.
Arguments:
transformation_file [str] - path to the transformation used to register
input_folder [str] - folder with unregistered data
source_prefix [str] - prefix of the source data to apply the registration to
target [str] - target of computation
max_jobs [int] - max number of jobs for computation
"""
tag, new_tag = get_tags()
print("Updating platy browser from", tag, "to", new_tag)
# make new folder structure
folder = os.path.join('data', tag)
new_folder = os.path.join('data', new_tag)
make_folder_structure(new_folder)
# copy the release folder
copy_release_folder(folder, new_folder)
# apply new registration to all files of the source prefix
apply_registration(input_folder, new_folder,
transformation_file, source_prefix,
target, max_jobs)
add_version(new_tag)
make_bdv_server_file(new_folder, os.path.join(new_folder, 'misc', 'bdv_server.txt'),
relative_paths=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Update prospr registration in platy-browser-data.')
parser.add_argument('transformation_file', type=str, help="path to transformation file")
parser.add_argument('--input_folder', type=str, default="data/rawdata/prospr",
help="Folder with (not registered) input files")
help_str = "Prefix for the input data. Please change this if you change the 'input_folder' from its default value"
parser.add_argument('--source_prefix', type=str, default="prospr-6dpf-1-whole",
help=help_str)
parser.add_argument('--target', type=str, default='slurm',
help="Computatin plaform, can be 'slurm' or 'local'")
parser.add_argument('--max_jobs', type=int, default=100,
help="Maximal number of jobs used for computation")
args = parser.parse_args()
update_regestration(args.transformation_file, args.input_folder, args.source_prefix,
args.target, args.max_jobs)