Commit b355fb8d authored by Martin Schorb's avatar Martin Schorb
Browse files

finalize XML

parent c3b2940f
Pipeline #22635 canceled with stage
...@@ -13,6 +13,7 @@ from dash.exceptions import PreventUpdate ...@@ -13,6 +13,7 @@ from dash.exceptions import PreventUpdate
from dash.dependencies import Input,Output,State from dash.dependencies import Input,Output,State
# import sys # import sys
import glob
import numpy as np import numpy as np
import os import os
import json import json
...@@ -34,6 +35,21 @@ parent = "finalize" ...@@ -34,6 +35,21 @@ parent = "finalize"
page=[html.Br()] page=[html.Br()]
# select output volume
page3 = html.Div([html.H4("Choose exported volume"),
dcc.Dropdown(id=label+'_input_dd',persistence=True)
])
page.append(page3)
# ============================================= # =============================================
# Start Button # Start Button
...@@ -51,6 +67,52 @@ gobutton = html.Div(children=[html.Br(), ...@@ -51,6 +67,52 @@ gobutton = html.Div(children=[html.Br(),
page.append(gobutton) page.append(gobutton)
# Volume selection list callback
# =============================================
@app.callback([Output(label+'_input_dd', 'options'),
Output(label+'_input_dd', 'value')],
[Input(parent+'_format_dd', 'value'),
Input('url', 'pathname')
])
def finalize_volume_dd(dd_in,url):
# if not dash.callback_context.triggered:
# raise PreventUpdate
expjson_list = glob.glob(os.path.join(params.json_run_dir,'*export_'+params.user+'*'))
dts = []
dd_options=list(dict())
for jsonfile in expjson_list:
with open(jsonfile,'r') as f:
export_json = json.load(f)
datetime = jsonfile.split(params.user+'_')[1].strip('.json')
dts.append(datetime)
vfile = export_json['--n5Path']
vdescr = ' - '.join([export_json['--project'],
export_json['--stack'],
datetime,
vfile.split('_')[-1].split('.')[0]])
dd_options.append({'label':vdescr,'value':jsonfile})
latest = dd_options[np.argsort(dts)[-1]]['value']
return dd_options, latest
# ============================================= # =============================================
# LAUNCH CALLBACK FUNCTION # LAUNCH CALLBACK FUNCTION
...@@ -65,10 +127,9 @@ page.append(gobutton) ...@@ -65,10 +127,9 @@ page.append(gobutton)
Output({'component': 'store_r_launch', 'module': parent},'data'), Output({'component': 'store_r_launch', 'module': parent},'data'),
], ],
[Input({'component': 'go', 'module': label}, 'n_clicks'), [Input({'component': 'go', 'module': label}, 'n_clicks'),
Input(parent+'_input_dd', 'value')] Input(label+'_input_dd', 'value')]
) )
def n5export_execute_gobutton(click,jsonfile): def n5export_execute_gobutton(click,jsonfile):
if not dash.callback_context.triggered: if not dash.callback_context.triggered:
raise PreventUpdate raise PreventUpdate
...@@ -85,17 +146,17 @@ def n5export_execute_gobutton(click,jsonfile): ...@@ -85,17 +146,17 @@ def n5export_execute_gobutton(click,jsonfile):
stack = export_json['--stack'] stack = export_json['--stack']
# if not os.path.exists(n5file): if not os.path.exists(n5file):
# return True, 'Input data file does not exist.', dash.no_update return True, 'Input data file does not exist.', dash.no_update
# if not os.access(n5file,os.W_OK | os.X_OK): if not os.access(n5file,os.W_OK | os.X_OK):
# return True,'Output directory not writable!', dash.no_update return True,'Output directory not writable!', dash.no_update
trigger = hf.trigger() trigger = hf.trigger()
# if 'input' in trigger: if 'input' in trigger:
# return False,'', dash.no_update return False,'', dash.no_update
# get stack parameters from render server # get stack parameters from render server
...@@ -103,17 +164,36 @@ def n5export_execute_gobutton(click,jsonfile): ...@@ -103,17 +164,36 @@ def n5export_execute_gobutton(click,jsonfile):
stackparams = requests.get(url).json() stackparams = requests.get(url).json()
res = [stackparams['currentVersion']['stackResolutionX'],stackparams['currentVersion']['stackResolutionY'],stackparams['currentVersion']['stackResolutionZ']] res = [stackparams['currentVersion']['stackResolutionZ'],stackparams['currentVersion']['stackResolutionX'],stackparams['currentVersion']['stackResolutionY']]
out = dict() out = dict()
out['state'] = 'launch' out['state'] = 'launch'
out['logfile'] = '' out['logfile'] = ''
mkxml_p = launch_jobs.run(target='standalone',pyscript='filetypes/make_xml.py', run_params = dict()
run_args=n5file+' '+str(res))
run_params['path'] = n5file
# run_params["scale_factors"] = 3 * [[2, 2, 2]],
run_params["resolution"] = res
# run_params["unit"] = 'micrometer'
param_file = params.json_run_dir + '/' + parent + '_' + params.run_prefix + '.json'
with open(param_file,'w') as f:
json.dump(run_params,f,indent=4)
log_file = params.render_log_dir + '/' + parent + '_' + params.run_prefix
err_file = log_file + '.err'
log_file += '.log'
mkxml_p = launch_jobs.run(target='standalone',pyscript='rendermodules/materialize/make_xml.py',json = param_file,
logfile=log_file,errfile=err_file)
params.processes[parent].extend(mkxml_p) params.processes[parent].extend(mkxml_p)
return True, 'Input data file does not exist.', out return True, '', out
from pybdv.metadata import write_xml_metadata, write_n5_metadata, validate_attributes
import sys
def make_render_xml(path, scale_factors = 3 * [[2, 2, 2]], resolution = [0.05, 0.015, 0.015], unit = 'micrometer'):
xml_path = path.replace('.n5', '.xml')
attrs = {'channel': {'id': None}}
attrs = validate_attributes(xml_path, attrs, setup_id=0,
enforce_consistency=False)
write_xml_metadata(xml_path, path, unit, resolution,
is_h5=False,
setup_id=0, timepoint=0,
setup_name=None,
affine=None,
attributes=attrs,
overwrite=False,
overwrite_data=False,
enforce_consistency=False)
write_n5_metadata(path, scale_factors, resolution, setup_id=0, timepoint=0, overwrite=True)
if __name__ == '__main__':
p = sys.argv[1]
res = sys.argv[2]
make_render_xml(p, resolution = res)
...@@ -12,10 +12,10 @@ from dash.exceptions import PreventUpdate ...@@ -12,10 +12,10 @@ from dash.exceptions import PreventUpdate
from dash.dependencies import Input,Output,State from dash.dependencies import Input,Output,State
import os # import os
import glob # import glob
import json # import json
import numpy as np # import numpy as np
# import requests # import requests
import params import params
...@@ -63,63 +63,6 @@ page.append(page2) ...@@ -63,63 +63,6 @@ page.append(page2)
# select output volume
page3 = html.Div([html.H4("Choose exported volume"),
dcc.Dropdown(id=module+'_input_dd',persistence=True)
])
page.append(page3)
@app.callback([Output(module+'_input_dd', 'options'),
Output(module+'_input_dd', 'value')],
[Input(module+'_format_dd', 'value'),
Input({'component': 'store_r_launch', 'module': previous},'modified_timestamp'),
Input('url', 'pathname')
])
def finalize_volume_dd(dd_in,prev_in,url):
if not dash.callback_context.triggered:
raise PreventUpdate
expjson_list = glob.glob(os.path.join(params.json_run_dir,'*export_'+params.user+'*'))
print(os.path.join(params.json_run_dir,'*export_'+params.user+'*'))
dts = []
dd_options=list(dict())
for jsonfile in expjson_list:
with open(jsonfile,'r') as f:
export_json = json.load(f)
datetime = jsonfile.split(params.user+'_')[1].strip('.json')
dts.append(datetime)
vfile = export_json['--n5Path']
vdescr = ' - '.join([export_json['--project'],
export_json['--stack'],
datetime,
vfile.split('_')[-1].split('.')[0]])
dd_options.append({'label':vdescr,'value':jsonfile})
latest = dd_options[np.argsort(dts)[-1]]['value']
return dd_options, latest
# ============================================= # =============================================
# # Page content for specific export call # # Page content for specific export call
......
...@@ -24,7 +24,7 @@ render_log_dir = '/g/emcf/software/render-logs' ...@@ -24,7 +24,7 @@ render_log_dir = '/g/emcf/software/render-logs'
hotknife_dir = "/g/emcf/schorb/code/hot-knife" hotknife_dir = "/g/emcf/schorb/code/hot-knife"
# base directory for launchers etc... # base directory for launchers etc...
workdir = '/g/emcf/software/volumealign/dash' workdir = '/g/emcf/schorb/code/volumealign/dash'
# emails # emails
......
...@@ -302,7 +302,7 @@ def canceljobs(job_ids): ...@@ -302,7 +302,7 @@ def canceljobs(job_ids):
def run(target='standalone',pyscript='thispyscript',json='',run_args=None,target_args=None,logfile=os.path.join(params.render_log_dir,'render.out'),errfile=os.path.join(params.render_log_dir,'render.err')): def run(target='standalone',pyscript='thispyscript',json='',run_args='',target_args=None,logfile=os.path.join(params.render_log_dir,'render.out'),errfile=os.path.join(params.render_log_dir,'render.err')):
my_env = os.environ.copy() my_env = os.environ.copy()
os.chdir(workdir) os.chdir(workdir)
command = '../'+target command = '../'+target
...@@ -322,13 +322,22 @@ def run(target='standalone',pyscript='thispyscript',json='',run_args=None,target ...@@ -322,13 +322,22 @@ def run(target='standalone',pyscript='thispyscript',json='',run_args=None,target
command += run_args command += run_args
print(command) print(command)
with open(logfile,"wb") as out, open(errfile,"wb") as err: with open(logfile,"wb") as out, open(errfile,"wb") as err:
p = subprocess.Popen(command, stdout=out,stderr=err, shell=True, env=my_env, executable='bash') p = subprocess.Popen(command, stdout=out,stderr=err, shell=True, env=my_env, executable='bash')
return [p] return [p]
elif target == 'generic':
command = pyscript
command += ' '+run_args
print(command)
with open(logfile,"wb") as out, open(errfile,"wb") as err:
p = subprocess.Popen(command, stdout=out,stderr=err, shell=True, env=my_env, executable='bash')
return [p]
elif target == 'slurm': elif target == 'slurm':
command += pyscript command += pyscript
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment