Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
P
platy-browser-data
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Christian Tischer
platy-browser-data
Commits
e8bec9d1
Commit
e8bec9d1
authored
5 years ago
by
Constantin Pape
Browse files
Options
Downloads
Patches
Plain Diff
Update apply_registration
parent
975cc6b3
No related branches found
Branches containing commit
No related tags found
Tags containing commit
1 merge request
!6
Add python scripts to run registration on the cluster
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
scripts/extension/registration/apply_registration.py
+18
-11
18 additions, 11 deletions
scripts/extension/registration/apply_registration.py
test/registration/check_wrapper.py
+16
-47
16 additions, 47 deletions
test/registration/check_wrapper.py
with
34 additions
and
58 deletions
scripts/extension/registration/apply_registration.py
+
18
−
11
View file @
e8bec9d1
...
...
@@ -15,6 +15,8 @@ from cluster_tools.cluster_tasks import SlurmTask, LocalTask, LSFTask
class
ApplyRegistrationBase
(
luigi
.
Task
):
"""
ApplyRegistration base class
"""
default_fiji
=
'
/g/arendt/EM_6dpf_segmentation/platy-browser-data/software/Fiji.app/ImageJ-linux64
'
default_elastix
=
'
/g/arendt/EM_6dpf_segmentation/platy-browser-data/software/elastix_v4.8
'
task_name
=
'
apply_registration
'
src_file
=
os
.
path
.
abspath
(
__file__
)
...
...
@@ -23,8 +25,8 @@ class ApplyRegistrationBase(luigi.Task):
input_path_file
=
luigi
.
Parameter
()
output_path_file
=
luigi
.
Parameter
()
transformation_file
=
luigi
.
Parameter
()
fiji_executable
=
luigi
.
Parameter
(
default
=
'
/g/almf/software/Fiji.app/ImageJ-linux64
'
)
elastix_directory
=
luigi
.
Parameter
(
default
=
'
/g/almf/software/
elastix
_v4.8
'
)
fiji_executable
=
luigi
.
Parameter
(
default
=
default_fiji
)
elastix_directory
=
luigi
.
Parameter
(
default
=
default_
elastix
)
dependency
=
luigi
.
TaskParameter
(
default
=
DummyTask
())
def
requires
(
self
):
...
...
@@ -108,12 +110,12 @@ def apply_for_file(input_path, output_path,
# with individual arguments comma separated
# the argument to transformaix needs to be one large comma separated string
transformix_argument
=
[
"
elastixDirectory=
\'
%s
\'
"
%
elastix_directory
,
"
workingDirectory=
\'
%s
\'
"
%
tmp_folder
,
"
workingDirectory=
\'
%s
\'
"
%
os
.
path
.
abspath
(
tmp_folder
)
,
"
inputImageFile=
\'
%s
\'
"
%
input_path
,
"
transformationFile=
\'
%s
\'
"
%
transformation_file
,
"
outputFile=
\'
%s
\'
"
%
output_path
,
"
outputModality=
\'
Save as BigDataViewer .xml/.h5
\'
"
,
"
numThreads=
\'
1
\'
"
]
# TODO why do we use numThreads=1 and not the same as in -c?
"
numThreads=
\'
%i
\'
"
%
n_threads
]
transformix_argument
=
"
,
"
.
join
(
transformix_argument
)
transformix_argument
=
"
\"
%s
\"
"
%
transformix_argument
...
...
@@ -129,14 +131,16 @@ def apply_for_file(input_path, output_path,
fu
.
log
(
"
Calling the following command:
"
)
fu
.
log
(
cmd_str
)
# the elastix wrapper only works properly if we set these as environment variables as well, see
# TODO make issue about this
# os.environ['TMPDIR'] = tmp_folder
os
.
environ
[
'
TRAFO
'
]
=
transformation_file
cwd
=
os
.
getcwd
()
try
:
# we need to change the working dir to the transformation directroy, so that relative paths in
# the transformations are correct
trafo_dir
=
os
.
path
.
split
(
transformation_file
)[
0
]
fu
.
log
(
"
Change directory to %s
"
%
trafo_dir
)
os
.
chdir
(
trafo_dir
)
# check_output(cmd)
# the CLI parser is very awkward
(to put it nicely)
.
# the CLI parser is very awkward.
# I could only get it to work by passing the whole command string
# and setting shell to True.
# otherwise, it would parse something wrong, and do nothing but
...
...
@@ -144,7 +148,10 @@ def apply_for_file(input_path, output_path,
# [WARNING] Ignoring invalid argument: --run
check_output
([
cmd_str
],
shell
=
True
)
except
CalledProcessError
as
e
:
raise
RuntimeError
(
e
.
output
)
raise
RuntimeError
(
e
.
output
.
decode
(
'
utf-8
'
))
finally
:
fu
.
log
(
"
Go back to cwd: %s
"
%
cwd
)
os
.
chdir
(
cwd
)
def
apply_registration
(
job_id
,
config_path
):
...
...
This diff is collapsed.
Click to expand it.
test/registration/check_wrapper.py
+
16
−
47
View file @
e8bec9d1
...
...
@@ -2,17 +2,14 @@ import os
import
json
import
luigi
# TODO remove path hack once we merge this into master
import
sys
sys
.
path
.
insert
(
0
,
'
../..
'
)
from
scripts.extension.registration
import
ApplyRegistrationLocal
def
check_wrapper
_simple
():
in_path
=
'
/g/
almf/software/elastix-test/muscles
.tif
'
def
check_wrapper
():
in_path
=
'
/g/
kreshuk/pape/Work/my_projects/platy-browser-data/registration/9.9.9/ProSPr/stomach
.tif
'
tmp_folder
=
'
/g/kreshuk/pape/Work/my_projects/dev-platy/test/registration/
tmp_registration
_simple
'
out_path
=
os
.
path
.
join
(
tmp_folder
,
'
out
'
)
tmp_folder
=
os
.
path
.
abspath
(
'
tmp_registration
'
)
out_path
=
os
.
path
.
join
(
tmp_folder
,
'
stomach_prospr_registered
'
)
in_list
=
[
in_path
]
out_list
=
[
out_path
]
...
...
@@ -33,51 +30,23 @@ def check_wrapper_simple():
with
open
(
os
.
path
.
join
(
conf_dir
,
'
global.config
'
),
'
w
'
)
as
f
:
json
.
dump
(
global_conf
,
f
)
trafo
=
'
/g/almf/software/elastix-test/TransformParameters.RotationPreAlign.0.txt
'
t
=
task
(
tmp_folder
=
tmp_folder
,
config_dir
=
conf_dir
,
max_jobs
=
1
,
input_path_file
=
in_file
,
output_path_file
=
out_file
,
transformation_file
=
trafo
)
ret
=
luigi
.
build
([
t
],
local_scheduler
=
True
)
assert
ret
expected_out_xml
=
out_path
+
'
.xml
'
assert
os
.
path
.
exists
(
expected_out_xml
),
expected_out_xml
expected_out_h5
=
out_path
+
'
.h5
'
assert
os
.
path
.
exists
(
expected_out_h5
),
expected_out_h5
def
check_wrapper
():
in_path
=
os
.
path
.
join
(
'
/g/kreshuk/pape/Work/my_projects/platy-browser-data/registration/9.9.9/images/ProSPr
'
,
'
Stomach_forRegistration.tif
'
)
out_path
=
'
/g/kreshuk/pape/Work/my_projects/dev-platy/test/registration/somach_prospr_registered
'
in_list
=
[
in_path
]
out_list
=
[
out_path
]
in_file
=
'
./in_list.json
'
with
open
(
in_file
,
'
w
'
)
as
f
:
json
.
dump
(
in_list
,
f
)
out_file
=
'
./out_list.json
'
with
open
(
out_file
,
'
w
'
)
as
f
:
json
.
dump
(
out_list
,
f
)
trafo_dir
=
'
/g/kreshuk/pape/Work/my_projects/platy-browser-data/registration/0.0.0/transformations
'
task
=
ApplyRegistrationLocal
conf_dir
=
'
./configs
'
os
.
makedirs
(
conf_dir
,
exist_ok
=
True
)
# This is the full transformation, but it takes a lot of time!
trafo
=
os
.
path
.
join
(
trafo_dir
,
'
TransformParameters.BSpline10-3Channels.0.txt
'
)
global_conf
=
task
.
default_global_config
()
shebang
=
'
/g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env37/bin/python
'
global_conf
.
update
({
'
shebang
'
:
shebang
})
with
open
(
os
.
path
.
join
(
conf_dir
,
'
global.config
'
),
'
w
'
)
as
f
:
json
.
dump
(
global_conf
,
f
)
# For now, we use the similarity trafo to save time
trafo
=
os
.
path
.
join
(
trafo_dir
,
'
TransformParameters.Similarity-3Channels.0.txt
'
)
# TODO which one is the correct trafo ?
trafo
=
os
.
path
.
join
(
'
/g/kreshuk/pape/Work/my_projects/platy-browser-data/registration/0.0.0/transformations/0.0.0
'
,
'
TransformParameters.BSpline10-3Channels.0.txt
'
)
t
=
task
(
tmp_folder
=
'
tmp_registration
'
,
config_dir
=
conf_dir
,
max_jobs
=
1
,
t
=
task
(
tmp_folder
=
tmp_folder
,
config_dir
=
conf_dir
,
max_jobs
=
1
,
input_path_file
=
in_file
,
output_path_file
=
out_file
,
transformation_file
=
trafo
)
ret
=
luigi
.
build
([
t
],
local_scheduler
=
True
)
assert
ret
expected_out
=
out_path
+
'
.xml
'
assert
os
.
path
.
exists
(
expected_out
),
expected_out
expected_xml
=
out_path
+
'
.xml
'
assert
os
.
path
.
exists
(
expected_xml
),
expected_xml
expected_h5
=
out_path
+
'
.h5
'
assert
os
.
path
.
exists
(
expected_h5
),
expected_h5
# check_wrapper()
check_wrapper
_simple
()
if
__name__
==
'
__main__
'
:
check_wrapper
()
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment