Commit aac27b94 authored by Constantin Pape's avatar Constantin Pape

Remove deprecated utils module

parent 56cadac3
# EMBL-Tools
Tools for data visualisation and to submit gpu jobs on the embl cluster.
## Installation
Activate your conda environment, e.g. from `../stardist/environment-gpu.yaml` and run
```
pip install -e .
```
## Usage
This will install two scripts:
```
view_data /path/to/folder
```
that can be used to visualise training data and predictions stored in our training data layout.
```
submit_slurm <SCRIPT_NAME> <SCRIPT_ARGS>
```
to run an arbitrary script on the cluster gpu queue.
from setuptools import setup, find_packages
setup(
name="ai4ia.embl_tools",
packages=find_packages(),
version="0.0.1",
author="Constantin Pape",
url="https://git.embl.de/grp-bio-it/ai4ia",
license='MIT',
entry_points={
"console_scripts": [
"view_data = utils_impl.view_data:main",
"submit_slurm = utils_impl.submit_to_slurm:main"
]
},
)
#! /usr/bin/python3
import os
import sys
import inspect
import subprocess
from datetime import datetime
# two days in minutes
TWO_DAYS = 2 * 24 * 60
def write_slurm_template(script, out_path, env_name,
n_threads, gpu_type, n_gpus,
mem_limit, time_limit, qos,
mail_address):
slurm_template = ("#!/bin/bash\n"
"#SBATCH -N 1\n"
"#SBATCH -c %s\n"
"#SBATCH --mem %s\n"
"#SBATCH -t %i\n"
"#SBATCH --qos=%s\n"
"#SBATCH -p gpu\n"
"#SBATCH -C gpu=%s\n"
"#SBATCH --gres=gpu:%i\n") % (n_threads,
mem_limit, time_limit,
qos, gpu_type, n_gpus)
if mail_address is not None:
slurm_template += ("#SBATCH --mail-type=FAIL,BEGIN,END\n"
"#SBATCH --mail-user=%s") % mail_address
slurm_template += ("\n"
"module purge \n"
"module load GCC \n"
"source activate %s\n"
"\n"
"python %s $@ \n") % (env_name, script)
with open(out_path, 'w') as f:
f.write(slurm_template)
def submit_slurm(script, input_, n_threads=7, n_gpus=1,
gpu_type='2080Ti|1080Ti', mem_limit='64G',
time_limit=TWO_DAYS, qos='normal',
env_name=None, mail_address=None):
""" Submit python script that needs gpus with given inputs on a slurm node.
"""
tmp_folder = os.path.expanduser('~/.deep-cell/slurm')
os.makedirs(tmp_folder, exist_ok=True)
print("Submitting training script %s to cluster" % script)
print("with arguments %s" % " ".join(input_))
script_name = os.path.split(script)[1]
dt = datetime.now().strftime('%Y_%m_%d_%M')
tmp_name = os.path.splitext(script_name)[0] + dt
batch_script = os.path.join(tmp_folder, '%s.sh' % tmp_name)
log = os.path.join(tmp_folder, '%s.log' % tmp_name)
err = os.path.join(tmp_folder, '%s.err' % tmp_name)
if env_name is None:
env_name = os.environ.get('CONDA_DEFAULT_ENV', None)
if env_name is None:
raise RuntimeError("Could not find conda")
print("Batch script saved at", batch_script)
print("Log will be written to %s, error log to %s" % (log, err))
write_slurm_template(script, batch_script, env_name,
int(n_threads), gpu_type, int(n_gpus),
mem_limit, int(time_limit), qos, mail_address)
cmd = ['sbatch', '-o', log, '-e', err, '-J', script_name, batch_script]
cmd.extend(input_)
subprocess.run(cmd)
def scrape_kwargs(input_):
params = inspect.signature(submit_slurm).parameters
kwarg_names = [name for name in params
if params[name].default != inspect._empty]
kwarg_positions = [i for i, inp in enumerate(input_)
if inp in kwarg_names]
kwargs = {input_[i]: input_[i + 1] for i in kwarg_positions}
kwarg_positions += [i + 1 for i in kwarg_positions]
input_ = [inp for i, inp in enumerate(input_) if i not in kwarg_positions]
return input_, kwargs
def main():
script = os.path.realpath(os.path.abspath(sys.argv[1]))
input_ = sys.argv[2:]
# scrape the additional arguments (n_threads, mem_limit, etc. from the input)
input_, kwargs = scrape_kwargs(input_)
submit_slurm(script, input_, **kwargs)
if __name__ == '__main__':
main()
import argparse
import os
from glob import glob
import imageio
import napari
def view_data(root, image_folder, labels_folder, prediction_folder,
ext, prediction_is_labels):
image_folder = os.path.join(root, image_folder)
assert os.path.exists(image_folder), f"Could not find {image_folder}"
if labels_folder is not None:
labels_folder = os.path.join(root, labels_folder)
assert os.path.exists(labels_folder), f"Could not find {labels_folder}"
if prediction_folder is not None:
prediction_folder = os.path.join(root, prediction_folder)
assert os.path.exists(prediction_folder), f"Could not find {prediction_folder}"
files = glob(os.path.join(image_folder, f"*{ext}"))
files.sort()
def _load(path):
try:
im = imageio.imread(path)
name = os.path.split(path)[1]
except Exception as e:
print(f"Could not open {path}")
print(f"Failed with {e}")
im, name = None, None
return im, name
# TODO instead of looping over images load them in napari with selection gui
for ff in files:
im, name = _load(ff)
if im is None:
continue
if labels_folder is not None:
label_file = os.path.join(labels_folder, name)
labels, _ = _load(label_file)
else:
labels = None
if prediction_folder is not None:
pred_file = os.path.join(prediction_folder, name)
prediction, _ = _load(pred_file)
else:
prediction = None
with napari.gui_qt():
viewer = napari.Viewer(title=name)
viewer.add_image(im)
if labels is not None:
viewer.add_labels(labels)
if prediction is not None:
if prediction_is_labels:
viewer.add_labels(prediction)
else:
viewer.add_image(prediction)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('root')
parser.add_argument('--image_folder', type=str, default='images')
parser.add_argument('--labels_folder', type=str, default=None)
parser.add_argument('--prediction_folder', type=str, default=None)
parser.add_argument('--prediction_is_labels', type=int, default=1)
parser.add_argument('--ext', type=str, default='.tif')
args = parser.parse_args()
view_data(args.root, args.image_folder, args.labels_folder, args.prediction_folder,
args.ext, bool(args.prediction_is_labels))
if __name__ == '__main__':
main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment