Update scripts for intensity correction and downscaling

analysis/correct_intensities.py

@@ -4,7 +4,10 @@ import numpy as np
 import h5py
 import z5py
 import vigra
+
+from scipy.ndimage.morphology import binary_dilation
 from scripts.transformation import intensity_correction
+from pybdv import make_bdv
 
 
 def combine_mask():
@@ -18,6 +21,7 @@ def combine_mask():
     with h5py.File(mask_path1, 'r') as f:
         key = 't00000/s00/0/cells'
         mask1 = f[key][:].astype('bool')
+        mask1 = binary_dilation(mask1, iterations=4)
     print("Load resin mask ..")
     with h5py.File(mask_path2, 'r') as f:
         key = 't00000/s00/1/cells'
@@ -27,13 +31,13 @@ def combine_mask():
     mask2 = vigra.sampling.resize(mask2.astype('float32'), mask1.shape, order=0).astype('bool')
     mask = np.logical_or(mask1, mask2).astype('uint8')
 
-    out_path = 'mask.n5'
-    out_key = 'data'
-    with z5py.File(out_path) as f:
-        f.create_dataset(out_key, data=mask, compression='gzip', n_threads=8)
+    res = [.4, .32, .32]
+    ds_factors = [[2, 2, 2], [2, 2, 2], [2, 2, 2]]
+    make_bdv(mask, 'mask.h5', ds_factors,
+             resolution=res, unit='micrometer')
 
 
-def correct_intensities(target='slurm', max_jobs=250):
+def correct_intensities_test(target='local', max_jobs=32):
     raw_path = '../../EM-Prospr/em-raw-samplexy.h5'
     tmp_folder = './tmp_intensity_correction'
 
@@ -51,6 +55,26 @@ def correct_intensities(target='slurm', max_jobs=250):
                          target=target, max_jobs=max_jobs)
 
 
+def correct_intensities(target='slurm', max_jobs=250):
+    raw_path = '../data/rawdata/sbem-6dpf-1-whole-raw.h5'
+    tmp_folder = './tmp_intensity_correction'
+
+    mask_path = 'mask.h5'
+    mask_key = 't00000/s00/0/cells'
+
+    out_path = 'em-raw-wholecorrected.h5'
+
+    # trafo = './new_vals.csv'
+    trafo = './new_vals.json'
+    tmp_path = '/g/kreshuk/pape/Work/platy_tmp.n5'
+
+    resolution = [0.025, 0.01, 0.01]
+    intensity_correction(raw_path, out_path, mask_path, mask_key,
+                         trafo, tmp_folder, resolution,
+                         target=target, max_jobs=max_jobs,
+                         tmp_path=tmp_path)
+
+
 if __name__ == '__main__':
     # combine_mask()
-    correct_intensities('local', 32)
+    correct_intensities('local', 64)

scripts/export/export_segmentation.py

@@ -24,7 +24,7 @@ def get_scale_factors(paintera_path, paintera_key):
         rel_factor = [int(sf // prev) for sf, prev in zip(factor, scale_factors[-1])]
         scale_factors.append(factor)
         rel_scales.append(rel_factor[::-1])
-    return rel_scales
+    return rel_scales[1:]
 
 
 def downscale(path, in_key, out_key,

scripts/transformation/intensity_correction.py

@@ -42,7 +42,7 @@ def downsample(ref_path, in_path, in_key, out_path, resolution,
 
             s0 = ds0.shape
             s1 = ds1.shape
-            factor = [sh0 // sh1 for sh0, sh1 in zip(s0, s1)]
+            factor = [int(round(float(sh0) / sh1, 0)) for sh0, sh1 in zip(s0, s1)]
 
             sample_factors.append(factor)
         assert len(sample_factors) == len(levels) - 1
@@ -50,7 +50,7 @@ def downsample(ref_path, in_path, in_key, out_path, resolution,
     config_dir = os.path.join(tmp_folder, 'configs')
     task = DownscalingWorkflow
     config = task.get_config()['downscaling']
-    config.update({'library': 'skimage'})
+    config.update({'library': 'skimage', 'time_limit': 240, 'mem_limit': 4})
     with open(os.path.join(config_dir, 'downscaling.config'), 'w') as f:
         json.dump(config, f)
 
@@ -85,7 +85,7 @@ def downsample(ref_path, in_path, in_key, out_path, resolution,
 
 def intensity_correction(in_path, out_path, mask_path, mask_key,
                          trafo_path, tmp_folder, resolution,
-                         target='slurm', max_jobs=250):
+                         target='slurm', max_jobs=250, tmp_path=None):
     trafo_ext = os.path.splitext(trafo_path)[1]
     if trafo_ext == '.csv':
         trafo_path = csv_to_json(trafo_path)
@@ -98,10 +98,16 @@ def intensity_correction(in_path, out_path, mask_path, mask_key,
     config_dir = os.path.join(tmp_folder, 'configs')
     write_default_global_config(config_dir)
 
-    tmp_path = os.path.join(tmp_folder, 'data.n5')
+    if tmp_path is None:
+        tmp_path = os.path.join(tmp_folder, 'data.n5')
     tmp_key = 'data'
 
     task = LinearTransformationWorkflow
+    conf = task.get_config()['linear']
+    conf.update({'time_limit': 600, 'mem_limit': 4})
+    with open(os.path.join(config_dir, 'linear.config'), 'w') as f:
+        json.dump(conf, f)
+
     t = task(tmp_folder=tmp_folder, config_dir=config_dir,
              target=target, max_jobs=max_jobs,
              input_path=in_path, input_key=key,