From bad6ac30b05ae3a349bce337efb284b4e6e78e8e Mon Sep 17 00:00:00 2001
From: Constantin Pape <constantin.pape@iwr.uni-heidelberg.de>
Date: Thu, 6 Feb 2020 13:29:05 +0100
Subject: [PATCH] Change file names in data/rawdata
---
.gitignore | 1 +
...sbem-6dpf-1-whole-segmented-chromatin.xml} | 78 +++++++++----------
...m-6dpf-1-whole-segmented-extrapolated.xml} | 2 +-
...> sbem-6dpf-1-whole-segmented-ganglia.xml} | 2 +-
...=> sbem-6dpf-1-whole-segmented-inside.xml} | 2 +-
... sbem-6dpf-1-whole-segmented-neuropil.xml} | 2 +-
...> sbem-6dpf-1-whole-segmented-outside.xml} | 2 +-
... => sbem-6dpf-1-whole-segmented-resin.xml} | 2 +-
...=> sbem-6dpf-1-whole-segmented-tissue.xml} | 2 +-
.../default.csv | 0
.../default.csv | 0
.../default.csv | 0
mmpb/files/migration.py | 43 +++++-----
mmpb/files/name_lookup.py | 12 ++-
14 files changed, 76 insertions(+), 72 deletions(-)
rename data/rawdata/{sbem-6dpf-1-whole-segmented-chromatin-labels.xml => sbem-6dpf-1-whole-segmented-chromatin.xml} (94%)
rename data/rawdata/{sbem-6dpf-1-whole-mask-extrapolated.xml => sbem-6dpf-1-whole-segmented-extrapolated.xml} (92%)
rename data/rawdata/{sbem-6dpf-1-whole-segmented-ganglia-labels.xml => sbem-6dpf-1-whole-segmented-ganglia.xml} (92%)
rename data/rawdata/{sbem-6dpf-1-whole-mask-inside.xml => sbem-6dpf-1-whole-segmented-inside.xml} (93%)
rename data/rawdata/{sbem-6dpf-1-whole-segmented-ariande-neuropil.xml => sbem-6dpf-1-whole-segmented-neuropil.xml} (92%)
rename data/rawdata/{sbem-6dpf-1-whole-mask-outside.xml => sbem-6dpf-1-whole-segmented-outside.xml} (93%)
rename data/rawdata/{sbem-6dpf-1-whole-mask-resin.xml => sbem-6dpf-1-whole-segmented-resin.xml} (93%)
rename data/rawdata/{sbem-6dpf-1-whole-segmented-tissue-labels.xml => sbem-6dpf-1-whole-segmented-tissue.xml} (92%)
rename data/rawdata/tables/{sbem-6dpf-1-whole-segmented-chromatin-labels => sbem-6dpf-1-whole-segmented-chromatin}/default.csv (100%)
rename data/rawdata/tables/{sbem-6dpf-1-whole-segmented-ganglia-labels => sbem-6dpf-1-whole-segmented-ganglia}/default.csv (100%)
rename data/rawdata/tables/{sbem-6dpf-1-whole-segmented-tissue-labels => sbem-6dpf-1-whole-segmented-tissue}/default.csv (100%)
diff --git a/.gitignore b/.gitignore
index e14350c..61c9b32 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,4 @@ software
*.zip
*.n5
data/bkp
+data/rawdata/evaluation
diff --git a/data/rawdata/sbem-6dpf-1-whole-segmented-chromatin-labels.xml b/data/rawdata/sbem-6dpf-1-whole-segmented-chromatin.xml
similarity index 94%
rename from data/rawdata/sbem-6dpf-1-whole-segmented-chromatin-labels.xml
rename to data/rawdata/sbem-6dpf-1-whole-segmented-chromatin.xml
index 07cf101..e8f1b31 100644
--- a/data/rawdata/sbem-6dpf-1-whole-segmented-chromatin-labels.xml
+++ b/data/rawdata/sbem-6dpf-1-whole-segmented-chromatin.xml
@@ -1,39 +1,39 @@
-<SpimData version="0.2">
- <BasePath type="relative">.</BasePath>
- <SequenceDescription>
- <ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-segmented-chromatin-labels.h5</hdf5>
- </ImageLoader>
- <ViewSetups>
- <Attributes name="channel">
- <Channel>
- <id>0</id>
- <name>0</name>
- </Channel>
- </Attributes>
- <ViewSetup>
- <id>0</id>
- <name>Setup0</name>
- <size>13750 12958 11416</size>
- <voxelSize>
- <unit>micrometer</unit>
- <size>0.02 0.02 0.025</size>
- </voxelSize>
- <attributes>
- <channel>0</channel>
- </attributes>
- </ViewSetup>
- </ViewSetups>
- <Timepoints type="range">
- <first>0</first>
- <last>0</last>
- </Timepoints>
- </SequenceDescription>
- <ViewRegistrations>
- <ViewRegistration setup="0" timepoint="0">
- <ViewTransform type="affine">
- <affine>0.02 0.0 0.0 0.0 0.0 0.02 0.0 0.0 0.0 0.0 0.025 0.0</affine>
- </ViewTransform>
- </ViewRegistration>
- </ViewRegistrations>
-</SpimData>
+<SpimData version="0.2">
+ <BasePath type="relative">.</BasePath>
+ <SequenceDescription>
+ <ImageLoader format="bdv.hdf5">
+ <hdf5 type="relative">sbem-6dpf-1-whole-segmented-chromatin.h5</hdf5>
+ </ImageLoader>
+ <ViewSetups>
+ <Attributes name="channel">
+ <Channel>
+ <id>0</id>
+ <name>0</name>
+ </Channel>
+ </Attributes>
+ <ViewSetup>
+ <id>0</id>
+ <name>Setup0</name>
+ <size>13750 12958 11416</size>
+ <voxelSize>
+ <unit>micrometer</unit>
+ <size>0.02 0.02 0.025</size>
+ </voxelSize>
+ <attributes>
+ <channel>0</channel>
+ </attributes>
+ </ViewSetup>
+ </ViewSetups>
+ <Timepoints type="range">
+ <first>0</first>
+ <last>0</last>
+ </Timepoints>
+ </SequenceDescription>
+ <ViewRegistrations>
+ <ViewRegistration setup="0" timepoint="0">
+ <ViewTransform type="affine">
+ <affine>0.02 0.0 0.0 0.0 0.0 0.02 0.0 0.0 0.0 0.0 0.025 0.0</affine>
+ </ViewTransform>
+ </ViewRegistration>
+ </ViewRegistrations>
+</SpimData>
diff --git a/data/rawdata/sbem-6dpf-1-whole-mask-extrapolated.xml b/data/rawdata/sbem-6dpf-1-whole-segmented-extrapolated.xml
similarity index 92%
rename from data/rawdata/sbem-6dpf-1-whole-mask-extrapolated.xml
rename to data/rawdata/sbem-6dpf-1-whole-segmented-extrapolated.xml
index e4410ea..6499239 100644
--- a/data/rawdata/sbem-6dpf-1-whole-mask-extrapolated.xml
+++ b/data/rawdata/sbem-6dpf-1-whole-segmented-extrapolated.xml
@@ -2,7 +2,7 @@
<BasePath type="relative">.</BasePath>
<SequenceDescription>
<ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-mask-extrapolated.h5</hdf5>
+ <hdf5 type="relative">sbem-6dpf-1-whole-segmented-extrapolated.h5</hdf5>
</ImageLoader>
<ViewSetups>
<Attributes name="channel">
diff --git a/data/rawdata/sbem-6dpf-1-whole-segmented-ganglia-labels.xml b/data/rawdata/sbem-6dpf-1-whole-segmented-ganglia.xml
similarity index 92%
rename from data/rawdata/sbem-6dpf-1-whole-segmented-ganglia-labels.xml
rename to data/rawdata/sbem-6dpf-1-whole-segmented-ganglia.xml
index d61eae7..6e3a4e2 100644
--- a/data/rawdata/sbem-6dpf-1-whole-segmented-ganglia-labels.xml
+++ b/data/rawdata/sbem-6dpf-1-whole-segmented-ganglia.xml
@@ -2,7 +2,7 @@
<BasePath type="relative">.</BasePath>
<SequenceDescription>
<ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-segmented-ganglia-labels.h5</hdf5>
+ <hdf5 type="relative">sbem-6dpf-1-whole-segmented-ganglia.h5</hdf5>
</ImageLoader>
<ViewSetups>
<Attributes name="channel">
diff --git a/data/rawdata/sbem-6dpf-1-whole-mask-inside.xml b/data/rawdata/sbem-6dpf-1-whole-segmented-inside.xml
similarity index 93%
rename from data/rawdata/sbem-6dpf-1-whole-mask-inside.xml
rename to data/rawdata/sbem-6dpf-1-whole-segmented-inside.xml
index c101d64..0a0b464 100644
--- a/data/rawdata/sbem-6dpf-1-whole-mask-inside.xml
+++ b/data/rawdata/sbem-6dpf-1-whole-segmented-inside.xml
@@ -2,7 +2,7 @@
<BasePath type="relative">.</BasePath>
<SequenceDescription>
<ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-mask-inside.h5</hdf5>
+ <hdf5 type="relative">sbem-6dpf-1-whole-segmented-inside.h5</hdf5>
</ImageLoader>
<ViewSetups>
<Attributes name="channel">
diff --git a/data/rawdata/sbem-6dpf-1-whole-segmented-ariande-neuropil.xml b/data/rawdata/sbem-6dpf-1-whole-segmented-neuropil.xml
similarity index 92%
rename from data/rawdata/sbem-6dpf-1-whole-segmented-ariande-neuropil.xml
rename to data/rawdata/sbem-6dpf-1-whole-segmented-neuropil.xml
index 42cd10e..19c31d1 100644
--- a/data/rawdata/sbem-6dpf-1-whole-segmented-ariande-neuropil.xml
+++ b/data/rawdata/sbem-6dpf-1-whole-segmented-neuropil.xml
@@ -2,7 +2,7 @@
<BasePath type="relative">.</BasePath>
<SequenceDescription>
<ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-segmented-ariande-neuropil.h5</hdf5>
+ <hdf5 type="relative">sbem-6dpf-1-whole-segmented-neuropil.h5</hdf5>
</ImageLoader>
<ViewSetups>
<ViewSetup>
diff --git a/data/rawdata/sbem-6dpf-1-whole-mask-outside.xml b/data/rawdata/sbem-6dpf-1-whole-segmented-outside.xml
similarity index 93%
rename from data/rawdata/sbem-6dpf-1-whole-mask-outside.xml
rename to data/rawdata/sbem-6dpf-1-whole-segmented-outside.xml
index 6f197e0..4206415 100644
--- a/data/rawdata/sbem-6dpf-1-whole-mask-outside.xml
+++ b/data/rawdata/sbem-6dpf-1-whole-segmented-outside.xml
@@ -2,7 +2,7 @@
<BasePath type="relative">.</BasePath>
<SequenceDescription>
<ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-mask-outside.h5</hdf5>
+ <hdf5 type="relative">sbem-6dpf-1-whole-segmented-outside.h5</hdf5>
</ImageLoader>
<ViewSetups>
<Attributes name="channel">
diff --git a/data/rawdata/sbem-6dpf-1-whole-mask-resin.xml b/data/rawdata/sbem-6dpf-1-whole-segmented-resin.xml
similarity index 93%
rename from data/rawdata/sbem-6dpf-1-whole-mask-resin.xml
rename to data/rawdata/sbem-6dpf-1-whole-segmented-resin.xml
index f0f52cd..b8f678c 100644
--- a/data/rawdata/sbem-6dpf-1-whole-mask-resin.xml
+++ b/data/rawdata/sbem-6dpf-1-whole-segmented-resin.xml
@@ -2,7 +2,7 @@
<BasePath type="relative">.</BasePath>
<SequenceDescription>
<ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-mask-resin.h5</hdf5>
+ <hdf5 type="relative">sbem-6dpf-1-whole-segmented-resin.h5</hdf5>
</ImageLoader>
<ViewSetups>
<Attributes name="channel">
diff --git a/data/rawdata/sbem-6dpf-1-whole-segmented-tissue-labels.xml b/data/rawdata/sbem-6dpf-1-whole-segmented-tissue.xml
similarity index 92%
rename from data/rawdata/sbem-6dpf-1-whole-segmented-tissue-labels.xml
rename to data/rawdata/sbem-6dpf-1-whole-segmented-tissue.xml
index b29549d..163eddf 100644
--- a/data/rawdata/sbem-6dpf-1-whole-segmented-tissue-labels.xml
+++ b/data/rawdata/sbem-6dpf-1-whole-segmented-tissue.xml
@@ -2,7 +2,7 @@
<BasePath type="relative">.</BasePath>
<SequenceDescription>
<ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-segmented-tissue-labels.h5</hdf5>
+ <hdf5 type="relative">sbem-6dpf-1-whole-segmented-tissue.h5</hdf5>
</ImageLoader>
<ViewSetups>
<Attributes name="channel">
diff --git a/data/rawdata/tables/sbem-6dpf-1-whole-segmented-chromatin-labels/default.csv b/data/rawdata/tables/sbem-6dpf-1-whole-segmented-chromatin/default.csv
similarity index 100%
rename from data/rawdata/tables/sbem-6dpf-1-whole-segmented-chromatin-labels/default.csv
rename to data/rawdata/tables/sbem-6dpf-1-whole-segmented-chromatin/default.csv
diff --git a/data/rawdata/tables/sbem-6dpf-1-whole-segmented-ganglia-labels/default.csv b/data/rawdata/tables/sbem-6dpf-1-whole-segmented-ganglia/default.csv
similarity index 100%
rename from data/rawdata/tables/sbem-6dpf-1-whole-segmented-ganglia-labels/default.csv
rename to data/rawdata/tables/sbem-6dpf-1-whole-segmented-ganglia/default.csv
diff --git a/data/rawdata/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv b/data/rawdata/tables/sbem-6dpf-1-whole-segmented-tissue/default.csv
similarity index 100%
rename from data/rawdata/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv
rename to data/rawdata/tables/sbem-6dpf-1-whole-segmented-tissue/default.csv
diff --git a/mmpb/files/migration.py b/mmpb/files/migration.py
index 3efdb7f..75aa281 100644
--- a/mmpb/files/migration.py
+++ b/mmpb/files/migration.py
@@ -5,8 +5,7 @@ from glob import glob
from mmpb.files.name_lookup import look_up_filename, get_image_properties
from mmpb.files.xml_utils import get_h5_path_from_xml, copy_xml_with_newpath
-# ROOT = '/g/arendt/...'
-ROOT = '/home/pape/Work/my_projects/platy-browser-data/data'
+ROOT = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data'
DRY_RUN = True
@@ -184,6 +183,15 @@ def migrate_rawfolder():
# set path in xml
copy_xml_with_newpath(new_xml_path, new_xml_path, new_rel_data_path)
+ # rename the tables folder if it exists
+ table_folder = os.path.join(raw_folder, 'tables', name)
+ if os.path.exists(table_folder):
+ new_table_folder = os.path.join(raw_folder, 'tables', new_name)
+ if DRY_RUN:
+ print("Rename", table_folder, "to", new_table_folder)
+ else:
+ os.rename(table_folder, new_table_folder)
+
# iterate over all the xmls in this version, follow the links
# and replace h5 files with n5 (if necessary)
@@ -202,22 +210,22 @@ def remove_deprecated_data():
def remove_deprecated_seg(folder, pattern):
# remove xml for traces
- trace_files = glob(os.path.join(vfolder, 'segmentations', pattern))
- if len(trace_files) > 0:
- assert len(trace_files) == 1
+ files = glob(os.path.join(vfolder, 'segmentations', pattern))
+ if len(files) > 0:
+ assert len(files) == 1
if DRY_RUN:
- print("Remove", trace_files[0])
+ print("Remove", files[0])
else:
- os.remove(trace_files[0])
+ os.remove(files[0])
# remove tables for traces
- trace_files = glob(os.path.join(vfolder, 'tables', pattern))
- if len(trace_files) > 0:
- assert len(trace_files) == 1
+ files = glob(os.path.join(vfolder, 'tables', pattern))
+ if len(files) > 0:
+ assert len(files) == 1
if DRY_RUN:
- print("Remove", trace_files[0])
+ print("Remove", files[0])
else:
- shutil.rmtree(trace_files[0])
+ shutil.rmtree(files[0])
# remove xmls from the version folders
# (data from rawfolder should be backed up by hand!)
@@ -228,14 +236,11 @@ def remove_deprecated_data():
if __name__ == '__main__':
- # TODO before doing any migration:
- # - replace the chromatin segmentation and table
- # - check nephridia segmentation
-
- # remove the data we don't want to upload (yet):
- remove_deprecated_data()
+ # remove the data we don't want to upload (yet)
+ # remove_deprecated_data()
- # migrate_rawfolder()
+ # change names and xmls in the rawfolder
+ migrate_rawfolder()
# version = '0.0.0'
# migrate_version(version)
diff --git a/mmpb/files/name_lookup.py b/mmpb/files/name_lookup.py
index e9e91e2..0b5e9db 100644
--- a/mmpb/files/name_lookup.py
+++ b/mmpb/files/name_lookup.py
@@ -41,9 +41,7 @@ NEW_GENE_NAMES = {
"ENR69": "BCA1",
"ENR71": "Patched"
}
-
-# ROOT = '/g/arendt/...'
-ROOT = '/home/pape/Work/my_projects/platy-browser-data/data'
+ROOT = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data'
FILE_NAME_LUT = {}
IMAGE_PROPERTIES = {}
@@ -165,7 +163,7 @@ def get_image_properties(name):
if __name__ == '__main__':
- # x = json.dumps(FILE_NAME_LUT, sort_keys=True, indent=2)
- # print(x)
- with open('/home/pape/new_names.json', 'w') as f:
- json.dump(FILE_NAME_LUT, f, sort_keys=True, indent=2)
+ x = json.dumps(FILE_NAME_LUT, sort_keys=True, indent=2)
+ print(x)
+ # with open('/home/pape/new_names.json', 'w') as f:
+ # json.dump(FILE_NAME_LUT, f, sort_keys=True, indent=2)
--
GitLab