From c943b07fe6881ca4147ddac2e4c5bc997bb55cde Mon Sep 17 00:00:00 2001
From: Constantin Pape <constantin.pape@iwr.uni-heidelberg.de>
Date: Thu, 15 Aug 2019 10:51:55 +0200
Subject: [PATCH] add_image is working and added all existing image data
---
add_existing_data.py | 19 +++++---
data/images.json | 1 +
data/sources.json | 4 +-
scripts/__init__.py | 1 -
scripts/files/__init__.py | 10 +++--
scripts/files/copy_helper.py | 2 +-
scripts/{ => files}/sources.py | 80 +++++++++++++++++++---------------
7 files changed, 69 insertions(+), 48 deletions(-)
create mode 100644 data/images.json
rename scripts/{ => files}/sources.py (82%)
diff --git a/add_existing_data.py b/add_existing_data.py
index 9177985..f518779 100644
--- a/add_existing_data.py
+++ b/add_existing_data.py
@@ -1,6 +1,8 @@
+#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
+
import os
from glob import glob
-from scripts.sources import add_source, add_image, add_segmentation
+from scripts.files import add_source, add_image, add_segmentation
def add_sources():
@@ -12,12 +14,19 @@ def add_sources():
def add_images():
base_folder = './data/0.2.1/images'
+
+ # add sbem raw data
+ sbem_prefix = 'sbem-6dpf-1-whole'
+ sbem_raw = './data/0.2.1/images/sbem-6dpf-1-whole-raw.xml'
+ name = 'raw'
+ add_image(sbem_raw, sbem_prefix, name, copy_data=False)
+
# add all prospr images
prospr_prefix = 'prospr-6dpf-1-whole'
- prospr_ims = glob.glob(os.path.join(base_folder, 'prospr-6dpf-1-whole-*'))
+ prospr_ims = glob(os.path.join(base_folder, 'prospr-6dpf-1-whole-*'))
for impath in prospr_ims:
name = os.path.split(impath)[1]
- name, ext = os.path.splitext()
+ name, ext = os.path.splitext(name)
if ext != '.xml':
continue
name = name[(len(prospr_prefix) + 1):]
@@ -32,8 +41,8 @@ def add_existing_data():
""" Add existing data to the json files that keep track of
sources, image data and segmentations.
"""
- add_sources()
- add_images()
+ # add_sources()
+ # add_images()
add_segmentations()
diff --git a/data/images.json b/data/images.json
new file mode 100644
index 0000000..0c5ba0f
--- /dev/null
+++ b/data/images.json
@@ -0,0 +1 @@
+["sbem-6dpf-1-whole-raw", "prospr-6dpf-1-whole-AChE-MED", "prospr-6dpf-1-whole-ANPRA-MED", "prospr-6dpf-1-whole-ANPRB-MED", "prospr-6dpf-1-whole-AP2-MED", "prospr-6dpf-1-whole-ASCb-MED", "prospr-6dpf-1-whole-AcTub-MED", "prospr-6dpf-1-whole-AllCR1-MED", "prospr-6dpf-1-whole-AllCR2-MED", "prospr-6dpf-1-whole-Arx-MED", "prospr-6dpf-1-whole-Ash-MED", "prospr-6dpf-1-whole-BAIAP-MED", "prospr-6dpf-1-whole-BarH1-MED", "prospr-6dpf-1-whole-Beta3-MED", "prospr-6dpf-1-whole-Bmpr1-MED", "prospr-6dpf-1-whole-Brn3a-MED", "prospr-6dpf-1-whole-COE-MED", "prospr-6dpf-1-whole-Cal2-MED", "prospr-6dpf-1-whole-Calmodulin-MED", "prospr-6dpf-1-whole-ChAT-MED", "prospr-6dpf-1-whole-Chx10-MED", "prospr-6dpf-1-whole-Dach-MED", "prospr-6dpf-1-whole-DbhLike-MED", "prospr-6dpf-1-whole-DopaRD2-MED", "prospr-6dpf-1-whole-ENR1-MED", "prospr-6dpf-1-whole-ENR10-MED", "prospr-6dpf-1-whole-ENR12-MED", "prospr-6dpf-1-whole-ENR13-MED", "prospr-6dpf-1-whole-ENR16-MED", "prospr-6dpf-1-whole-ENR19-MED", "prospr-6dpf-1-whole-ENR2-MED", "prospr-6dpf-1-whole-ENR20-MED", "prospr-6dpf-1-whole-ENR22-MED", "prospr-6dpf-1-whole-ENR25-MED", "prospr-6dpf-1-whole-ENR29-MED", "prospr-6dpf-1-whole-ENR3-MED", "prospr-6dpf-1-whole-ENR30-MED", "prospr-6dpf-1-whole-ENR31-MED", "prospr-6dpf-1-whole-ENR32-MED", "prospr-6dpf-1-whole-ENR34-MED", "prospr-6dpf-1-whole-ENR39-MED", "prospr-6dpf-1-whole-ENR4-MED", "prospr-6dpf-1-whole-ENR46-MED", "prospr-6dpf-1-whole-ENR6-MED", "prospr-6dpf-1-whole-ENR54-MED", "prospr-6dpf-1-whole-ENR57-MED", "prospr-6dpf-1-whole-ENR62-MED", "prospr-6dpf-1-whole-ENR64-MED", "prospr-6dpf-1-whole-ENR69-MED", "prospr-6dpf-1-whole-ENR71-MED", "prospr-6dpf-1-whole-ENR8-MED", "prospr-6dpf-1-whole-ENR9-MED", "prospr-6dpf-1-whole-ElaV-MED", "prospr-6dpf-1-whole-FOR-MED", "prospr-6dpf-1-whole-FVRI-MED", "prospr-6dpf-1-whole-FoxD3-MED", "prospr-6dpf-1-whole-FoxN4-MED", "prospr-6dpf-1-whole-GAD-MED", "prospr-6dpf-1-whole-GCB-MED", "prospr-6dpf-1-whole-GCD-MED", "prospr-6dpf-1-whole-GPCR2-MED", "prospr-6dpf-1-whole-GPCR203-MED", "prospr-6dpf-1-whole-GPCR209-MED", "prospr-6dpf-1-whole-GPCR210-MED", "prospr-6dpf-1-whole-GPCR32-MED", "prospr-6dpf-1-whole-GPPCR21-MED", "prospr-6dpf-1-whole-Gata123-MED", "prospr-6dpf-1-whole-Glt1-MED", "prospr-6dpf-1-whole-GlyT-MED", "prospr-6dpf-1-whole-GnRH-MED", "prospr-6dpf-1-whole-GnrhR-MED", "prospr-6dpf-1-whole-Grm7-MED", "prospr-6dpf-1-whole-Gucy-MED", "prospr-6dpf-1-whole-HCN1-MED", "prospr-6dpf-1-whole-HNF6-MED", "prospr-6dpf-1-whole-Hand-MED", "prospr-6dpf-1-whole-Hb9-MED", "prospr-6dpf-1-whole-Hox5-MED", "prospr-6dpf-1-whole-Hox7-MED", "prospr-6dpf-1-whole-Hr38-MED", "prospr-6dpf-1-whole-Isl-MED", "prospr-6dpf-1-whole-Kv33b-MED", "prospr-6dpf-1-whole-Kv33e-MED", "prospr-6dpf-1-whole-Lhx15-MED", "prospr-6dpf-1-whole-Lhx2-MED", "prospr-6dpf-1-whole-Lhx3-MED", "prospr-6dpf-1-whole-Lmo4-MED", "prospr-6dpf-1-whole-Lmx1-MED", "prospr-6dpf-1-whole-Loc28250-MED", "prospr-6dpf-1-whole-Loc5285ct-MED", "prospr-6dpf-1-whole-Loc77859-MED", "prospr-6dpf-1-whole-Loc8913nt-MED", "prospr-6dpf-1-whole-MHCL4-MED", "prospr-6dpf-1-whole-MRLC2-MED", "prospr-6dpf-1-whole-Maf-MED", "prospr-6dpf-1-whole-Mitf-MED", "prospr-6dpf-1-whole-Munc22-MED", "prospr-6dpf-1-whole-NMDAR-MED", "prospr-6dpf-1-whole-NOV1-MED", "prospr-6dpf-1-whole-NOV15-MED", "prospr-6dpf-1-whole-NOV18-MED", "prospr-6dpf-1-whole-NOV2-MED", "prospr-6dpf-1-whole-NOV29-MED", "prospr-6dpf-1-whole-NOV45-MED", "prospr-6dpf-1-whole-NOV50-MED", "prospr-6dpf-1-whole-NOV52-MED", "prospr-6dpf-1-whole-NOV6-MED", "prospr-6dpf-1-whole-NPR1a-MED", "prospr-6dpf-1-whole-Netrin-MED", "prospr-6dpf-1-whole-NeuroD-MED", "prospr-6dpf-1-whole-Ngb-MED", "prospr-6dpf-1-whole-Olig-MED", "prospr-6dpf-1-whole-P56-MED", "prospr-6dpf-1-whole-PCDH15-MED", "prospr-6dpf-1-whole-PDF-MED", "prospr-6dpf-1-whole-PLCg-MED", "prospr-6dpf-1-whole-Pax258-MED", "prospr-6dpf-1-whole-Pax6-MED", "prospr-6dpf-1-whole-Pde9-MED", "prospr-6dpf-1-whole-Pea3-MED", "prospr-6dpf-1-whole-Phc2-MED", "prospr-6dpf-1-whole-Phox2b-MED", "prospr-6dpf-1-whole-Pikachu-MED", "prospr-6dpf-1-whole-PitxB-MED", "prospr-6dpf-1-whole-Prdm8-MED", "prospr-6dpf-1-whole-Prox-MED", "prospr-6dpf-1-whole-Ptf1-MED", "prospr-6dpf-1-whole-Pty2-MED", "prospr-6dpf-1-whole-RIMS-MED", "prospr-6dpf-1-whole-Rab3-MED", "prospr-6dpf-1-whole-Robo-MED", "prospr-6dpf-1-whole-Scn8aa-MED", "prospr-6dpf-1-whole-Sema2-MED", "prospr-6dpf-1-whole-Sim1-MED", "prospr-6dpf-1-whole-Slit-MED", "prospr-6dpf-1-whole-Sox2-MED", "prospr-6dpf-1-whole-Sox4-MED", "prospr-6dpf-1-whole-SoxB12-MED", "prospr-6dpf-1-whole-SoxB2-MED", "prospr-6dpf-1-whole-Sp8-MED", "prospr-6dpf-1-whole-Syt-MED", "prospr-6dpf-1-whole-Syt12-MED", "prospr-6dpf-1-whole-Syt7-MED", "prospr-6dpf-1-whole-Syta-MED", "prospr-6dpf-1-whole-TH-MED", "prospr-6dpf-1-whole-TIMP-MED", "prospr-6dpf-1-whole-TRPV4-MED", "prospr-6dpf-1-whole-TRPV5-MED", "prospr-6dpf-1-whole-Tal-MED", "prospr-6dpf-1-whole-Tbh-MED", "prospr-6dpf-1-whole-Tbx20-MED", "prospr-6dpf-1-whole-TrpH-MED", "prospr-6dpf-1-whole-TrpV-MED", "prospr-6dpf-1-whole-Twist-MED", "prospr-6dpf-1-whole-UNCS-MED", "prospr-6dpf-1-whole-VAChT-MED", "prospr-6dpf-1-whole-VAT1L-MED", "prospr-6dpf-1-whole-VGluT-MED", "prospr-6dpf-1-whole-VegfR-MED", "prospr-6dpf-1-whole-Wnt16-MED", "prospr-6dpf-1-whole-Wnt5-MED", "prospr-6dpf-1-whole-ascI-MED", "prospr-6dpf-1-whole-ascII-MED", "prospr-6dpf-1-whole-asicalpha-MED", "prospr-6dpf-1-whole-bsx-MED", "prospr-6dpf-1-whole-catL-MED", "prospr-6dpf-1-whole-cnga-MED", "prospr-6dpf-1-whole-dbx1-MED", "prospr-6dpf-1-whole-delta-MED", "prospr-6dpf-1-whole-dlx-MED", "prospr-6dpf-1-whole-edu3to4at6dpf-MED", "prospr-6dpf-1-whole-edu4to5at6dpf-MED", "prospr-6dpf-1-whole-edu5to6at6dpf-MED", "prospr-6dpf-1-whole-eve-MED", "prospr-6dpf-1-whole-eya-MED", "prospr-6dpf-1-whole-fezf-MED", "prospr-6dpf-1-whole-gpb-MED", "prospr-6dpf-1-whole-gsx-MED", "prospr-6dpf-1-whole-hox1-MED", "prospr-6dpf-1-whole-irx-MED", "prospr-6dpf-1-whole-irx2546-MED", "prospr-6dpf-1-whole-lhx6-MED", "prospr-6dpf-1-whole-lbx1b-MED", "prospr-6dpf-1-whole-msx-MED", "prospr-6dpf-1-whole-mecom-MED", "prospr-6dpf-1-whole-neog-MED", "prospr-6dpf-1-whole-muncB-MED", "prospr-6dpf-1-whole-nAchR-MED", "prospr-6dpf-1-whole-ngn-MED", "prospr-6dpf-1-whole-nk21-MED", "prospr-6dpf-1-whole-nk22-MED", "prospr-6dpf-1-whole-nk6-MED", "prospr-6dpf-1-whole-otp-MED", "prospr-6dpf-1-whole-ntrps-MED", "prospr-6dpf-1-whole-rx-MED", "prospr-6dpf-1-whole-paraxis-MED", "prospr-6dpf-1-whole-sert-MED", "prospr-6dpf-1-whole-six4-MED", "prospr-6dpf-1-whole-segmented-CrypticSegment", "prospr-6dpf-1-whole-tlx-MED", "prospr-6dpf-1-whole-trk-MED", "prospr-6dpf-1-whole-uncx-MED", "prospr-6dpf-1-whole-segmented-Head", "prospr-6dpf-1-whole-segmented-PNS", "prospr-6dpf-1-whole-segmented-Pygidium", "prospr-6dpf-1-whole-segmented-RestOfAnimal", "prospr-6dpf-1-whole-segmented-Stomodeum", "prospr-6dpf-1-whole-segmented-VNC", "prospr-6dpf-1-whole-six12-MED"]
\ No newline at end of file
diff --git a/data/sources.json b/data/sources.json
index 154ea26..06d143a 100644
--- a/data/sources.json
+++ b/data/sources.json
@@ -1,3 +1 @@
-[{"modality": "sbem", "stage": "6dpf", "id": "1", "region": "whole"},
- {"modality": "prospr", "stage": "6dpf", "id": "1", "region": "whole"},
- {"modality": "fibsem", "stage": "6dpf", "id": "1", "region": "parapod"}]
+[{"modality": "sbem", "stage": "6dpf", "id": "1", "region": "whole"}, {"modality": "prospr", "stage": "6dpf", "id": "1", "region": "whole"}]
\ No newline at end of file
diff --git a/scripts/__init__.py b/scripts/__init__.py
index fdf8054..e69de29 100644
--- a/scripts/__init__.py
+++ b/scripts/__init__.py
@@ -1 +0,0 @@
-from .sources import get_name_prefixes
diff --git a/scripts/files/__init__.py b/scripts/files/__init__.py
index 390c39c..c587313 100644
--- a/scripts/files/__init__.py
+++ b/scripts/files/__init__.py
@@ -1,6 +1,8 @@
from .copy_helper import copy_tables, copy_segmentation, copy_image_data, copy_misc_data
-from .folders import make_folder_structure
-from .xml_utils import get_h5_path_from_xml, copy_xml_with_abspath, write_simple_xml
-from .xml_utils import copy_xml_with_newpath
+
from .bdv_server import make_bdv_server_file
-from .checks import check_bdv
+
+from .folders import make_folder_structure
+
+from .sources import add_image, add_segmentation, add_source
+from .sources import get_image_names, get_segmentation_names, get_source_names
diff --git a/scripts/files/copy_helper.py b/scripts/files/copy_helper.py
index 9e4d232..9fac746 100644
--- a/scripts/files/copy_helper.py
+++ b/scripts/files/copy_helper.py
@@ -1,7 +1,7 @@
import os
import shutil
from .xml_utils import copy_xml_with_newpath, get_h5_path_from_xml
-from ..sources import get_image_names, RAW_FOLDER
+from .sources import get_image_names, RAW_FOLDER
def copy_file(xml_in, xml_out):
diff --git a/scripts/sources.py b/scripts/files/sources.py
similarity index 82%
rename from scripts/sources.py
rename to scripts/files/sources.py
index 6b113cc..a52b068 100644
--- a/scripts/sources.py
+++ b/scripts/files/sources.py
@@ -1,7 +1,8 @@
import json
import os
from shutil import copyfile
-from.files import check_bdv, get_h5_path_from_xml, copy_xml_with_newpath
+from .checks import check_bdv
+from .xml_utils import get_h5_path_from_xml, copy_xml_with_newpath
RAW_FOLDER = 'data/rawdata'
SOURCE_FILE = 'data/sources.json'
@@ -15,31 +16,13 @@ def get_sources():
See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
for the source naming conventions.
"""
+ if not os.path.exists(SOURCE_FILE):
+ return []
with open(SOURCE_FILE) as f:
sources = json.load(f)
return sources
-def add_source(modality, stage, id=1, region='whole'):
- """ Add a new data source
-
- See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
- for the source naming conventions.
- """
- if not isinstance(modality, str):
- raise ValueError("Expected modality to be a string, not %s" % type(modality))
- if not isinstance(stage, str):
- raise ValueError("Expected stage to be a string, not %s" % type(id))
- if not isinstance(id, int):
- raise ValueError("Expected id to be an integer, not %s" % type(id))
- if not isinstance(region, str):
- raise ValueError("Expected region to be a string, not %s" % type(id))
- sources = get_sources()
- sources.append({'modality': modality, 'stage': stage, 'id': str(id), 'region': region})
- with open(SOURCE_FILE, 'w') as f:
- json.dump(sources, f)
-
-
def source_to_prefix(source):
return '%s-%s-%s-%s' % (source['modality'],
source['stage'],
@@ -47,7 +30,7 @@ def source_to_prefix(source):
source['region'])
-def get_name_prefixes():
+def get_source_names():
""" Get the name prefixes corresponding to all sources.
"""
sources = get_sources()
@@ -56,12 +39,47 @@ def get_name_prefixes():
def get_image_names():
+ if not os.path.exists(IMAGE_FILE):
+ return []
with open(IMAGE_FILE) as f:
names = json.load(f)
return names
-def add_image(input_path, source_prefix, name, copy_data=True):
+def get_segmentation_names():
+ if not os.path.exists(SEGMENTATION_FILE):
+ return []
+ with open(SEGMENTATION_FILE) as f:
+ names = list(json.load(f).keys())
+ return names
+
+
+def add_source(modality, stage, id=1, region='whole'):
+ """ Add a new data source
+
+ See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
+ for the source naming conventions.
+ """
+ if not isinstance(modality, str):
+ raise ValueError("Expected modality to be a string, not %s" % type(modality))
+ if not isinstance(stage, str):
+ raise ValueError("Expected stage to be a string, not %s" % type(id))
+ if not isinstance(id, int):
+ raise ValueError("Expected id to be an integer, not %s" % type(id))
+ if not isinstance(region, str):
+ raise ValueError("Expected region to be a string, not %s" % type(id))
+ sources = get_sources()
+ source = {'modality': modality, 'stage': stage, 'id': str(id), 'region': region}
+
+ if source in sources:
+ raise RuntimeError("Source is already present")
+
+ sources.append(source)
+ with open(SOURCE_FILE, 'w') as f:
+ json.dump(sources, f)
+
+
+def add_image(input_path, source_name, name, copy_data=True):
""" Add image volume to the platy browser data.
Parameter:
@@ -73,14 +91,14 @@ def add_image(input_path, source_prefix, name, copy_data=True):
unless adding an image that is already in the rawdata folder. (default: True)
"""
# validate the inputs
- prefixes = get_name_prefixes()
- if source_prefix not in prefixes:
- raise ValueError("""Source prefix %s is not in the current sources.
- Use 'add_source' to add a new source.""" % source_prefix)
+ source_names = get_source_names()
+ if source_name not in source_names:
+ raise ValueError("""Source %s is not in the current sources.
+ Use 'add_source' to add a new source.""" % source_name)
is_bdv = check_bdv(input_path)
if not is_bdv:
raise ValueError("Expect input to be in bdv format")
- output_name = '%s-%s' % (source_prefix, name)
+ output_name = '%s-%s' % (source_name, name)
names = get_image_names()
if output_name in names:
raise ValueError("Name %s is already taken" % output_name)
@@ -104,12 +122,6 @@ def add_image(input_path, source_prefix, name, copy_data=True):
json.dump(names, f)
-def get_segmentation_names():
- with open(SEGMENTATION_FILE) as f:
- names = list(json.load(f).keys())
- return names
-
-
def add_segmentation(source_prefix, name):
""" Add segmentation volume to the platy browser data.
"""
--
GitLab