Skip to content
Snippets Groups Projects
Commit c943b07f authored by Constantin Pape's avatar Constantin Pape
Browse files

add_image is working and added all existing image data

parent 083d67bd
No related branches found
No related tags found
1 merge request!3Refactor sources
Hide whitespace changes
Inline Side-by-side
add_existing_data.py
+ 14
− 5
View file @ c943b07f
#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
import os import os
from glob import glob from glob import glob
from scripts.sources import add_source, add_image, add_segmentation from scripts.files import add_source, add_image, add_segmentation
def add_sources(): def add_sources():
...@@ -12,12 +14,19 @@ def add_sources(): ...@@ -12,12 +14,19 @@ def add_sources():
def add_images(): def add_images():
base_folder = './data/0.2.1/images' base_folder = './data/0.2.1/images'
# add sbem raw data
sbem_prefix = 'sbem-6dpf-1-whole'
sbem_raw = './data/0.2.1/images/sbem-6dpf-1-whole-raw.xml'
name = 'raw'
add_image(sbem_raw, sbem_prefix, name, copy_data=False)
# add all prospr images # add all prospr images
prospr_prefix = 'prospr-6dpf-1-whole' prospr_prefix = 'prospr-6dpf-1-whole'
prospr_ims = glob.glob(os.path.join(base_folder, 'prospr-6dpf-1-whole-*')) prospr_ims = glob(os.path.join(base_folder, 'prospr-6dpf-1-whole-*'))
for impath in prospr_ims: for impath in prospr_ims:
name = os.path.split(impath)[1] name = os.path.split(impath)[1]
name, ext = os.path.splitext() name, ext = os.path.splitext(name)
if ext != '.xml': if ext != '.xml':
continue continue
name = name[(len(prospr_prefix) + 1):] name = name[(len(prospr_prefix) + 1):]
...@@ -32,8 +41,8 @@ def add_existing_data(): ...@@ -32,8 +41,8 @@ def add_existing_data():
""" Add existing data to the json files that keep track of """ Add existing data to the json files that keep track of
sources, image data and segmentations. sources, image data and segmentations.
""" """
add_sources() # add_sources()
add_images() # add_images()
add_segmentations() add_segmentations()
......
data/images.json 0 → 100644
+ 1
− 0
View file @ c943b07f
["sbem-6dpf-1-whole-raw", "prospr-6dpf-1-whole-AChE-MED", "prospr-6dpf-1-whole-ANPRA-MED", "prospr-6dpf-1-whole-ANPRB-MED", "prospr-6dpf-1-whole-AP2-MED", "prospr-6dpf-1-whole-ASCb-MED", "prospr-6dpf-1-whole-AcTub-MED", "prospr-6dpf-1-whole-AllCR1-MED", "prospr-6dpf-1-whole-AllCR2-MED", "prospr-6dpf-1-whole-Arx-MED", "prospr-6dpf-1-whole-Ash-MED", "prospr-6dpf-1-whole-BAIAP-MED", "prospr-6dpf-1-whole-BarH1-MED", "prospr-6dpf-1-whole-Beta3-MED", "prospr-6dpf-1-whole-Bmpr1-MED", "prospr-6dpf-1-whole-Brn3a-MED", "prospr-6dpf-1-whole-COE-MED", "prospr-6dpf-1-whole-Cal2-MED", "prospr-6dpf-1-whole-Calmodulin-MED", "prospr-6dpf-1-whole-ChAT-MED", "prospr-6dpf-1-whole-Chx10-MED", "prospr-6dpf-1-whole-Dach-MED", "prospr-6dpf-1-whole-DbhLike-MED", "prospr-6dpf-1-whole-DopaRD2-MED", "prospr-6dpf-1-whole-ENR1-MED", "prospr-6dpf-1-whole-ENR10-MED", "prospr-6dpf-1-whole-ENR12-MED", "prospr-6dpf-1-whole-ENR13-MED", "prospr-6dpf-1-whole-ENR16-MED", "prospr-6dpf-1-whole-ENR19-MED", "prospr-6dpf-1-whole-ENR2-MED", "prospr-6dpf-1-whole-ENR20-MED", "prospr-6dpf-1-whole-ENR22-MED", "prospr-6dpf-1-whole-ENR25-MED", "prospr-6dpf-1-whole-ENR29-MED", "prospr-6dpf-1-whole-ENR3-MED", "prospr-6dpf-1-whole-ENR30-MED", "prospr-6dpf-1-whole-ENR31-MED", "prospr-6dpf-1-whole-ENR32-MED", "prospr-6dpf-1-whole-ENR34-MED", "prospr-6dpf-1-whole-ENR39-MED", "prospr-6dpf-1-whole-ENR4-MED", "prospr-6dpf-1-whole-ENR46-MED", "prospr-6dpf-1-whole-ENR6-MED", "prospr-6dpf-1-whole-ENR54-MED", "prospr-6dpf-1-whole-ENR57-MED", "prospr-6dpf-1-whole-ENR62-MED", "prospr-6dpf-1-whole-ENR64-MED", "prospr-6dpf-1-whole-ENR69-MED", "prospr-6dpf-1-whole-ENR71-MED", "prospr-6dpf-1-whole-ENR8-MED", "prospr-6dpf-1-whole-ENR9-MED", "prospr-6dpf-1-whole-ElaV-MED", "prospr-6dpf-1-whole-FOR-MED", "prospr-6dpf-1-whole-FVRI-MED", "prospr-6dpf-1-whole-FoxD3-MED", "prospr-6dpf-1-whole-FoxN4-MED", "prospr-6dpf-1-whole-GAD-MED", "prospr-6dpf-1-whole-GCB-MED", "prospr-6dpf-1-whole-GCD-MED", "prospr-6dpf-1-whole-GPCR2-MED", "prospr-6dpf-1-whole-GPCR203-MED", "prospr-6dpf-1-whole-GPCR209-MED", "prospr-6dpf-1-whole-GPCR210-MED", "prospr-6dpf-1-whole-GPCR32-MED", "prospr-6dpf-1-whole-GPPCR21-MED", "prospr-6dpf-1-whole-Gata123-MED", "prospr-6dpf-1-whole-Glt1-MED", "prospr-6dpf-1-whole-GlyT-MED", "prospr-6dpf-1-whole-GnRH-MED", "prospr-6dpf-1-whole-GnrhR-MED", "prospr-6dpf-1-whole-Grm7-MED", "prospr-6dpf-1-whole-Gucy-MED", "prospr-6dpf-1-whole-HCN1-MED", "prospr-6dpf-1-whole-HNF6-MED", "prospr-6dpf-1-whole-Hand-MED", "prospr-6dpf-1-whole-Hb9-MED", "prospr-6dpf-1-whole-Hox5-MED", "prospr-6dpf-1-whole-Hox7-MED", "prospr-6dpf-1-whole-Hr38-MED", "prospr-6dpf-1-whole-Isl-MED", "prospr-6dpf-1-whole-Kv33b-MED", "prospr-6dpf-1-whole-Kv33e-MED", "prospr-6dpf-1-whole-Lhx15-MED", "prospr-6dpf-1-whole-Lhx2-MED", "prospr-6dpf-1-whole-Lhx3-MED", "prospr-6dpf-1-whole-Lmo4-MED", "prospr-6dpf-1-whole-Lmx1-MED", "prospr-6dpf-1-whole-Loc28250-MED", "prospr-6dpf-1-whole-Loc5285ct-MED", "prospr-6dpf-1-whole-Loc77859-MED", "prospr-6dpf-1-whole-Loc8913nt-MED", "prospr-6dpf-1-whole-MHCL4-MED", "prospr-6dpf-1-whole-MRLC2-MED", "prospr-6dpf-1-whole-Maf-MED", "prospr-6dpf-1-whole-Mitf-MED", "prospr-6dpf-1-whole-Munc22-MED", "prospr-6dpf-1-whole-NMDAR-MED", "prospr-6dpf-1-whole-NOV1-MED", "prospr-6dpf-1-whole-NOV15-MED", "prospr-6dpf-1-whole-NOV18-MED", "prospr-6dpf-1-whole-NOV2-MED", "prospr-6dpf-1-whole-NOV29-MED", "prospr-6dpf-1-whole-NOV45-MED", "prospr-6dpf-1-whole-NOV50-MED", "prospr-6dpf-1-whole-NOV52-MED", "prospr-6dpf-1-whole-NOV6-MED", "prospr-6dpf-1-whole-NPR1a-MED", "prospr-6dpf-1-whole-Netrin-MED", "prospr-6dpf-1-whole-NeuroD-MED", "prospr-6dpf-1-whole-Ngb-MED", "prospr-6dpf-1-whole-Olig-MED", "prospr-6dpf-1-whole-P56-MED", "prospr-6dpf-1-whole-PCDH15-MED", "prospr-6dpf-1-whole-PDF-MED", "prospr-6dpf-1-whole-PLCg-MED", "prospr-6dpf-1-whole-Pax258-MED", "prospr-6dpf-1-whole-Pax6-MED", "prospr-6dpf-1-whole-Pde9-MED", "prospr-6dpf-1-whole-Pea3-MED", "prospr-6dpf-1-whole-Phc2-MED", "prospr-6dpf-1-whole-Phox2b-MED", "prospr-6dpf-1-whole-Pikachu-MED", "prospr-6dpf-1-whole-PitxB-MED", "prospr-6dpf-1-whole-Prdm8-MED", "prospr-6dpf-1-whole-Prox-MED", "prospr-6dpf-1-whole-Ptf1-MED", "prospr-6dpf-1-whole-Pty2-MED", "prospr-6dpf-1-whole-RIMS-MED", "prospr-6dpf-1-whole-Rab3-MED", "prospr-6dpf-1-whole-Robo-MED", "prospr-6dpf-1-whole-Scn8aa-MED", "prospr-6dpf-1-whole-Sema2-MED", "prospr-6dpf-1-whole-Sim1-MED", "prospr-6dpf-1-whole-Slit-MED", "prospr-6dpf-1-whole-Sox2-MED", "prospr-6dpf-1-whole-Sox4-MED", "prospr-6dpf-1-whole-SoxB12-MED", "prospr-6dpf-1-whole-SoxB2-MED", "prospr-6dpf-1-whole-Sp8-MED", "prospr-6dpf-1-whole-Syt-MED", "prospr-6dpf-1-whole-Syt12-MED", "prospr-6dpf-1-whole-Syt7-MED", "prospr-6dpf-1-whole-Syta-MED", "prospr-6dpf-1-whole-TH-MED", "prospr-6dpf-1-whole-TIMP-MED", "prospr-6dpf-1-whole-TRPV4-MED", "prospr-6dpf-1-whole-TRPV5-MED", "prospr-6dpf-1-whole-Tal-MED", "prospr-6dpf-1-whole-Tbh-MED", "prospr-6dpf-1-whole-Tbx20-MED", "prospr-6dpf-1-whole-TrpH-MED", "prospr-6dpf-1-whole-TrpV-MED", "prospr-6dpf-1-whole-Twist-MED", "prospr-6dpf-1-whole-UNCS-MED", "prospr-6dpf-1-whole-VAChT-MED", "prospr-6dpf-1-whole-VAT1L-MED", "prospr-6dpf-1-whole-VGluT-MED", "prospr-6dpf-1-whole-VegfR-MED", "prospr-6dpf-1-whole-Wnt16-MED", "prospr-6dpf-1-whole-Wnt5-MED", "prospr-6dpf-1-whole-ascI-MED", "prospr-6dpf-1-whole-ascII-MED", "prospr-6dpf-1-whole-asicalpha-MED", "prospr-6dpf-1-whole-bsx-MED", "prospr-6dpf-1-whole-catL-MED", "prospr-6dpf-1-whole-cnga-MED", "prospr-6dpf-1-whole-dbx1-MED", "prospr-6dpf-1-whole-delta-MED", "prospr-6dpf-1-whole-dlx-MED", "prospr-6dpf-1-whole-edu3to4at6dpf-MED", "prospr-6dpf-1-whole-edu4to5at6dpf-MED", "prospr-6dpf-1-whole-edu5to6at6dpf-MED", "prospr-6dpf-1-whole-eve-MED", "prospr-6dpf-1-whole-eya-MED", "prospr-6dpf-1-whole-fezf-MED", "prospr-6dpf-1-whole-gpb-MED", "prospr-6dpf-1-whole-gsx-MED", "prospr-6dpf-1-whole-hox1-MED", "prospr-6dpf-1-whole-irx-MED", "prospr-6dpf-1-whole-irx2546-MED", "prospr-6dpf-1-whole-lhx6-MED", "prospr-6dpf-1-whole-lbx1b-MED", "prospr-6dpf-1-whole-msx-MED", "prospr-6dpf-1-whole-mecom-MED", "prospr-6dpf-1-whole-neog-MED", "prospr-6dpf-1-whole-muncB-MED", "prospr-6dpf-1-whole-nAchR-MED", "prospr-6dpf-1-whole-ngn-MED", "prospr-6dpf-1-whole-nk21-MED", "prospr-6dpf-1-whole-nk22-MED", "prospr-6dpf-1-whole-nk6-MED", "prospr-6dpf-1-whole-otp-MED", "prospr-6dpf-1-whole-ntrps-MED", "prospr-6dpf-1-whole-rx-MED", "prospr-6dpf-1-whole-paraxis-MED", "prospr-6dpf-1-whole-sert-MED", "prospr-6dpf-1-whole-six4-MED", "prospr-6dpf-1-whole-segmented-CrypticSegment", "prospr-6dpf-1-whole-tlx-MED", "prospr-6dpf-1-whole-trk-MED", "prospr-6dpf-1-whole-uncx-MED", "prospr-6dpf-1-whole-segmented-Head", "prospr-6dpf-1-whole-segmented-PNS", "prospr-6dpf-1-whole-segmented-Pygidium", "prospr-6dpf-1-whole-segmented-RestOfAnimal", "prospr-6dpf-1-whole-segmented-Stomodeum", "prospr-6dpf-1-whole-segmented-VNC", "prospr-6dpf-1-whole-six12-MED"]
\ No newline at end of file
data/sources.json
+ 1
− 3
View file @ c943b07f
[{"modality": "sbem", "stage": "6dpf", "id": "1", "region": "whole"}, [{"modality": "sbem", "stage": "6dpf", "id": "1", "region": "whole"}, {"modality": "prospr", "stage": "6dpf", "id": "1", "region": "whole"}]
{"modality": "prospr", "stage": "6dpf", "id": "1", "region": "whole"}, \ No newline at end of file
{"modality": "fibsem", "stage": "6dpf", "id": "1", "region": "parapod"}]
scripts/__init__.py
+ 0
− 1
View file @ c943b07f
from .sources import get_name_prefixes
scripts/files/__init__.py
+ 6
− 4
View file @ c943b07f
from .copy_helper import copy_tables, copy_segmentation, copy_image_data, copy_misc_data from .copy_helper import copy_tables, copy_segmentation, copy_image_data, copy_misc_data
from .folders import make_folder_structure
from .xml_utils import get_h5_path_from_xml, copy_xml_with_abspath, write_simple_xml
from .xml_utils import copy_xml_with_newpath
from .bdv_server import make_bdv_server_file from .bdv_server import make_bdv_server_file
from .checks import check_bdv
from .folders import make_folder_structure
from .sources import add_image, add_segmentation, add_source
from .sources import get_image_names, get_segmentation_names, get_source_names
scripts/files/copy_helper.py
+ 1
− 1
View file @ c943b07f
import os import os
import shutil import shutil
from .xml_utils import copy_xml_with_newpath, get_h5_path_from_xml from .xml_utils import copy_xml_with_newpath, get_h5_path_from_xml
from ..sources import get_image_names, RAW_FOLDER from .sources import get_image_names, RAW_FOLDER
def copy_file(xml_in, xml_out): def copy_file(xml_in, xml_out):
......
scripts/sources.py → scripts/files/sources.py
+ 46
− 34
View file @ c943b07f
import json import json
import os import os
from shutil import copyfile from shutil import copyfile
from.files import check_bdv, get_h5_path_from_xml, copy_xml_with_newpath from .checks import check_bdv
from .xml_utils import get_h5_path_from_xml, copy_xml_with_newpath
RAW_FOLDER = 'data/rawdata' RAW_FOLDER = 'data/rawdata'
SOURCE_FILE = 'data/sources.json' SOURCE_FILE = 'data/sources.json'
...@@ -15,31 +16,13 @@ def get_sources(): ...@@ -15,31 +16,13 @@ def get_sources():
See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
for the source naming conventions. for the source naming conventions.
""" """
if not os.path.exists(SOURCE_FILE):
return []
with open(SOURCE_FILE) as f: with open(SOURCE_FILE) as f:
sources = json.load(f) sources = json.load(f)
return sources return sources
def add_source(modality, stage, id=1, region='whole'):
""" Add a new data source
See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
for the source naming conventions.
"""
if not isinstance(modality, str):
raise ValueError("Expected modality to be a string, not %s" % type(modality))
if not isinstance(stage, str):
raise ValueError("Expected stage to be a string, not %s" % type(id))
if not isinstance(id, int):
raise ValueError("Expected id to be an integer, not %s" % type(id))
if not isinstance(region, str):
raise ValueError("Expected region to be a string, not %s" % type(id))
sources = get_sources()
sources.append({'modality': modality, 'stage': stage, 'id': str(id), 'region': region})
with open(SOURCE_FILE, 'w') as f:
json.dump(sources, f)
def source_to_prefix(source): def source_to_prefix(source):
return '%s-%s-%s-%s' % (source['modality'], return '%s-%s-%s-%s' % (source['modality'],
source['stage'], source['stage'],
...@@ -47,7 +30,7 @@ def source_to_prefix(source): ...@@ -47,7 +30,7 @@ def source_to_prefix(source):
source['region']) source['region'])
def get_name_prefixes(): def get_source_names():
""" Get the name prefixes corresponding to all sources. """ Get the name prefixes corresponding to all sources.
""" """
sources = get_sources() sources = get_sources()
...@@ -56,12 +39,47 @@ def get_name_prefixes(): ...@@ -56,12 +39,47 @@ def get_name_prefixes():
def get_image_names(): def get_image_names():
if not os.path.exists(IMAGE_FILE):
return []
with open(IMAGE_FILE) as f: with open(IMAGE_FILE) as f:
names = json.load(f) names = json.load(f)
return names return names
def add_image(input_path, source_prefix, name, copy_data=True): def get_segmentation_names():
if not os.path.exists(SEGMENTATION_FILE):
return []
with open(SEGMENTATION_FILE) as f:
names = list(json.load(f).keys())
return names
def add_source(modality, stage, id=1, region='whole'):
""" Add a new data source
See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
for the source naming conventions.
"""
if not isinstance(modality, str):
raise ValueError("Expected modality to be a string, not %s" % type(modality))
if not isinstance(stage, str):
raise ValueError("Expected stage to be a string, not %s" % type(id))
if not isinstance(id, int):
raise ValueError("Expected id to be an integer, not %s" % type(id))
if not isinstance(region, str):
raise ValueError("Expected region to be a string, not %s" % type(id))
sources = get_sources()
source = {'modality': modality, 'stage': stage, 'id': str(id), 'region': region}
if source in sources:
raise RuntimeError("Source is already present")
sources.append(source)
with open(SOURCE_FILE, 'w') as f:
json.dump(sources, f)
def add_image(input_path, source_name, name, copy_data=True):
""" Add image volume to the platy browser data. """ Add image volume to the platy browser data.
Parameter: Parameter:
...@@ -73,14 +91,14 @@ def add_image(input_path, source_prefix, name, copy_data=True): ...@@ -73,14 +91,14 @@ def add_image(input_path, source_prefix, name, copy_data=True):
unless adding an image that is already in the rawdata folder. (default: True) unless adding an image that is already in the rawdata folder. (default: True)
""" """
# validate the inputs # validate the inputs
prefixes = get_name_prefixes() source_names = get_source_names()
if source_prefix not in prefixes: if source_name not in source_names:
raise ValueError("""Source prefix %s is not in the current sources. raise ValueError("""Source %s is not in the current sources.
Use 'add_source' to add a new source.""" % source_prefix) Use 'add_source' to add a new source.""" % source_name)
is_bdv = check_bdv(input_path) is_bdv = check_bdv(input_path)
if not is_bdv: if not is_bdv:
raise ValueError("Expect input to be in bdv format") raise ValueError("Expect input to be in bdv format")
output_name = '%s-%s' % (source_prefix, name) output_name = '%s-%s' % (source_name, name)
names = get_image_names() names = get_image_names()
if output_name in names: if output_name in names:
raise ValueError("Name %s is already taken" % output_name) raise ValueError("Name %s is already taken" % output_name)
...@@ -104,12 +122,6 @@ def add_image(input_path, source_prefix, name, copy_data=True): ...@@ -104,12 +122,6 @@ def add_image(input_path, source_prefix, name, copy_data=True):
json.dump(names, f) json.dump(names, f)
def get_segmentation_names():
with open(SEGMENTATION_FILE) as f:
names = list(json.load(f).keys())
return names
def add_segmentation(source_prefix, name): def add_segmentation(source_prefix, name):
""" Add segmentation volume to the platy browser data. """ Add segmentation volume to the platy browser data.
""" """
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment