Skip to content
Snippets Groups Projects
Commit c943b07f authored by Constantin Pape's avatar Constantin Pape
Browse files

add_image is working and added all existing image data

parent 083d67bd
No related branches found
No related tags found
1 merge request!3Refactor sources
This commit is part of merge request !3. Comments created here will be created in the context of that merge request.
Hide whitespace changes
Inline Side-by-side
add_existing_data.py
+ 14
− 5
Edit View file @ c943b07f
#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
import os
from glob import glob
from scripts.sources import add_source, add_image, add_segmentation
from scripts.files import add_source, add_image, add_segmentation
def add_sources():
......@@ -12,12 +14,19 @@ def add_sources():
def add_images():
base_folder = './data/0.2.1/images'
# add sbem raw data
sbem_prefix = 'sbem-6dpf-1-whole'
sbem_raw = './data/0.2.1/images/sbem-6dpf-1-whole-raw.xml'
name = 'raw'
add_image(sbem_raw, sbem_prefix, name, copy_data=False)
# add all prospr images
prospr_prefix = 'prospr-6dpf-1-whole'
prospr_ims = glob.glob(os.path.join(base_folder, 'prospr-6dpf-1-whole-*'))
prospr_ims = glob(os.path.join(base_folder, 'prospr-6dpf-1-whole-*'))
for impath in prospr_ims:
name = os.path.split(impath)[1]
name, ext = os.path.splitext()
name, ext = os.path.splitext(name)
if ext != '.xml':
continue
name = name[(len(prospr_prefix) + 1):]
......@@ -32,8 +41,8 @@ def add_existing_data():
""" Add existing data to the json files that keep track of
sources, image data and segmentations.
"""
add_sources()
add_images()
# add_sources()
# add_images()
add_segmentations()
......
data/images.json 0 → 100644
+ 1
− 0
Edit View file @ c943b07f
["sbem-6dpf-1-whole-raw", "prospr-6dpf-1-whole-AChE-MED", "prospr-6dpf-1-whole-ANPRA-MED", "prospr-6dpf-1-whole-ANPRB-MED", "prospr-6dpf-1-whole-AP2-MED", "prospr-6dpf-1-whole-ASCb-MED", "prospr-6dpf-1-whole-AcTub-MED", "prospr-6dpf-1-whole-AllCR1-MED", "prospr-6dpf-1-whole-AllCR2-MED", "prospr-6dpf-1-whole-Arx-MED", "prospr-6dpf-1-whole-Ash-MED", "prospr-6dpf-1-whole-BAIAP-MED", "prospr-6dpf-1-whole-BarH1-MED", "prospr-6dpf-1-whole-Beta3-MED", "prospr-6dpf-1-whole-Bmpr1-MED", "prospr-6dpf-1-whole-Brn3a-MED", "prospr-6dpf-1-whole-COE-MED", "prospr-6dpf-1-whole-Cal2-MED", "prospr-6dpf-1-whole-Calmodulin-MED", "prospr-6dpf-1-whole-ChAT-MED", "prospr-6dpf-1-whole-Chx10-MED", "prospr-6dpf-1-whole-Dach-MED", "prospr-6dpf-1-whole-DbhLike-MED", "prospr-6dpf-1-whole-DopaRD2-MED", "prospr-6dpf-1-whole-ENR1-MED", "prospr-6dpf-1-whole-ENR10-MED", "prospr-6dpf-1-whole-ENR12-MED", "prospr-6dpf-1-whole-ENR13-MED", "prospr-6dpf-1-whole-ENR16-MED", "prospr-6dpf-1-whole-ENR19-MED", "prospr-6dpf-1-whole-ENR2-MED", "prospr-6dpf-1-whole-ENR20-MED", "prospr-6dpf-1-whole-ENR22-MED", "prospr-6dpf-1-whole-ENR25-MED", "prospr-6dpf-1-whole-ENR29-MED", "prospr-6dpf-1-whole-ENR3-MED", "prospr-6dpf-1-whole-ENR30-MED", "prospr-6dpf-1-whole-ENR31-MED", "prospr-6dpf-1-whole-ENR32-MED", "prospr-6dpf-1-whole-ENR34-MED", "prospr-6dpf-1-whole-ENR39-MED", "prospr-6dpf-1-whole-ENR4-MED", "prospr-6dpf-1-whole-ENR46-MED", "prospr-6dpf-1-whole-ENR6-MED", "prospr-6dpf-1-whole-ENR54-MED", "prospr-6dpf-1-whole-ENR57-MED", "prospr-6dpf-1-whole-ENR62-MED", "prospr-6dpf-1-whole-ENR64-MED", "prospr-6dpf-1-whole-ENR69-MED", "prospr-6dpf-1-whole-ENR71-MED", "prospr-6dpf-1-whole-ENR8-MED", "prospr-6dpf-1-whole-ENR9-MED", "prospr-6dpf-1-whole-ElaV-MED", "prospr-6dpf-1-whole-FOR-MED", "prospr-6dpf-1-whole-FVRI-MED", "prospr-6dpf-1-whole-FoxD3-MED", "prospr-6dpf-1-whole-FoxN4-MED", "prospr-6dpf-1-whole-GAD-MED", "prospr-6dpf-1-whole-GCB-MED", "prospr-6dpf-1-whole-GCD-MED", "prospr-6dpf-1-whole-GPCR2-MED", "prospr-6dpf-1-whole-GPCR203-MED", "prospr-6dpf-1-whole-GPCR209-MED", "prospr-6dpf-1-whole-GPCR210-MED", "prospr-6dpf-1-whole-GPCR32-MED", "prospr-6dpf-1-whole-GPPCR21-MED", "prospr-6dpf-1-whole-Gata123-MED", "prospr-6dpf-1-whole-Glt1-MED", "prospr-6dpf-1-whole-GlyT-MED", "prospr-6dpf-1-whole-GnRH-MED", "prospr-6dpf-1-whole-GnrhR-MED", "prospr-6dpf-1-whole-Grm7-MED", "prospr-6dpf-1-whole-Gucy-MED", "prospr-6dpf-1-whole-HCN1-MED", "prospr-6dpf-1-whole-HNF6-MED", "prospr-6dpf-1-whole-Hand-MED", "prospr-6dpf-1-whole-Hb9-MED", "prospr-6dpf-1-whole-Hox5-MED", "prospr-6dpf-1-whole-Hox7-MED", "prospr-6dpf-1-whole-Hr38-MED", "prospr-6dpf-1-whole-Isl-MED", "prospr-6dpf-1-whole-Kv33b-MED", "prospr-6dpf-1-whole-Kv33e-MED", "prospr-6dpf-1-whole-Lhx15-MED", "prospr-6dpf-1-whole-Lhx2-MED", "prospr-6dpf-1-whole-Lhx3-MED", "prospr-6dpf-1-whole-Lmo4-MED", "prospr-6dpf-1-whole-Lmx1-MED", "prospr-6dpf-1-whole-Loc28250-MED", "prospr-6dpf-1-whole-Loc5285ct-MED", "prospr-6dpf-1-whole-Loc77859-MED", "prospr-6dpf-1-whole-Loc8913nt-MED", "prospr-6dpf-1-whole-MHCL4-MED", "prospr-6dpf-1-whole-MRLC2-MED", "prospr-6dpf-1-whole-Maf-MED", "prospr-6dpf-1-whole-Mitf-MED", "prospr-6dpf-1-whole-Munc22-MED", "prospr-6dpf-1-whole-NMDAR-MED", "prospr-6dpf-1-whole-NOV1-MED", "prospr-6dpf-1-whole-NOV15-MED", "prospr-6dpf-1-whole-NOV18-MED", "prospr-6dpf-1-whole-NOV2-MED", "prospr-6dpf-1-whole-NOV29-MED", "prospr-6dpf-1-whole-NOV45-MED", "prospr-6dpf-1-whole-NOV50-MED", "prospr-6dpf-1-whole-NOV52-MED", "prospr-6dpf-1-whole-NOV6-MED", "prospr-6dpf-1-whole-NPR1a-MED", "prospr-6dpf-1-whole-Netrin-MED", "prospr-6dpf-1-whole-NeuroD-MED", "prospr-6dpf-1-whole-Ngb-MED", "prospr-6dpf-1-whole-Olig-MED", "prospr-6dpf-1-whole-P56-MED", "prospr-6dpf-1-whole-PCDH15-MED", "prospr-6dpf-1-whole-PDF-MED", "prospr-6dpf-1-whole-PLCg-MED", "prospr-6dpf-1-whole-Pax258-MED", "prospr-6dpf-1-whole-Pax6-MED", "prospr-6dpf-1-whole-Pde9-MED", "prospr-6dpf-1-whole-Pea3-MED", "prospr-6dpf-1-whole-Phc2-MED", "prospr-6dpf-1-whole-Phox2b-MED", "prospr-6dpf-1-whole-Pikachu-MED", "prospr-6dpf-1-whole-PitxB-MED", "prospr-6dpf-1-whole-Prdm8-MED", "prospr-6dpf-1-whole-Prox-MED", "prospr-6dpf-1-whole-Ptf1-MED", "prospr-6dpf-1-whole-Pty2-MED", "prospr-6dpf-1-whole-RIMS-MED", "prospr-6dpf-1-whole-Rab3-MED", "prospr-6dpf-1-whole-Robo-MED", "prospr-6dpf-1-whole-Scn8aa-MED", "prospr-6dpf-1-whole-Sema2-MED", "prospr-6dpf-1-whole-Sim1-MED", "prospr-6dpf-1-whole-Slit-MED", "prospr-6dpf-1-whole-Sox2-MED", "prospr-6dpf-1-whole-Sox4-MED", "prospr-6dpf-1-whole-SoxB12-MED", "prospr-6dpf-1-whole-SoxB2-MED", "prospr-6dpf-1-whole-Sp8-MED", "prospr-6dpf-1-whole-Syt-MED", "prospr-6dpf-1-whole-Syt12-MED", "prospr-6dpf-1-whole-Syt7-MED", "prospr-6dpf-1-whole-Syta-MED", "prospr-6dpf-1-whole-TH-MED", "prospr-6dpf-1-whole-TIMP-MED", "prospr-6dpf-1-whole-TRPV4-MED", "prospr-6dpf-1-whole-TRPV5-MED", "prospr-6dpf-1-whole-Tal-MED", "prospr-6dpf-1-whole-Tbh-MED", "prospr-6dpf-1-whole-Tbx20-MED", "prospr-6dpf-1-whole-TrpH-MED", "prospr-6dpf-1-whole-TrpV-MED", "prospr-6dpf-1-whole-Twist-MED", "prospr-6dpf-1-whole-UNCS-MED", "prospr-6dpf-1-whole-VAChT-MED", "prospr-6dpf-1-whole-VAT1L-MED", "prospr-6dpf-1-whole-VGluT-MED", "prospr-6dpf-1-whole-VegfR-MED", "prospr-6dpf-1-whole-Wnt16-MED", "prospr-6dpf-1-whole-Wnt5-MED", "prospr-6dpf-1-whole-ascI-MED", "prospr-6dpf-1-whole-ascII-MED", "prospr-6dpf-1-whole-asicalpha-MED", "prospr-6dpf-1-whole-bsx-MED", "prospr-6dpf-1-whole-catL-MED", "prospr-6dpf-1-whole-cnga-MED", "prospr-6dpf-1-whole-dbx1-MED", "prospr-6dpf-1-whole-delta-MED", "prospr-6dpf-1-whole-dlx-MED", "prospr-6dpf-1-whole-edu3to4at6dpf-MED", "prospr-6dpf-1-whole-edu4to5at6dpf-MED", "prospr-6dpf-1-whole-edu5to6at6dpf-MED", "prospr-6dpf-1-whole-eve-MED", "prospr-6dpf-1-whole-eya-MED", "prospr-6dpf-1-whole-fezf-MED", "prospr-6dpf-1-whole-gpb-MED", "prospr-6dpf-1-whole-gsx-MED", "prospr-6dpf-1-whole-hox1-MED", "prospr-6dpf-1-whole-irx-MED", "prospr-6dpf-1-whole-irx2546-MED", "prospr-6dpf-1-whole-lhx6-MED", "prospr-6dpf-1-whole-lbx1b-MED", "prospr-6dpf-1-whole-msx-MED", "prospr-6dpf-1-whole-mecom-MED", "prospr-6dpf-1-whole-neog-MED", "prospr-6dpf-1-whole-muncB-MED", "prospr-6dpf-1-whole-nAchR-MED", "prospr-6dpf-1-whole-ngn-MED", "prospr-6dpf-1-whole-nk21-MED", "prospr-6dpf-1-whole-nk22-MED", "prospr-6dpf-1-whole-nk6-MED", "prospr-6dpf-1-whole-otp-MED", "prospr-6dpf-1-whole-ntrps-MED", "prospr-6dpf-1-whole-rx-MED", "prospr-6dpf-1-whole-paraxis-MED", "prospr-6dpf-1-whole-sert-MED", "prospr-6dpf-1-whole-six4-MED", "prospr-6dpf-1-whole-segmented-CrypticSegment", "prospr-6dpf-1-whole-tlx-MED", "prospr-6dpf-1-whole-trk-MED", "prospr-6dpf-1-whole-uncx-MED", "prospr-6dpf-1-whole-segmented-Head", "prospr-6dpf-1-whole-segmented-PNS", "prospr-6dpf-1-whole-segmented-Pygidium", "prospr-6dpf-1-whole-segmented-RestOfAnimal", "prospr-6dpf-1-whole-segmented-Stomodeum", "prospr-6dpf-1-whole-segmented-VNC", "prospr-6dpf-1-whole-six12-MED"]
\ No newline at end of file
data/sources.json
+ 1
− 3
Edit View file @ c943b07f
[{"modality": "sbem", "stage": "6dpf", "id": "1", "region": "whole"},
{"modality": "prospr", "stage": "6dpf", "id": "1", "region": "whole"},
{"modality": "fibsem", "stage": "6dpf", "id": "1", "region": "parapod"}]
[{"modality": "sbem", "stage": "6dpf", "id": "1", "region": "whole"}, {"modality": "prospr", "stage": "6dpf", "id": "1", "region": "whole"}]
\ No newline at end of file
scripts/__init__.py
+ 0
− 1
Edit View file @ c943b07f
from .sources import get_name_prefixes
scripts/files/__init__.py
+ 6
− 4
Edit View file @ c943b07f
from .copy_helper import copy_tables, copy_segmentation, copy_image_data, copy_misc_data
from .folders import make_folder_structure
from .xml_utils import get_h5_path_from_xml, copy_xml_with_abspath, write_simple_xml
from .xml_utils import copy_xml_with_newpath
from .bdv_server import make_bdv_server_file
from .checks import check_bdv
from .folders import make_folder_structure
from .sources import add_image, add_segmentation, add_source
from .sources import get_image_names, get_segmentation_names, get_source_names
scripts/files/copy_helper.py
+ 1
− 1
Edit View file @ c943b07f
import os
import shutil
from .xml_utils import copy_xml_with_newpath, get_h5_path_from_xml
from ..sources import get_image_names, RAW_FOLDER
from .sources import get_image_names, RAW_FOLDER
def copy_file(xml_in, xml_out):
......
scripts/sources.py → scripts/files/sources.py
+ 46
− 34
Edit View file @ c943b07f
import json
import os
from shutil import copyfile
from.files import check_bdv, get_h5_path_from_xml, copy_xml_with_newpath
from .checks import check_bdv
from .xml_utils import get_h5_path_from_xml, copy_xml_with_newpath
RAW_FOLDER = 'data/rawdata'
SOURCE_FILE = 'data/sources.json'
......@@ -15,31 +16,13 @@ def get_sources():
See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
for the source naming conventions.
"""
if not os.path.exists(SOURCE_FILE):
return []
with open(SOURCE_FILE) as f:
sources = json.load(f)
return sources
def add_source(modality, stage, id=1, region='whole'):
""" Add a new data source
See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
for the source naming conventions.
"""
if not isinstance(modality, str):
raise ValueError("Expected modality to be a string, not %s" % type(modality))
if not isinstance(stage, str):
raise ValueError("Expected stage to be a string, not %s" % type(id))
if not isinstance(id, int):
raise ValueError("Expected id to be an integer, not %s" % type(id))
if not isinstance(region, str):
raise ValueError("Expected region to be a string, not %s" % type(id))
sources = get_sources()
sources.append({'modality': modality, 'stage': stage, 'id': str(id), 'region': region})
with open(SOURCE_FILE, 'w') as f:
json.dump(sources, f)
def source_to_prefix(source):
return '%s-%s-%s-%s' % (source['modality'],
source['stage'],
......@@ -47,7 +30,7 @@ def source_to_prefix(source):
source['region'])
def get_name_prefixes():
def get_source_names():
""" Get the name prefixes corresponding to all sources.
"""
sources = get_sources()
......@@ -56,12 +39,47 @@ def get_name_prefixes():
def get_image_names():
if not os.path.exists(IMAGE_FILE):
return []
with open(IMAGE_FILE) as f:
names = json.load(f)
return names
def add_image(input_path, source_prefix, name, copy_data=True):
def get_segmentation_names():
if not os.path.exists(SEGMENTATION_FILE):
return []
with open(SEGMENTATION_FILE) as f:
names = list(json.load(f).keys())
return names
def add_source(modality, stage, id=1, region='whole'):
""" Add a new data source
See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
for the source naming conventions.
"""
if not isinstance(modality, str):
raise ValueError("Expected modality to be a string, not %s" % type(modality))
if not isinstance(stage, str):
raise ValueError("Expected stage to be a string, not %s" % type(id))
if not isinstance(id, int):
raise ValueError("Expected id to be an integer, not %s" % type(id))
if not isinstance(region, str):
raise ValueError("Expected region to be a string, not %s" % type(id))
sources = get_sources()
source = {'modality': modality, 'stage': stage, 'id': str(id), 'region': region}
if source in sources:
raise RuntimeError("Source is already present")
sources.append(source)
with open(SOURCE_FILE, 'w') as f:
json.dump(sources, f)
def add_image(input_path, source_name, name, copy_data=True):
""" Add image volume to the platy browser data.
Parameter:
......@@ -73,14 +91,14 @@ def add_image(input_path, source_prefix, name, copy_data=True):
unless adding an image that is already in the rawdata folder. (default: True)
"""
# validate the inputs
prefixes = get_name_prefixes()
if source_prefix not in prefixes:
raise ValueError("""Source prefix %s is not in the current sources.
Use 'add_source' to add a new source.""" % source_prefix)
source_names = get_source_names()
if source_name not in source_names:
raise ValueError("""Source %s is not in the current sources.
Use 'add_source' to add a new source.""" % source_name)
is_bdv = check_bdv(input_path)
if not is_bdv:
raise ValueError("Expect input to be in bdv format")
output_name = '%s-%s' % (source_prefix, name)
output_name = '%s-%s' % (source_name, name)
names = get_image_names()
if output_name in names:
raise ValueError("Name %s is already taken" % output_name)
......@@ -104,12 +122,6 @@ def add_image(input_path, source_prefix, name, copy_data=True):
json.dump(names, f)
def get_segmentation_names():
with open(SEGMENTATION_FILE) as f:
names = list(json.load(f).keys())
return names
def add_segmentation(source_prefix, name):
""" Add segmentation volume to the platy browser data.
"""
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment