From af3f3d650e3d5da6a1a4a079b0d3d0820d8c11ec Mon Sep 17 00:00:00 2001
From: Constantin Pape <constantin.pape@iwr.uni-heidelberg.de>
Date: Tue, 13 Aug 2019 20:46:21 +0200
Subject: [PATCH] Update source logic
---
data/sources.json | 3 ++
.../legacy_export.py | 0
swap_links.py => deprecated/swap_links.py | 0
extract_from_platybrowser.py | 2 +
make_data_sources.py | 4 +-
scripts/default_config.py | 5 +-
scripts/sources.py | 54 ++++++++++++++++---
update_platy_browser.py | 2 +-
8 files changed, 58 insertions(+), 12 deletions(-)
create mode 100644 data/sources.json
rename legacy_export.py => deprecated/legacy_export.py (100%)
rename swap_links.py => deprecated/swap_links.py (100%)
diff --git a/data/sources.json b/data/sources.json
new file mode 100644
index 0000000..154ea26
--- /dev/null
+++ b/data/sources.json
@@ -0,0 +1,3 @@
+[{"modality": "sbem", "stage": "6dpf", "id": "1", "region": "whole"},
+ {"modality": "prospr", "stage": "6dpf", "id": "1", "region": "whole"},
+ {"modality": "fibsem", "stage": "6dpf", "id": "1", "region": "parapod"}]
diff --git a/legacy_export.py b/deprecated/legacy_export.py
similarity index 100%
rename from legacy_export.py
rename to deprecated/legacy_export.py
diff --git a/swap_links.py b/deprecated/swap_links.py
similarity index 100%
rename from swap_links.py
rename to deprecated/swap_links.py
diff --git a/extract_from_platybrowser.py b/extract_from_platybrowser.py
index a3a4bf7..0f57acc 100644
--- a/extract_from_platybrowser.py
+++ b/extract_from_platybrowser.py
@@ -1,3 +1,5 @@
+#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
+
import argparse
import os
import pandas as pd
diff --git a/make_data_sources.py b/make_data_sources.py
index 9942209..096246b 100644
--- a/make_data_sources.py
+++ b/make_data_sources.py
@@ -1,7 +1,9 @@
+#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
+
import os
import glob
-from scripts.files import get_h5_path_from_xml, copy_xml_with_newpath, write_simple_xml
from shutil import copyfile
+from scripts.files import get_h5_path_from_xml, copy_xml_with_newpath, write_simple_xml
def copy_xmls_and_symlink_h5(name_dict, src_folder, trgt_folder):
diff --git a/scripts/default_config.py b/scripts/default_config.py
index 38bb4f6..81427fb 100644
--- a/scripts/default_config.py
+++ b/scripts/default_config.py
@@ -2,8 +2,9 @@ import os
import json
from cluster_tools.cluster_tasks import BaseClusterTask
-DEFAULT_GROUP = 'kreshuk'
-DEFAULT_SHEBANG = '#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python'
+DEFAULT_GROUP = os.environ.get('PLATYBROWSER_GROUP', 'kreshuk')
+DEFAULT_SHEBANG = os.environ.get('PLATYBROWSER_SHEBANG',
+ '#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python')
DEFAULT_BLOCK_SHAPE = [64, 512, 512]
diff --git a/scripts/sources.py b/scripts/sources.py
index 9e736b1..95c02da 100644
--- a/scripts/sources.py
+++ b/scripts/sources.py
@@ -1,15 +1,39 @@
-# this folder contains information about the current data sources, see
-# https://git.embl.de/tischer/platy-browser-tables/blob/dev/README.md#file-naming
+import json
-# TODO maybe store this as exteral file in json
-# list of the current data sources
-SOURCES = [{'modality': 'sbem', 'stage': '6dpf', 'id': '1', 'region': 'whole'},
- {'modality': 'prospr', 'stage': '6dpf', 'id': '1', 'region': 'whole'},
- {'modality': 'fibsem', 'stage': '6dpf', 'id': '1', 'region': 'parapod'}]
+SOURCE_FILE = '../data/sources.json'
+SEGMENTATION_FILE = '../data/segmentations.json'
+IMAGE_FILE = '../data/images.json'
def get_sources():
- return SOURCES
+ """ Get names of the current data sources.
+
+ See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
+ for the source naming conventions.
+ """
+ with open(SOURCE_FILE) as f:
+ sources = json.load(f)
+ return sources
+
+
+def add_source(modality, stage, id=1, region='whole'):
+ """ Add a new data source
+
+ See https://git.embl.de/tischer/platy-browser-tables/README.md#file-naming
+ for the source naming conventions.
+ """
+ if not isinstance(modality, str):
+ raise ValueError("Expected modality to be a string, not %s" % type(modality))
+ if not isinstance(stage, str):
+ raise ValueError("Expected stage to be a string, not %s" % type(id))
+ if not isinstance(id, int):
+ raise ValueError("Expected id to be an integer, not %s" % type(id))
+ if not isinstance(region, str):
+ raise ValueError("Expected region to be a string, not %s" % type(id))
+ sources = get_sources()
+ sources.append({'modality': modality, 'stage': stage, 'id': str(id), 'region': region})
+ with open(SOURCE_FILE, 'w') as f:
+ json.dump(sources, f)
def source_to_prefix(source):
@@ -20,6 +44,20 @@ def source_to_prefix(source):
def get_name_prefixes():
+ """ Get the name prefixes corresponding to all sources.
+ """
sources = get_sources()
prefixes = [source_to_prefix(source) for source in sources]
return prefixes
+
+
+def add_image(input_path):
+ """ Add image volume to the platy browser data.
+ """
+ pass
+
+
+def add_segmentation():
+ """ Add segmentation volume to the platy browser data.
+ """
+ pass
diff --git a/update_platy_browser.py b/update_platy_browser.py
index 472cb75..59ead28 100755
--- a/update_platy_browser.py
+++ b/update_platy_browser.py
@@ -1,4 +1,4 @@
-#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env37/bin/python
+#! /g/arendt/pape/miniconda3/envs/platybrowser/bin/python
import os
import argparse
--
GitLab