Skip to content
Snippets Groups Projects
Commit 0bc30be5 authored by Constantin Pape's avatar Constantin Pape
Browse files

Update misc scripts

parent e8b7221c
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
misc/check_n5_file.py
+ 20
− 0
View file @ 0bc30be5
......@@ -34,6 +34,26 @@ def summarise_chunk_stats(path, key):
print("Min/max chunk size in MB:", np.min(sizes) / 1.e6, "/", np.max(sizes) / 1.e6)
# TODO update this to be more general
def estimate_chunk_sizes():
ref_chunk_shape = [128, 128, 128]
ref_chunk_size = float(np.prod(ref_chunk_shape))
ref_chunk_mb = 1.0160599442530536
ref_n_chunks = 1553606.
start = 64
stop = 128
step = 16
for chunk_len in range(start, stop + step, step):
print("Chunks: %i^3" % chunk_len)
rel_size = chunk_len ** 3 / ref_chunk_size
chunk_mb = ref_chunk_mb * rel_size
n_chunks = ref_n_chunks / rel_size
print("Nr. chunks at highest res:", int(n_chunks))
print("Mean chunk size in MB:", chunk_mb)
print()
if __name__ == '__main__':
p = '../data/0.6.5/images/local/sbem-6dpf-1-whole-segmented-cells.n5'
k = 'setup0/timepoint0/s0'
......
make_dev_folder.py → misc/make_dev_folder.py
+ 0
− 0
View file @ 0bc30be5
File moved
mmpb/files/migration.py → misc/migration.py
+ 0
− 0
View file @ 0bc30be5
File moved
mmpb/files/name_lookup.py → misc/name_lookup.py
+ 1
− 1
View file @ 0bc30be5
......@@ -48,9 +48,9 @@ DYNAMIC_SEGMENTATIONS = ['sbem-6dpf-1-whole-segmented-cells',
'sbem-6dpf-1-whole-segmented-nuclei']
ROOT = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data'
LUT_PATH = 'new_name_lut.json'
FILE_NAME_LUT = {}
LUT_PATH = os.path.join(ROOT, 'new_name_lut.json')
IMAGE_PROPERTIES = {}
......
misc/new_name_lut.json 0 → 100644
+ 233
− 0
View file @ 0bc30be5
{
"prospr-6dpf-1-whole-AChE-MED": "prospr-6dpf-1-whole-ache",
"prospr-6dpf-1-whole-ANPRA-MED": "prospr-6dpf-1-whole-anpra",
"prospr-6dpf-1-whole-ANPRB-MED": "prospr-6dpf-1-whole-anprb",
"prospr-6dpf-1-whole-AP2-MED": "prospr-6dpf-1-whole-ap2",
"prospr-6dpf-1-whole-ASCb-MED": "prospr-6dpf-1-whole-ascb",
"prospr-6dpf-1-whole-AllCR1-MED": "prospr-6dpf-1-whole-allcr1",
"prospr-6dpf-1-whole-AllCR2-MED": "prospr-6dpf-1-whole-allcr2",
"prospr-6dpf-1-whole-Arx-MED": "prospr-6dpf-1-whole-arx",
"prospr-6dpf-1-whole-Ash-MED": "prospr-6dpf-1-whole-ash",
"prospr-6dpf-1-whole-BAIAP-MED": "prospr-6dpf-1-whole-baiap",
"prospr-6dpf-1-whole-BarH1-MED": "prospr-6dpf-1-whole-barh1",
"prospr-6dpf-1-whole-Beta3-MED": "prospr-6dpf-1-whole-beta3",
"prospr-6dpf-1-whole-Bmpr1-MED": "prospr-6dpf-1-whole-bmpr1",
"prospr-6dpf-1-whole-Brn124-MED": "prospr-6dpf-1-whole-brn124",
"prospr-6dpf-1-whole-Brn3a-MED": "prospr-6dpf-1-whole-brn3a",
"prospr-6dpf-1-whole-COE-MED": "prospr-6dpf-1-whole-coe",
"prospr-6dpf-1-whole-Cal2-MED": "prospr-6dpf-1-whole-cal2",
"prospr-6dpf-1-whole-Calmodulin-MED": "prospr-6dpf-1-whole-calmodulin",
"prospr-6dpf-1-whole-ChAT-MED": "prospr-6dpf-1-whole-chat",
"prospr-6dpf-1-whole-Chx10-MED": "prospr-6dpf-1-whole-chx10",
"prospr-6dpf-1-whole-Dach-MED": "prospr-6dpf-1-whole-dach",
"prospr-6dpf-1-whole-DbhLike-MED": "prospr-6dpf-1-whole-dbhlike",
"prospr-6dpf-1-whole-DopaRD2-MED": "prospr-6dpf-1-whole-dopard2",
"prospr-6dpf-1-whole-ENR1-PGAM-MED": "prospr-6dpf-1-whole-pgam",
"prospr-6dpf-1-whole-ENR10-UPP-MED": "prospr-6dpf-1-whole-upp",
"prospr-6dpf-1-whole-ENR12-UNC22-MED": "prospr-6dpf-1-whole-unc22",
"prospr-6dpf-1-whole-ENR13-NDUS1-MED": "prospr-6dpf-1-whole-ndus1",
"prospr-6dpf-1-whole-ENR16-ODO2-MED": "prospr-6dpf-1-whole-odo2",
"prospr-6dpf-1-whole-ENR19-FboxLike-MED": "prospr-6dpf-1-whole-fxl21",
"prospr-6dpf-1-whole-ENR2-RYR2-MED": "prospr-6dpf-1-whole-ryr2",
"prospr-6dpf-1-whole-ENR20-PPIB-MED": "prospr-6dpf-1-whole-ppib",
"prospr-6dpf-1-whole-ENR22-CO1A1-MED": "prospr-6dpf-1-whole-co1a1",
"prospr-6dpf-1-whole-ENR25-Synaptopodin-MED": "prospr-6dpf-1-whole-synaptopodin",
"prospr-6dpf-1-whole-ENR29-USP9X-MED": "prospr-6dpf-1-whole-usp9x",
"prospr-6dpf-1-whole-ENR3-JKIP3-MED": "prospr-6dpf-1-whole-jkip3",
"prospr-6dpf-1-whole-ENR30-CCG5-MED": "prospr-6dpf-1-whole-ccvd",
"prospr-6dpf-1-whole-ENR31-Leucin-rich-MED": "prospr-6dpf-1-whole-leucin-rich",
"prospr-6dpf-1-whole-ENR32-GRIK3-MED": "prospr-6dpf-1-whole-grik3",
"prospr-6dpf-1-whole-ENR34-MTHFSD-MED": "prospr-6dpf-1-whole-mthfsd",
"prospr-6dpf-1-whole-ENR39-RPC2-MED": "prospr-6dpf-1-whole-rpc2",
"prospr-6dpf-1-whole-ENR4-SND1-MED": "prospr-6dpf-1-whole-snd1",
"prospr-6dpf-1-whole-ENR46-Calexcitin2-MED": "prospr-6dpf-1-whole-calexcitin2",
"prospr-6dpf-1-whole-ENR54-Boule-like-MED": "prospr-6dpf-1-whole-boule-like",
"prospr-6dpf-1-whole-ENR57-Junctophilin1-MED": "prospr-6dpf-1-whole-junctophilin1",
"prospr-6dpf-1-whole-ENR6-Nucleolin-MED": "prospr-6dpf-1-whole-nucleolin",
"prospr-6dpf-1-whole-ENR62-NB5R3-MED": "prospr-6dpf-1-whole-nb5r3",
"prospr-6dpf-1-whole-ENR64-PSMF1-MED": "prospr-6dpf-1-whole-psmf1",
"prospr-6dpf-1-whole-ENR69-BCA1-MED": "prospr-6dpf-1-whole-bca1",
"prospr-6dpf-1-whole-ENR71-Patched-MED": "prospr-6dpf-1-whole-patched",
"prospr-6dpf-1-whole-ENR8-Non-muscle-MHC-MED": "prospr-6dpf-1-whole-non-muscle-mhc",
"prospr-6dpf-1-whole-ENR9-NOE1-MED": "prospr-6dpf-1-whole-noe1",
"prospr-6dpf-1-whole-Edu42to48-MED": "prospr-6dpf-1-whole-edu42to48",
"prospr-6dpf-1-whole-ElaV-MED": "prospr-6dpf-1-whole-elav",
"prospr-6dpf-1-whole-FOR-MED": "prospr-6dpf-1-whole-for",
"prospr-6dpf-1-whole-FVRI-MED": "prospr-6dpf-1-whole-fvri",
"prospr-6dpf-1-whole-FoxD3-MED": "prospr-6dpf-1-whole-foxd3",
"prospr-6dpf-1-whole-FoxN4-MED": "prospr-6dpf-1-whole-foxn4",
"prospr-6dpf-1-whole-GAD-MED": "prospr-6dpf-1-whole-gad",
"prospr-6dpf-1-whole-GCB-MED": "prospr-6dpf-1-whole-gcb",
"prospr-6dpf-1-whole-GCD-MED": "prospr-6dpf-1-whole-gcd",
"prospr-6dpf-1-whole-GPCR2-MED": "prospr-6dpf-1-whole-gpcr2",
"prospr-6dpf-1-whole-GPCR203-MED": "prospr-6dpf-1-whole-gpcr203",
"prospr-6dpf-1-whole-GPCR209-MED": "prospr-6dpf-1-whole-gpcr209",
"prospr-6dpf-1-whole-GPCR210-MED": "prospr-6dpf-1-whole-gpcr210",
"prospr-6dpf-1-whole-GPCR32-MED": "prospr-6dpf-1-whole-gpcr32",
"prospr-6dpf-1-whole-GPPCR21-MED": "prospr-6dpf-1-whole-gppcr21",
"prospr-6dpf-1-whole-Gata123-MED": "prospr-6dpf-1-whole-gata123",
"prospr-6dpf-1-whole-Glt1-MED": "prospr-6dpf-1-whole-glt1",
"prospr-6dpf-1-whole-GlyT-MED": "prospr-6dpf-1-whole-glyt",
"prospr-6dpf-1-whole-GnRH-MED": "prospr-6dpf-1-whole-gnrh",
"prospr-6dpf-1-whole-GnrhR-MED": "prospr-6dpf-1-whole-gnrhr",
"prospr-6dpf-1-whole-Grm7-MED": "prospr-6dpf-1-whole-grm7",
"prospr-6dpf-1-whole-Gucy-MED": "prospr-6dpf-1-whole-gucy",
"prospr-6dpf-1-whole-HCN1-MED": "prospr-6dpf-1-whole-hcn1",
"prospr-6dpf-1-whole-HNF6-MED": "prospr-6dpf-1-whole-hnf6",
"prospr-6dpf-1-whole-Hand-MED": "prospr-6dpf-1-whole-hand",
"prospr-6dpf-1-whole-Hb9-MED": "prospr-6dpf-1-whole-hb9",
"prospr-6dpf-1-whole-Hox4-MED": "prospr-6dpf-1-whole-hox4",
"prospr-6dpf-1-whole-Hox7-MED": "prospr-6dpf-1-whole-hox7",
"prospr-6dpf-1-whole-Hr38-MED": "prospr-6dpf-1-whole-hr38",
"prospr-6dpf-1-whole-Isl-MED": "prospr-6dpf-1-whole-isl",
"prospr-6dpf-1-whole-Kv33b-MED": "prospr-6dpf-1-whole-kv33b",
"prospr-6dpf-1-whole-Kv33e-MED": "prospr-6dpf-1-whole-kv33e",
"prospr-6dpf-1-whole-Lhx15-MED": "prospr-6dpf-1-whole-lhx15",
"prospr-6dpf-1-whole-Lhx2-MED": "prospr-6dpf-1-whole-lhx2",
"prospr-6dpf-1-whole-Lhx3-MED": "prospr-6dpf-1-whole-lhx3",
"prospr-6dpf-1-whole-Lmo4-MED": "prospr-6dpf-1-whole-lmo4",
"prospr-6dpf-1-whole-Lmx1-MED": "prospr-6dpf-1-whole-lmx1",
"prospr-6dpf-1-whole-Loc28250-MED": "prospr-6dpf-1-whole-loc28250",
"prospr-6dpf-1-whole-Loc5285ct-MED": "prospr-6dpf-1-whole-loc5285ct",
"prospr-6dpf-1-whole-Loc77859-MED": "prospr-6dpf-1-whole-loc77859",
"prospr-6dpf-1-whole-Loc8913nt-MED": "prospr-6dpf-1-whole-loc8913nt",
"prospr-6dpf-1-whole-MHCL4-MED": "prospr-6dpf-1-whole-mhcl4",
"prospr-6dpf-1-whole-MRLC2-MED": "prospr-6dpf-1-whole-mrlc2",
"prospr-6dpf-1-whole-Maf-MED": "prospr-6dpf-1-whole-maf",
"prospr-6dpf-1-whole-Mitf-MED": "prospr-6dpf-1-whole-mitf",
"prospr-6dpf-1-whole-Munc22-MED": "prospr-6dpf-1-whole-munc22",
"prospr-6dpf-1-whole-NMDAR-MED": "prospr-6dpf-1-whole-nmdar",
"prospr-6dpf-1-whole-NOV1-globin-like-MED": "prospr-6dpf-1-whole-globin-like",
"prospr-6dpf-1-whole-NOV15-OLM2A-MED": "prospr-6dpf-1-whole-olm2a",
"prospr-6dpf-1-whole-NOV18-CEPU1-MED": "prospr-6dpf-1-whole-cepu1",
"prospr-6dpf-1-whole-NOV2-NOV2-MED": "prospr-6dpf-1-whole-nov2",
"prospr-6dpf-1-whole-NOV29-NOV29-MED": "prospr-6dpf-1-whole-nov29",
"prospr-6dpf-1-whole-NOV45-TMTC3-MED": "prospr-6dpf-1-whole-tmtc3",
"prospr-6dpf-1-whole-NOV50-GDPD1-MED": "prospr-6dpf-1-whole-gdpd1",
"prospr-6dpf-1-whole-NOV52-KANL3-MED": "prospr-6dpf-1-whole-kanl3",
"prospr-6dpf-1-whole-NOV6-Stathmin-MED": "prospr-6dpf-1-whole-stathmin",
"prospr-6dpf-1-whole-NPR1a-MED": "prospr-6dpf-1-whole-npr1a",
"prospr-6dpf-1-whole-Netrin-MED": "prospr-6dpf-1-whole-netrin",
"prospr-6dpf-1-whole-NeuroD-MED": "prospr-6dpf-1-whole-neurod",
"prospr-6dpf-1-whole-Ngb-MED": "prospr-6dpf-1-whole-ngb",
"prospr-6dpf-1-whole-Olig-MED": "prospr-6dpf-1-whole-olig",
"prospr-6dpf-1-whole-P56-MED": "prospr-6dpf-1-whole-p56",
"prospr-6dpf-1-whole-PCDH15-MED": "prospr-6dpf-1-whole-pcdh15",
"prospr-6dpf-1-whole-PDF-MED": "prospr-6dpf-1-whole-pdf",
"prospr-6dpf-1-whole-PLCg-MED": "prospr-6dpf-1-whole-plcg",
"prospr-6dpf-1-whole-Pax258-MED": "prospr-6dpf-1-whole-pax258",
"prospr-6dpf-1-whole-Pax6-MED": "prospr-6dpf-1-whole-pax6",
"prospr-6dpf-1-whole-Pde9-MED": "prospr-6dpf-1-whole-pde9",
"prospr-6dpf-1-whole-Pea3-MED": "prospr-6dpf-1-whole-pea3",
"prospr-6dpf-1-whole-Phc2-MED": "prospr-6dpf-1-whole-phc2",
"prospr-6dpf-1-whole-Phox2b-MED": "prospr-6dpf-1-whole-phox2b",
"prospr-6dpf-1-whole-Pikachu-MED": "prospr-6dpf-1-whole-pikachu",
"prospr-6dpf-1-whole-PitxB-MED": "prospr-6dpf-1-whole-pitxb",
"prospr-6dpf-1-whole-Prdm8-MED": "prospr-6dpf-1-whole-prdm8",
"prospr-6dpf-1-whole-Prox-MED": "prospr-6dpf-1-whole-prox",
"prospr-6dpf-1-whole-Ptf1-MED": "prospr-6dpf-1-whole-ptf1",
"prospr-6dpf-1-whole-RIMS-MED": "prospr-6dpf-1-whole-rims",
"prospr-6dpf-1-whole-Rab3-MED": "prospr-6dpf-1-whole-rab3",
"prospr-6dpf-1-whole-Robo-MED": "prospr-6dpf-1-whole-robo",
"prospr-6dpf-1-whole-Scn8aa-MED": "prospr-6dpf-1-whole-scn8aa",
"prospr-6dpf-1-whole-Sema2-MED": "prospr-6dpf-1-whole-sema2",
"prospr-6dpf-1-whole-Sim1-MED": "prospr-6dpf-1-whole-sim1",
"prospr-6dpf-1-whole-Slit-MED": "prospr-6dpf-1-whole-slit",
"prospr-6dpf-1-whole-Sox2-MED": "prospr-6dpf-1-whole-sox2",
"prospr-6dpf-1-whole-Sox4-MED": "prospr-6dpf-1-whole-sox4",
"prospr-6dpf-1-whole-SoxB12-MED": "prospr-6dpf-1-whole-soxb12",
"prospr-6dpf-1-whole-SoxB2-MED": "prospr-6dpf-1-whole-soxb2",
"prospr-6dpf-1-whole-Sp8-MED": "prospr-6dpf-1-whole-sp8",
"prospr-6dpf-1-whole-Syt-MED": "prospr-6dpf-1-whole-syt",
"prospr-6dpf-1-whole-Syt12-MED": "prospr-6dpf-1-whole-syt12",
"prospr-6dpf-1-whole-Syt7-MED": "prospr-6dpf-1-whole-syt7",
"prospr-6dpf-1-whole-Syta-MED": "prospr-6dpf-1-whole-syta",
"prospr-6dpf-1-whole-TH-MED": "prospr-6dpf-1-whole-th",
"prospr-6dpf-1-whole-TIMP-MED": "prospr-6dpf-1-whole-timp",
"prospr-6dpf-1-whole-TRPV4-MED": "prospr-6dpf-1-whole-trpv4",
"prospr-6dpf-1-whole-TRPV5-MED": "prospr-6dpf-1-whole-trpv5",
"prospr-6dpf-1-whole-Tal-MED": "prospr-6dpf-1-whole-tal",
"prospr-6dpf-1-whole-Tbh-MED": "prospr-6dpf-1-whole-tbh",
"prospr-6dpf-1-whole-Tbx20-MED": "prospr-6dpf-1-whole-tbx20",
"prospr-6dpf-1-whole-TrpH-MED": "prospr-6dpf-1-whole-trph",
"prospr-6dpf-1-whole-TrpV-MED": "prospr-6dpf-1-whole-trpv",
"prospr-6dpf-1-whole-Twist-MED": "prospr-6dpf-1-whole-twist",
"prospr-6dpf-1-whole-UNCS-MED": "prospr-6dpf-1-whole-uncs",
"prospr-6dpf-1-whole-VAChT-MED": "prospr-6dpf-1-whole-vacht",
"prospr-6dpf-1-whole-VAT1L-MED": "prospr-6dpf-1-whole-vat1l",
"prospr-6dpf-1-whole-VGluT-MED": "prospr-6dpf-1-whole-vglut",
"prospr-6dpf-1-whole-VegfR-MED": "prospr-6dpf-1-whole-vegfr",
"prospr-6dpf-1-whole-Wnt16-MED": "prospr-6dpf-1-whole-wnt16",
"prospr-6dpf-1-whole-Wnt5-MED": "prospr-6dpf-1-whole-wnt5",
"prospr-6dpf-1-whole-ascI-MED": "prospr-6dpf-1-whole-asci",
"prospr-6dpf-1-whole-ascII-MED": "prospr-6dpf-1-whole-ascii",
"prospr-6dpf-1-whole-asicalpha-MED": "prospr-6dpf-1-whole-asicalpha",
"prospr-6dpf-1-whole-ato2-MED": "prospr-6dpf-1-whole-ato2",
"prospr-6dpf-1-whole-bsx-MED": "prospr-6dpf-1-whole-bsx",
"prospr-6dpf-1-whole-catL-MED": "prospr-6dpf-1-whole-catl",
"prospr-6dpf-1-whole-cnga-MED": "prospr-6dpf-1-whole-cnga",
"prospr-6dpf-1-whole-dbx1-MED": "prospr-6dpf-1-whole-dbx1",
"prospr-6dpf-1-whole-delta-MED": "prospr-6dpf-1-whole-delta",
"prospr-6dpf-1-whole-dlx-MED": "prospr-6dpf-1-whole-dlx",
"prospr-6dpf-1-whole-edu3to4at6dpf-MED": "prospr-6dpf-1-whole-edu3to4at6dpf",
"prospr-6dpf-1-whole-edu4to5at6dpf-MED": "prospr-6dpf-1-whole-edu4to5at6dpf",
"prospr-6dpf-1-whole-edu5to6at6dpf-MED": "prospr-6dpf-1-whole-edu5to6at6dpf",
"prospr-6dpf-1-whole-eve-MED": "prospr-6dpf-1-whole-eve",
"prospr-6dpf-1-whole-eya-MED": "prospr-6dpf-1-whole-eya",
"prospr-6dpf-1-whole-fezf-MED": "prospr-6dpf-1-whole-fezf",
"prospr-6dpf-1-whole-gpb-MED": "prospr-6dpf-1-whole-gpb",
"prospr-6dpf-1-whole-gsx-MED": "prospr-6dpf-1-whole-gsx",
"prospr-6dpf-1-whole-hox1-MED": "prospr-6dpf-1-whole-hox1",
"prospr-6dpf-1-whole-irx2546-MED": "prospr-6dpf-1-whole-irx2546",
"prospr-6dpf-1-whole-irx6-MED": "prospr-6dpf-1-whole-irx6",
"prospr-6dpf-1-whole-lbx1b-MED": "prospr-6dpf-1-whole-lbx1b",
"prospr-6dpf-1-whole-lhx6-MED": "prospr-6dpf-1-whole-lhx6",
"prospr-6dpf-1-whole-mecom-MED": "prospr-6dpf-1-whole-mecom",
"prospr-6dpf-1-whole-msx-MED": "prospr-6dpf-1-whole-msx",
"prospr-6dpf-1-whole-muncB-MED": "prospr-6dpf-1-whole-muncb",
"prospr-6dpf-1-whole-nAchR-MED": "prospr-6dpf-1-whole-nachr",
"prospr-6dpf-1-whole-neog-MED": "prospr-6dpf-1-whole-neog",
"prospr-6dpf-1-whole-ngn-MED": "prospr-6dpf-1-whole-ngn",
"prospr-6dpf-1-whole-nk21-MED": "prospr-6dpf-1-whole-nk21",
"prospr-6dpf-1-whole-nk22-MED": "prospr-6dpf-1-whole-nk22",
"prospr-6dpf-1-whole-nk6-MED": "prospr-6dpf-1-whole-nk6",
"prospr-6dpf-1-whole-nompc3-MED": "prospr-6dpf-1-whole-nompc3",
"prospr-6dpf-1-whole-ntrps-MED": "prospr-6dpf-1-whole-ntrps",
"prospr-6dpf-1-whole-otp-MED": "prospr-6dpf-1-whole-otp",
"prospr-6dpf-1-whole-paraxis-MED": "prospr-6dpf-1-whole-paraxis",
"prospr-6dpf-1-whole-pkd1-MED": "prospr-6dpf-1-whole-pkd1",
"prospr-6dpf-1-whole-pkd2-MED": "prospr-6dpf-1-whole-pkd2",
"prospr-6dpf-1-whole-rx-MED": "prospr-6dpf-1-whole-rx",
"prospr-6dpf-1-whole-segmented-AllGlands": "prospr-6dpf-1-whole-segmented-allglands",
"prospr-6dpf-1-whole-segmented-CrypticSegment": "prospr-6dpf-1-whole-segmented-crypticsegment",
"prospr-6dpf-1-whole-segmented-Glands": "prospr-6dpf-1-whole-segmented-glands",
"prospr-6dpf-1-whole-segmented-Head": "prospr-6dpf-1-whole-segmented-head",
"prospr-6dpf-1-whole-segmented-PNS": "prospr-6dpf-1-whole-segmented-lateralectoderm",
"prospr-6dpf-1-whole-segmented-ProSPr6-Ref": "prospr-6dpf-1-whole-segmented-prospr6-ref",
"prospr-6dpf-1-whole-segmented-Pygidium": "prospr-6dpf-1-whole-segmented-pygidium",
"prospr-6dpf-1-whole-segmented-RestOfAnimal": "prospr-6dpf-1-whole-segmented-restofanimal",
"prospr-6dpf-1-whole-segmented-Stomodeum": "prospr-6dpf-1-whole-segmented-foregut",
"prospr-6dpf-1-whole-segmented-VNC": "prospr-6dpf-1-whole-segmented-vnc",
"prospr-6dpf-1-whole-sert-MED": "prospr-6dpf-1-whole-sert",
"prospr-6dpf-1-whole-six12-MED": "prospr-6dpf-1-whole-six12",
"prospr-6dpf-1-whole-six4-MED": "prospr-6dpf-1-whole-six4",
"prospr-6dpf-1-whole-tlx-MED": "prospr-6dpf-1-whole-tlx",
"prospr-6dpf-1-whole-trk-MED": "prospr-6dpf-1-whole-trk",
"prospr-6dpf-1-whole-uncx-MED": "prospr-6dpf-1-whole-uncx",
"prospr-6dpf-1-whole-virtual-cells-labels": "prospr-6dpf-1-whole-virtual-cells",
"sbem-6dpf-1-whole-mask-extrapolated": "sbem-6dpf-1-whole-segmented-extrapolated",
"sbem-6dpf-1-whole-mask-inside": "sbem-6dpf-1-whole-segmented-inside",
"sbem-6dpf-1-whole-mask-outside": "sbem-6dpf-1-whole-segmented-outside",
"sbem-6dpf-1-whole-mask-resin": "sbem-6dpf-1-whole-segmented-resin",
"sbem-6dpf-1-whole-raw": "sbem-6dpf-1-whole-raw",
"sbem-6dpf-1-whole-segmented-ariande-neuropil": "sbem-6dpf-1-whole-segmented-neuropil",
"sbem-6dpf-1-whole-segmented-cells-labels": "sbem-6dpf-1-whole-segmented-cells",
"sbem-6dpf-1-whole-segmented-chromatin-labels": "sbem-6dpf-1-whole-segmented-chromatin",
"sbem-6dpf-1-whole-segmented-cilia-labels": "sbem-6dpf-1-whole-segmented-cilia",
"sbem-6dpf-1-whole-segmented-ganglia-labels": "sbem-6dpf-1-whole-segmented-ganglia",
"sbem-6dpf-1-whole-segmented-midgut": "sbem-6dpf-1-whole-segmented-midgut",
"sbem-6dpf-1-whole-segmented-muscle": "sbem-6dpf-1-whole-segmented-muscle",
"sbem-6dpf-1-whole-segmented-nephridia": "sbem-6dpf-1-whole-segmented-nephridia",
"sbem-6dpf-1-whole-segmented-nuclei-labels": "sbem-6dpf-1-whole-segmented-nuclei",
"sbem-6dpf-1-whole-segmented-tissue-labels": "sbem-6dpf-1-whole-segmented-tissue"
}
\ No newline at end of file
mmpb/files/for_upload.py deleted 100644 → 0
+ 0
− 357
View file @ e8b7221c
import os
import xml.etree.ElementTree as ET
import numpy as np
from shutil import copyfile
from glob import glob
from elf.io import open_file
from pybdv.converter import copy_dataset
from pybdv.util import get_key, get_number_of_scales, get_scale_factors
from pybdv.metadata import write_n5_metadata, get_resolution, indent_xml, get_data_path
def normalize_scale_factors(scale_factors, start_scale):
# we expect scale_factors[0] == [1 1 1]
assert np.prod(scale_factors[0]) == 1
# convert to relative scale factors
rel_scales = [scale_factors[0]]
for scale in range(1, len(scale_factors)):
rel_factor = [sf / prev_sf for sf, prev_sf in zip(scale_factors[scale],
scale_factors[scale - 1])]
rel_scales.append(rel_factor)
# return the relative scales starting at the new scale
new_factors = [[1., 1., 1.]] + rel_scales[(start_scale + 1):]
return new_factors
def copy_file_to_bdv_n5(in_file, out_file, resolution,
chunks=None, start_scale=0):
# if we have the out-file already, do nothing
if os.path.exists(out_file):
return
n_threads = 16
n_scales = get_number_of_scales(in_file, 0, 0)
scale_factors = get_scale_factors(in_file, 0)
# double check newly implemented functions in pybdv
assert n_scales == len(scale_factors)
scale_factors = normalize_scale_factors(scale_factors, start_scale)
for out_scale, in_scale in enumerate(range(start_scale, n_scales)):
in_key = get_key(True, 0, 0, in_scale)
out_key = get_key(False, 0, 0, out_scale)
if chunks is None:
with open_file(in_file, 'r') as f:
chunks_ = f[in_key].chunks
else:
chunks_ = chunks
copy_dataset(in_file, in_key, out_file, out_key, False,
chunks_, n_threads)
write_n5_metadata(out_file, scale_factors, resolution, setup_id=0)
# TODO move to pybdv
def make_xml_s3(in_file, out_file, path_in_bucket,
s3_config, shape, resolution=None):
nt = 1
setup_id = 0
setup_name = None
setup_name = 'Setup%i' % setup_id if setup_name is None else setup_name
nz, ny, nx = tuple(shape)
# check if we have an xml already
tree = ET.parse(in_file)
root = tree.getroot()
# load the sequence description
seqdesc = root.find('SequenceDescription')
# update the image loader
# remove the old image loader
imgload = seqdesc.find('ImageLoader')
seqdesc.remove(imgload)
# write the new image loader
imgload = ET.SubElement(seqdesc, 'ImageLoader')
bdv_dtype = 'bdv.n5.s3'
imgload.set('format', bdv_dtype)
el = ET.SubElement(imgload, 'Key')
el.text = path_in_bucket
# TODO read this from the s3 config instead
el = ET.SubElement(imgload, 'ServiceEndpoint')
el.text = 'https://s3.embl.de'
el = ET.SubElement(imgload, 'BucketName')
el.text = 'platybrowser'
el = ET.SubElement(imgload, 'SigningRegion')
el.text = 'us-west-2'
# load the view descriptions
viewsets = seqdesc.find('ViewSetups')
# load the registration decriptions
vregs = root.find('ViewRegistrations')
# write new resolution and shape
oz, oy, ox = 0.0, 0.0, 0.0
dz, dy, dx = resolution
vs = viewsets.find('ViewSetup')
vss = vs.find('size')
vss.text = '{} {} {}'.format(nx, ny, nz)
vox = vs.find('voxelSize')
voxs = vox.find('size')
voxs.text = '{} {} {}'.format(dx, dy, dz)
for t in range(nt):
vreg = vregs.find('ViewRegistration')
vt = vreg.find('ViewTransform')
vt.set('type', 'affine')
vta = vt.find('affine')
vta.text = '{} 0.0 0.0 {} 0.0 {} 0.0 {} 0.0 0.0 {} {}'.format(dx, ox,
dy, oy,
dz, oz)
indent_xml(root)
tree = ET.ElementTree(root)
tree.write(out_file)
def copy_images(in_folder, out_folder, data_out_folder,
s3_config, images_to_copy, output_root):
os.makedirs(out_folder, exist_ok=True)
xml_s3_folder = os.path.join(out_folder, 's3-n5')
os.makedirs(xml_s3_folder, exist_ok=True)
# TODO make this one as well and copy xml ?
# xml_h5_folder = os.path.join(out_folder, 'embl-h5')
image_names = list(images_to_copy.keys())
input_files = glob(os.path.join(in_folder, '*.xml'))
input_names = [os.path.splitext(os.path.split(im)[1])[0]
for im in input_files]
assert all(im in input_names for im in image_names), str(image_names)
files_to_copy = [input_files[input_names.index(im)]
for im in image_names]
for im_name, in_file in zip(image_names, files_to_copy):
print("Copying", im_name, "...")
in_h5 = get_data_path(in_file, True)
# TODO we don't want to always copy to rawdata, but instead we
# need to copy to the correct path extract from the old path
out_file = os.path.join(data_out_folder, im_name + '.n5')
options = images_to_copy[im_name]
start_scale = options.get('start_scale', 0)
chunks = options.get('chunks', None)
resolution = options.get('resolution', None)
# read the resolution from the xml if it is None
if resolution is None:
resolution = get_resolution(in_file)
# copy from hdf5 to n5
copy_file_to_bdv_n5(in_h5, out_file, resolution, chunks, start_scale)
key = get_key(False, 0, 0, 0)
with open_file(out_file, 'r') as f:
shape = f[key].shape
# update and copy the xml
# path in bucket is the relative path from out_file to output_root
path_in_bucket = os.path.relpath(out_file, output_root)
out_file = os.path.join(xml_s3_folder, im_name + '.xml')
make_xml_s3(in_file, out_file, path_in_bucket, s3_config, shape, resolution)
def copy_segmentations(in_folder, out_folder, segmentations_to_copy, output_root):
# segmentation folders
seg_in = os.path.join(in_folder, 'segmentations')
seg_out = os.path.join(out_folder, 'segmentations')
os.makedirs(seg_out, exist_ok=True)
s3_folder = os.path.join(seg_out, 's3-n5')
os.makedirs(s3_folder, exist_ok=True)
# TODO make this one as well and copy xml ?
# xml_h5_folder = os.path.join(out_folder, 'embl-h5')
# table folders
table_in = os.path.join(in_folder, 'tables')
table_out = os.path.join(out_folder, 'tables')
os.makedirs(table_out, exist_ok=True)
seg_names = list(segmentations_to_copy.keys())
input_files = glob(os.path.join(seg_in, '*.xml'))
input_names = [os.path.splitext(os.path.split(im)[1])[0]
for im in input_files]
assert all(im in input_names for im in seg_names), str(seg_names)
files_to_copy = [input_files[input_names.index(im)]
for im in seg_names]
for seg_name, in_file in zip(seg_names, files_to_copy):
print("Copying", seg_name, "...")
in_h5 = get_data_path(in_file, True)
# TODO we don't want to always copy to rawdata, but instead we
# need to copy to the correct path extract from the old path
out_file = os.path.join(s3_folder, seg_name + '.n5')
options = segmentations_to_copy[seg_name]
start_scale = options.get('start_scale', 0)
chunks = options.get('chunks', None)
resolution = options.get('resolution', None)
# read the resolution from the xml if it is None
if resolution is None:
resolution = get_resolution(in_file)
# copy from hdf5 to n5
copy_file_to_bdv_n5(in_h5, out_file, resolution, chunks, start_scale)
key = get_key(False, 0, 0, 0)
with open_file(out_file, 'r') as f:
shape = f[key].shape
# update and copy the xml
# path in bucket is the relative path from out_file to output_root
path_in_bucket = os.path.relpath(out_file, output_root)
out_file = os.path.join(s3_folder, seg_name + '.xml')
make_xml_s3(in_file, out_file, path_in_bucket, None, shape, resolution)
# check if we need to copy tables
seg_table_in = os.path.join(table_in, seg_name)
if not os.path.exists(seg_table_in):
continue
# TODO! we don't want to copy the tables, but just put relative symlinks.
# only doing an explicit copy for the test!
# copy all tables
seg_table_out = os.path.join(table_out, seg_name)
os.makedirs(seg_table_out, exist_ok=True)
in_tables = glob(os.path.join(seg_table_in, '*'))
for in_table in in_tables:
table_name = os.path.split(in_table)[1]
out_table = os.path.join(seg_table_out, table_name)
if os.path.islink(in_table):
in_table = os.path.abspath(os.path.realpath(in_table))
copyfile(in_table, out_table)
# TODO allow changing chunks and lower start scale
def copy_folder_for_s3(version, images_to_copy, segmentations_to_copy, output_root, s3_config):
input_root = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data'
version_folder = os.path.join(input_root, version)
assert os.path.exists(version_folder), version
data_folder = os.path.join(output_root, 'rawdata')
out_folder = os.path.join(output_root, version)
os.makedirs(data_folder, exist_ok=True)
os.makedirs(out_folder, exist_ok=True)
# copy images:
image_in = os.path.join(version_folder, 'images')
image_out = os.path.join(out_folder, 'images')
copy_images(image_in, image_out, data_folder,
s3_config, images_to_copy, output_root)
# copy segmentations and tables:
copy_segmentations(version_folder, out_folder,
segmentations_to_copy, output_root)
def make_test_folder():
res = [.1, .08, .08]
im_names = {'sbem-6dpf-1-whole-raw': {'start_scale': 3, 'resolution': res},
'prospr-6dpf-1-whole-AChE-MED': {'resolution': [.55, .55, .55]}}
seg_names = {'sbem-6dpf-1-whole-segmented-cells-labels': {'start_scale': 2,
'resolution': res,
'chunks': [32, 512, 512]}}
out = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/test_n5'
s3_config = {}
copy_folder_for_s3('0.6.5', im_names, seg_names, out, s3_config)
def make_different_chunkings():
path = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/rawdata/sbem-6dpf-1-whole-raw.h5'
xml_path = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/0.6.5/images/sbem-6dpf-1-whole-raw.xml'
output_root = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/test_n5'
data_out_folder = os.path.join(output_root, 'rawdata')
start_scale = 3
resolution = [.1, .08, .08]
chunk_shapes = [(32, 256, 256), (32, 128, 128), (64, 64, 64)]
chunk_shapes = [(128, 128, 128)]
for ii, chunks in enumerate(chunk_shapes, 4):
out_path = os.path.join(data_out_folder, 'sbem-6dpf-1-whole-raw-%i.n5' % ii)
copy_file_to_bdv_n5(path, out_path, resolution, chunks, start_scale)
# make the xml
path_in_bucket = os.path.relpath(out_path, output_root)
with open_file(out_path, 'r') as f:
shape = f['setup0/timepoint0/s0'].shape
out_path = os.path.join(data_out_folder, 'sbem-6dpf-1-whole-raw-%i.xml' % ii)
make_xml_s3(xml_path, out_path, path_in_bucket, None, shape, resolution)
def iterate_chunks(path, key):
with open_file(path, 'r') as f:
ds = f[key]
n_chunks = ds.number_of_chunks
ds_path = os.path.join(path, key)
chunk_sizes = []
for root, dirs, files in os.walk(ds_path):
for name in files:
if name == 'attributes.json':
continue
size = os.path.getsize(os.path.join(root, name))
chunk_sizes.append(size)
n_filled = len(chunk_sizes)
percent_filled = float(n_filled) / n_chunks
return percent_filled, chunk_sizes
def check_block_shapes():
import nifty.tools as nt
full_path = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/rawdata/sbem-6dpf-1-whole-raw.h5'
key = 't00000/s00/0/cells'
with open_file(full_path, 'r') as f:
shape = f[key].shape
prefix = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/test_n5/rawdata/sbem-6dpf-1-whole-raw-%i.n5'
ds_key = 'setup0/timepoint0/s0'
block_shapes = [[32, 256, 256], [32, 128, 128], [64, 64, 64], [128, 128, 128]]
for ii, block_shape in enumerate(block_shapes, 1):
path = prefix % ii
percent_filled, sizes = iterate_chunks(path, ds_key)
n_total = nt.blocking([0, 0, 0], shape, block_shape).numberOfBlocks
n_filled = int(n_total * percent_filled)
print("Chunk-shape:", block_shape)
print("Nr. chunks at highest res:", n_filled)
print("Mean chunk size in MB:", np.mean(sizes) / 1.e6, "+-", np.std(sizes) / 1.e6)
print("Min/max chunk size in MB:", np.min(sizes) / 1.e6, "/", np.max(sizes) / 1.e6)
print()
def estimate_chunk_sizes():
ref_chunk_shape = [128, 128, 128]
ref_chunk_size = float(np.prod(ref_chunk_shape))
ref_chunk_mb = 1.0160599442530536
ref_n_chunks = 1553606.
start = 64
stop = 128
step = 16
for chunk_len in range(start, stop + step, step):
print("Chunks: %i^3" % chunk_len)
rel_size = chunk_len ** 3 / ref_chunk_size
chunk_mb = ref_chunk_mb * rel_size
n_chunks = ref_n_chunks / rel_size
print("Nr. chunks at highest res:", int(n_chunks))
print("Mean chunk size in MB:", chunk_mb)
print()
if __name__ == '__main__':
# make_test_folder()
# make_different_chunkings()
# check_block_shapes()
estimate_chunk_sizes()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment