From 65e0301a333f20a413917da52840985ea9ef27b2 Mon Sep 17 00:00:00 2001
From: Constantin Pape <constantin.pape@iwr.uni-heidelberg.de>
Date: Fri, 31 Jan 2020 11:47:59 +0100
Subject: [PATCH] Start to implement prototype for n5 s3 folder
---
.../images/prospr-6dpf-1-whole-AChE-MED.xml | 39 ---
data/test/images/sbem-6dpf-1-whole-raw.xml | 39 ---
data/test/misc/bdv_server.txt | 3 -
...m-6dpf-1-whole-segmented-tissue-labels.xml | 39 ---
.../default.csv | 96 ------
.../default.csv | 96 ------
scripts/files/for_upload.py | 280 ++++++++++++++++++
7 files changed, 280 insertions(+), 312 deletions(-)
delete mode 100644 data/test/images/prospr-6dpf-1-whole-AChE-MED.xml
delete mode 100644 data/test/images/sbem-6dpf-1-whole-raw.xml
delete mode 100644 data/test/misc/bdv_server.txt
delete mode 100644 data/test/segmentations/sbem-6dpf-1-whole-segmented-tissue-labels.xml
delete mode 100644 data/test/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv
delete mode 100644 data/test2/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv
create mode 100644 scripts/files/for_upload.py
diff --git a/data/test/images/prospr-6dpf-1-whole-AChE-MED.xml b/data/test/images/prospr-6dpf-1-whole-AChE-MED.xml
deleted file mode 100644
index 20a7342..0000000
--- a/data/test/images/prospr-6dpf-1-whole-AChE-MED.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<SpimData version="0.2">
- <BasePath type="relative">.</BasePath>
- <SequenceDescription>
- <ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">../../rawdata/prospr-6dpf-1-whole-AChE-MED.h5</hdf5>
- </ImageLoader>
- <ViewSetups>
- <ViewSetup>
- <id>0</id>
- <name>channel 1</name>
- <size>550 518 570</size>
- <voxelSize>
- <unit>micrometer</unit>
- <size>0.5 0.5 0.5</size>
- </voxelSize>
- <attributes>
- <channel>1</channel>
- </attributes>
- </ViewSetup>
- <Attributes name="channel">
- <Channel>
- <id>1</id>
- <name>1</name>
- </Channel>
- </Attributes>
- </ViewSetups>
- <Timepoints type="range">
- <first>0</first>
- <last>0</last>
- </Timepoints>
- </SequenceDescription>
- <ViewRegistrations>
- <ViewRegistration setup="0" timepoint="0">
- <ViewTransform type="affine">
- <affine>0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.5 0.0</affine>
- </ViewTransform>
- </ViewRegistration>
- </ViewRegistrations>
-</SpimData>
diff --git a/data/test/images/sbem-6dpf-1-whole-raw.xml b/data/test/images/sbem-6dpf-1-whole-raw.xml
deleted file mode 100644
index fcf8dfd..0000000
--- a/data/test/images/sbem-6dpf-1-whole-raw.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<SpimData version="0.2">
- <BasePath type="relative">.</BasePath>
- <SequenceDescription>
- <ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">../../rawdata/sbem-6dpf-1-whole-raw.h5</hdf5>
- </ImageLoader>
- <ViewSetups>
- <Attributes name="channel">
- <Channel>
- <id>1</id>
- <name>1</name>
- </Channel>
- </Attributes>
- <ViewSetup>
- <id>0</id>
- <name>channel 1</name>
- <size>27499 25916 11416</size>
- <voxelSize>
- <unit>micrometer</unit>
- <size>0.01 0.01 0.025</size>
- </voxelSize>
- <attributes>
- <channel>1</channel>
- </attributes>
- </ViewSetup>
- </ViewSetups>
- <Timepoints type="range">
- <first>0</first>
- <last>0</last>
- </Timepoints>
- </SequenceDescription>
- <ViewRegistrations>
- <ViewRegistration setup="0" timepoint="0">
- <ViewTransform type="affine">
- <affine>0.01 0.0 0.0 0.0 0.0 0.01 0.0 0.0 0.0 0.0 0.025 0.0</affine>
- </ViewTransform>
- </ViewRegistration>
- </ViewRegistrations>
-</SpimData>
diff --git a/data/test/misc/bdv_server.txt b/data/test/misc/bdv_server.txt
deleted file mode 100644
index 0f5bcfa..0000000
--- a/data/test/misc/bdv_server.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-sbem-6dpf-1-whole-raw ../images/sbem-6dpf-1-whole-raw.xml
-prospr-6dpf-1-whole-AChE-MED ../images/prospr-6dpf-1-whole-AChE-MED.xml
-sbem-6dpf-1-whole-segmented-tissue-labels ../segmentations/sbem-6dpf-1-whole-segmented-tissue-labels.xml
diff --git a/data/test/segmentations/sbem-6dpf-1-whole-segmented-tissue-labels.xml b/data/test/segmentations/sbem-6dpf-1-whole-segmented-tissue-labels.xml
deleted file mode 100644
index b29549d..0000000
--- a/data/test/segmentations/sbem-6dpf-1-whole-segmented-tissue-labels.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<SpimData version="0.2">
- <BasePath type="relative">.</BasePath>
- <SequenceDescription>
- <ImageLoader format="bdv.hdf5">
- <hdf5 type="relative">sbem-6dpf-1-whole-segmented-tissue-labels.h5</hdf5>
- </ImageLoader>
- <ViewSetups>
- <Attributes name="channel">
- <Channel>
- <id>1</id>
- <name>1</name>
- </Channel>
- </Attributes>
- <ViewSetup>
- <id>0</id>
- <name>channel 1</name>
- <size>3438 3240 2854</size>
- <voxelSize>
- <unit>micrometer</unit>
- <size>0.08 0.08 0.1</size>
- </voxelSize>
- <attributes>
- <channel>1</channel>
- </attributes>
- </ViewSetup>
- </ViewSetups>
- <Timepoints type="range">
- <first>0</first>
- <last>0</last>
- </Timepoints>
- </SequenceDescription>
- <ViewRegistrations>
- <ViewRegistration setup="0" timepoint="0">
- <ViewTransform type="affine">
- <affine>0.08 0.0 0.0 0.0 0.0 0.08 0.0 0.0 0.0 0.0 0.1 0.0</affine>
- </ViewTransform>
- </ViewRegistration>
- </ViewRegistrations>
-</SpimData>
diff --git a/data/test/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv b/data/test/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv
deleted file mode 100644
index 2c4eda7..0000000
--- a/data/test/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv
+++ /dev/null
@@ -1,96 +0,0 @@
-label_id anchor_x anchor_y anchor_z bb_min_x bb_min_y bb_min_z bb_max_x bb_max_y bb_max_z n_pixels secretory gut empty gland yolk neuropil cuticle
-0.0 152.3932370794297 141.86494410598675 139.16990729311948 0.0 0.0 0.0 274.96 259.12 285.3 9056765421.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-1.0 153.46500430822326 142.81201541888674 139.92478888769764 38.48 22.4 0.0 246.48 259.12 284.9000000000001 149388736.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0
-2.0 156.2118867063274 144.11069483514112 106.88507755220681 73.12 64.8 23.6 226.24 212.56 259.3 180663718.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0
-3.0 95.9430218736102 187.9346072049488 180.95152832248414 88.0 172.88 177.20000000000005 102.32 194.48 182.5 49466.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-4.0 86.50228575978954 127.37352376925585 235.7283791390804 41.6 112.16 219.4 106.4 144.56 249.1 12369445.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-5.0 75.49980949627009 186.80654894336064 162.1348569711461 43.04 129.28 155.8 98.32 199.76 241.7 1853612.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-6.0 101.8548987544939 148.7236824098839 242.14370393301888 48.56 118.0 158.8 123.68 199.76 255.7 25354114.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-7.0 72.06159196458657 157.25522813773938 238.35779498237557 60.72 137.12 225.0 90.96 181.68 254.9 377031.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-8.0 78.53790814891039 155.94266774922232 248.53374076545256 61.76 146.56 235.8 103.04 161.12 256.5 248794.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-9.0 86.08360621442854 151.20562651899706 251.4257637286572 68.72 132.56 244.2 100.88 169.04 256.3 260040.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-10.0 110.63975680651563 210.0998840168789 166.4813765740736 85.52 200.72 157.4 144.08 220.16 178.10000000000005 210203.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-11.0 94.41382809125538 157.08930773157417 168.01835414457926 86.32000000000002 150.56 159.60000000000005 103.68 165.52 174.4 101868.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-12.0 111.68925789933131 196.47290917745013 167.75729510402056 88.08 177.52 147.4 130.56 227.92 188.1 20992408.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-13.0 117.26508241796766 131.70814521016152 210.5255297073893 89.04 112.96 164.5 128.72 160.8 227.8 641911.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-14.0 96.26132283864416 171.14301383775557 234.1989843849181 93.28 166.24 178.0 104.88 192.88 240.9 15754.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-15.0 106.28217979965551 172.056924392267 183.09994066228543 96.64 121.04 173.9 117.36 189.76 200.5 313460.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-16.0 105.29444571570892 124.70090737151794 240.59130065924148 97.92 118.56 229.8 112.64 129.44 249.8 128481.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-17.0 101.99484150636684 149.71444053102147 168.4181522622596 100.0 148.96 166.9 103.68 150.48 169.9 3691.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-18.0 103.96430850189121 104.12675875678345 237.6575275448117 101.84 100.16 233.2 105.36 110.16 242.9 24324.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-19.0 138.81000448141103 178.20082108414314 123.75153734808023 103.2 153.6 112.9 157.76 199.84 137.5 892576.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-20.0 151.41186897668015 150.08451232374497 184.64375419738258 104.16 91.28 140.8 204.72 202.16 243.7 308555396.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0
-21.0 113.87512183964388 132.726154147776 176.8510345954336 104.56 118.72 170.0 120.16 184.64 197.7 122415.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-22.0 130.53117268077284 110.99392315306915 250.91027972999268 105.92 88.16 195.2 159.44 143.44 272.1 8617989.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0
-23.0 162.7387061363089 148.6900041182222 160.15657801690688 106.72 87.84 27.0 212.56 202.32 273.7 764368680.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0
-24.0 169.81609434594847 156.85003484792438 91.80936878823314 106.56 88.48 27.1 198.08 193.6 272.40000000000003 8181549.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-25.0 176.46937520155385 164.47543517355538 103.0995758317822 107.28 116.32 48.6 208.24 203.76 201.5 10757053.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-26.0 112.64885279954667 151.3752582381267 168.34145028798036 108.16 149.28 165.8 116.24 153.68 170.0 21182.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-27.0 111.09791780489277 111.57802464639107 244.23398288107813 108.08 103.84 240.4 113.76 118.0 247.4 16473.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-28.0 110.34746222987191 120.50781411359728 250.48064639510426 108.64 118.32 249.0 112.56 123.92 252.0 5229.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-29.0 112.67221236500242 80.93783239943653 250.34849334794185 108.96 77.68 247.8 117.52 84.56 253.1 159725.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-30.0 113.86475615190372 143.8654532851414 228.66487102044857 110.08 140.72 226.8 116.72 147.84 230.6 20197.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-31.0 117.49357040491631 168.62341664844905 145.21520561587602 112.56 81.92 140.4 123.6 180.88 249.5 41169.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-32.0 118.19006297176702 150.91271028816905 138.98541640602193 113.52 148.32 136.3 126.48 155.28 143.1 47958.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-33.0 116.83728977167252 208.72911824763324 149.16781139780952 115.04 203.44 147.6 119.2 210.64 150.1 5387.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-34.0 122.18553300808992 114.1927242635706 265.98660779349103 116.64 106.48 261.5 131.52 124.48 269.7 80965.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-35.0 129.87210621102298 95.6164543736139 258.8090766566945 118.08 89.12 242.9 145.84 106.72 268.1 201561.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-36.0 134.3368456987235 90.30840386256628 239.07428871436386 120.8 81.12 230.3 146.32 96.08 247.7 208566.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-37.0 151.6524155491729 129.7259097001629 252.18607358656848 119.84 96.16 236.3 168.96 153.76 260.5 788595.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-38.0 125.33486808860684 87.66274869263763 226.36287678477 120.32 75.04 221.9 128.4 94.8 230.5 34038.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-39.0 130.6832090490835 147.87950600053566 154.1502052921559 121.2 137.52 139.70000000000002 139.28 153.36 167.20000000000005 246478.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-40.0 172.7859170106233 156.12428799909748 133.9586758503506 121.12 97.6 102.2 218.96 202.72 155.5 73338327.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0
-41.0 138.64611303630363 188.38468894389442 108.7390594059406 121.44 97.6 98.8 208.24 212.72 155.3 96960.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-42.0 129.52333506322543 73.12855056137106 195.4809640140248 122.4 67.84 185.0 139.44 78.8 206.8 158006.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-43.0 140.46938889823562 177.29651848206817 89.11615158572236 123.84 152.8 78.0 153.92000000000004 193.36 106.2 790523.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-44.0 137.509471308919 75.47112990615553 225.2348564087993 124.0 26.08 204.2 149.68 96.0 236.7 12242707.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-45.0 143.93570066630213 145.04471753941428 118.88855503541849 128.16 26.16 49.40000000000001 155.84 168.08 231.3 16526412.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
-46.0 137.60112879330944 143.848454958184 102.43412903225807 124.88 68.48 49.40000000000001 143.28 158.48 225.2 83700.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-47.0 135.41065199186505 113.4351190333772 138.4417035530566 130.96 88.48 86.4 143.28 149.68 172.10000000000005 16718.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-48.0 134.3805002908668 96.9257242582897 173.7996218731821 131.52 95.12 172.60000000000005 136.24 100.0 174.9 3438.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-49.0 138.64344191763982 167.32506199266686 141.6212788504334 131.28 164.72 138.8 144.96 171.20000000000005 144.5 47183.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-50.0 136.65472822378894 95.90322492608598 228.72365249033436 133.6 93.52 226.8 140.96 99.52 230.6 8794.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-51.0 138.9016847256204 160.19201677735055 80.29217056973087 135.12 157.68 78.4 141.52 164.24 82.10000000000002 5722.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-52.0 160.56399301216388 84.7636948706983 232.86393339355698 134.24 26.56 219.0 186.24 103.92 245.7 22712037.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-53.0 155.67539955212024 102.02580625586941 167.92204760528787 135.68 30.96 113.5 186.24 150.08 238.4 276860.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-54.0 154.9512100207855 130.1035863654507 123.4334694947734 139.20000000000002 112.96 52.40000000000001 174.08 145.04 148.5 18364230.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
-55.0 145.94204452320298 63.35105105770295 225.80843947280982 139.92000000000002 55.52 54.6 150.16 133.28 235.3 18058.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-56.0 162.52148649326642 200.6838244590498 104.40585423191877 141.12 118.96 80.0 190.16 213.2 139.70000000000002 279046.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-57.0 152.33912517506656 84.25113280694981 167.49795705035825 142.08 70.96000000000002 159.60000000000005 157.84 100.96 172.9 295602.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-58.0 153.05821876622244 54.78435672261645 218.26258273490228 142.4 40.56 210.7 160.64 75.2 227.9 369856.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-59.0 146.95122012381472 77.89932450434921 163.9777094271609 144.56 74.48 159.60000000000005 153.6 81.28 168.70000000000005 25522.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-60.0 158.87240620442796 63.36010139201903 239.09346626010867 145.28 42.64 236.5 174.0 73.2 240.9 188575.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-61.0 153.9285350543782 127.50815251172207 146.34779653553088 145.12 48.4 139.9 162.56 138.96 239.7 277041.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-62.0 157.5597522493154 177.63744425609593 46.22148911200938 146.16 170.32 40.7 162.96 187.6 51.3 30676.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-63.0 164.9602285518104 149.1612331661701 226.3080479501045 147.84 120.8 211.6 179.84 165.92000000000004 240.3 244671.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-64.0 156.29177994496743 95.1689032813582 126.36853415195705 148.08 87.52 48.0 170.16 187.28 135.0 53423.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-65.0 152.0022234053881 84.10277925673519 207.91449792354385 148.72 78.0 204.6 157.36 91.28 213.3 37564.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-66.0 189.87714450512001 141.6613530312171 157.97695868852622 150.24 78.24 124.1 209.52 195.28 190.6 4086395.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-67.0 154.36772005383582 113.4557631224765 118.02678331090176 150.96 88.56 117.0 195.84 115.28 185.3 3715.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-68.0 172.2024426273725 107.94862427626072 220.846281633 153.12 96.88 209.0 182.64 121.6 240.4 270643.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-69.0 175.5015921214865 68.02619134703149 161.60071784812294 153.52 26.72 148.20000000000005 190.48 84.48 173.0 4810349.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-70.0 158.5970736431976 108.72755560936407 138.11145756518485 154.8 26.72 130.4 189.52 118.88 150.1 90857.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-71.0 187.96219173022786 167.10326073278674 8.845422290572785 155.44 73.68 1.3 214.8 192.08 158.70000000000005 1981941.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-72.0 164.4752823745731 109.49591279222471 117.37050170738114 157.28 104.96 116.2 169.92000000000004 114.32 119.4 15228.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-73.0 162.46209539823445 60.43883979130091 233.42464825301056 158.96 50.16 222.2 165.36 67.12 237.7 80882.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-74.0 167.81198935013313 195.9327348408145 122.06046799415009 162.4 194.08 116.2 175.68 198.24 126.3 26667.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-75.0 163.9545669957687 102.63372073342748 253.2270803949224 162.4 100.64 251.4 165.84 105.84 255.2 14180.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-76.0 183.3531535919833 125.04469528371521 90.71662862608332 164.16 106.72 79.80000000000003 207.44 138.32 104.4 1025659.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-77.0 170.80756071805703 79.62986272439281 236.31816261879624 167.04 76.16 235.4 174.4 85.68 237.7 26516.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-78.0 186.74247809520455 119.96362012958514 122.4281296640972 164.96 106.56 116.6 206.32 138.48 130.8 455608.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-79.0 181.71359573146216 94.94621203815355 143.13076904335534 172.16 81.2 139.1 189.44 106.4 147.70000000000005 82089.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-80.0 179.01022166778614 118.62634296132809 79.49983686786298 177.20000000000005 116.96 78.0 180.88 120.8 81.10000000000002 10421.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-81.0 180.34760127783687 90.11223482568636 233.6319922218621 178.72 85.52 231.4 182.08 94.72 236.4 21599.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-82.0 198.84361336319589 92.1530082820987 164.84832334379098 180.16 40.88 143.8 241.68 112.8 184.5 22200049.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-83.0 196.93263200703683 124.07919911475126 105.25274630437045 181.28 44.24 96.9 237.04 139.44 157.3 422932.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-84.0 187.91466520135003 59.233877070413676 160.2959690713557 186.0 52.8 156.20000000000005 190.8 65.68 164.10000000000005 25478.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-85.0 190.9351367302774 215.2153019870155 59.349267165060006 188.32 212.16 56.8 193.68 218.0 61.7 20332.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-86.0 198.0166320379633 115.6992176478133 115.85219528878628 189.28 112.16 112.6 207.28 120.48 119.9 46782.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-87.0 207.22834466559533 76.65577034439231 164.44898074708343 189.84 52.56 156.20000000000005 218.88 90.96 172.9 461073.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-88.0 205.21601557054746 98.64442761142016 97.9210347049618 195.04 85.04 93.5 213.52 111.12 103.0 152082.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-89.0 212.0230957417456 163.28301296790215 103.8358203807597 210.4 155.44 100.7 213.84 170.96 108.1 43492.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-90.0 217.47300745128288 129.52113738401675 65.79442177728069 213.76 125.12 61.90000000000001 221.52 132.56 68.9 126958.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-91.0 0.0 0.0 0.0 1.4381545078898533e+307 1.4381545078898533e+307 1.7976931348623168e+307 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0
-92.0 163.9316056372069 151.7256710280716 62.93796717592157 142.8 132.08 36.0 189.84 172.88 144.5 14730345.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-93.0 153.2293184160122 142.89307873187906 58.77492916060809 143.44 131.36 36.6 189.84 172.48 137.20000000000002 1435289.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-94.0 149.38857142857142 143.62095238095242 66.1857142857143 148.32 135.6 65.9 155.6 145.36 67.0 42.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
diff --git a/data/test2/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv b/data/test2/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv
deleted file mode 100644
index 2c4eda7..0000000
--- a/data/test2/tables/sbem-6dpf-1-whole-segmented-tissue-labels/default.csv
+++ /dev/null
@@ -1,96 +0,0 @@
-label_id anchor_x anchor_y anchor_z bb_min_x bb_min_y bb_min_z bb_max_x bb_max_y bb_max_z n_pixels secretory gut empty gland yolk neuropil cuticle
-0.0 152.3932370794297 141.86494410598675 139.16990729311948 0.0 0.0 0.0 274.96 259.12 285.3 9056765421.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-1.0 153.46500430822326 142.81201541888674 139.92478888769764 38.48 22.4 0.0 246.48 259.12 284.9000000000001 149388736.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0
-2.0 156.2118867063274 144.11069483514112 106.88507755220681 73.12 64.8 23.6 226.24 212.56 259.3 180663718.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0
-3.0 95.9430218736102 187.9346072049488 180.95152832248414 88.0 172.88 177.20000000000005 102.32 194.48 182.5 49466.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-4.0 86.50228575978954 127.37352376925585 235.7283791390804 41.6 112.16 219.4 106.4 144.56 249.1 12369445.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-5.0 75.49980949627009 186.80654894336064 162.1348569711461 43.04 129.28 155.8 98.32 199.76 241.7 1853612.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-6.0 101.8548987544939 148.7236824098839 242.14370393301888 48.56 118.0 158.8 123.68 199.76 255.7 25354114.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-7.0 72.06159196458657 157.25522813773938 238.35779498237557 60.72 137.12 225.0 90.96 181.68 254.9 377031.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-8.0 78.53790814891039 155.94266774922232 248.53374076545256 61.76 146.56 235.8 103.04 161.12 256.5 248794.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-9.0 86.08360621442854 151.20562651899706 251.4257637286572 68.72 132.56 244.2 100.88 169.04 256.3 260040.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-10.0 110.63975680651563 210.0998840168789 166.4813765740736 85.52 200.72 157.4 144.08 220.16 178.10000000000005 210203.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-11.0 94.41382809125538 157.08930773157417 168.01835414457926 86.32000000000002 150.56 159.60000000000005 103.68 165.52 174.4 101868.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-12.0 111.68925789933131 196.47290917745013 167.75729510402056 88.08 177.52 147.4 130.56 227.92 188.1 20992408.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-13.0 117.26508241796766 131.70814521016152 210.5255297073893 89.04 112.96 164.5 128.72 160.8 227.8 641911.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-14.0 96.26132283864416 171.14301383775557 234.1989843849181 93.28 166.24 178.0 104.88 192.88 240.9 15754.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-15.0 106.28217979965551 172.056924392267 183.09994066228543 96.64 121.04 173.9 117.36 189.76 200.5 313460.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-16.0 105.29444571570892 124.70090737151794 240.59130065924148 97.92 118.56 229.8 112.64 129.44 249.8 128481.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-17.0 101.99484150636684 149.71444053102147 168.4181522622596 100.0 148.96 166.9 103.68 150.48 169.9 3691.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-18.0 103.96430850189121 104.12675875678345 237.6575275448117 101.84 100.16 233.2 105.36 110.16 242.9 24324.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-19.0 138.81000448141103 178.20082108414314 123.75153734808023 103.2 153.6 112.9 157.76 199.84 137.5 892576.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-20.0 151.41186897668015 150.08451232374497 184.64375419738258 104.16 91.28 140.8 204.72 202.16 243.7 308555396.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0
-21.0 113.87512183964388 132.726154147776 176.8510345954336 104.56 118.72 170.0 120.16 184.64 197.7 122415.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-22.0 130.53117268077284 110.99392315306915 250.91027972999268 105.92 88.16 195.2 159.44 143.44 272.1 8617989.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0
-23.0 162.7387061363089 148.6900041182222 160.15657801690688 106.72 87.84 27.0 212.56 202.32 273.7 764368680.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0
-24.0 169.81609434594847 156.85003484792438 91.80936878823314 106.56 88.48 27.1 198.08 193.6 272.40000000000003 8181549.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-25.0 176.46937520155385 164.47543517355538 103.0995758317822 107.28 116.32 48.6 208.24 203.76 201.5 10757053.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-26.0 112.64885279954667 151.3752582381267 168.34145028798036 108.16 149.28 165.8 116.24 153.68 170.0 21182.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-27.0 111.09791780489277 111.57802464639107 244.23398288107813 108.08 103.84 240.4 113.76 118.0 247.4 16473.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-28.0 110.34746222987191 120.50781411359728 250.48064639510426 108.64 118.32 249.0 112.56 123.92 252.0 5229.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-29.0 112.67221236500242 80.93783239943653 250.34849334794185 108.96 77.68 247.8 117.52 84.56 253.1 159725.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-30.0 113.86475615190372 143.8654532851414 228.66487102044857 110.08 140.72 226.8 116.72 147.84 230.6 20197.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-31.0 117.49357040491631 168.62341664844905 145.21520561587602 112.56 81.92 140.4 123.6 180.88 249.5 41169.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-32.0 118.19006297176702 150.91271028816905 138.98541640602193 113.52 148.32 136.3 126.48 155.28 143.1 47958.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-33.0 116.83728977167252 208.72911824763324 149.16781139780952 115.04 203.44 147.6 119.2 210.64 150.1 5387.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-34.0 122.18553300808992 114.1927242635706 265.98660779349103 116.64 106.48 261.5 131.52 124.48 269.7 80965.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-35.0 129.87210621102298 95.6164543736139 258.8090766566945 118.08 89.12 242.9 145.84 106.72 268.1 201561.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-36.0 134.3368456987235 90.30840386256628 239.07428871436386 120.8 81.12 230.3 146.32 96.08 247.7 208566.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-37.0 151.6524155491729 129.7259097001629 252.18607358656848 119.84 96.16 236.3 168.96 153.76 260.5 788595.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-38.0 125.33486808860684 87.66274869263763 226.36287678477 120.32 75.04 221.9 128.4 94.8 230.5 34038.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-39.0 130.6832090490835 147.87950600053566 154.1502052921559 121.2 137.52 139.70000000000002 139.28 153.36 167.20000000000005 246478.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-40.0 172.7859170106233 156.12428799909748 133.9586758503506 121.12 97.6 102.2 218.96 202.72 155.5 73338327.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0
-41.0 138.64611303630363 188.38468894389442 108.7390594059406 121.44 97.6 98.8 208.24 212.72 155.3 96960.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-42.0 129.52333506322543 73.12855056137106 195.4809640140248 122.4 67.84 185.0 139.44 78.8 206.8 158006.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-43.0 140.46938889823562 177.29651848206817 89.11615158572236 123.84 152.8 78.0 153.92000000000004 193.36 106.2 790523.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-44.0 137.509471308919 75.47112990615553 225.2348564087993 124.0 26.08 204.2 149.68 96.0 236.7 12242707.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-45.0 143.93570066630213 145.04471753941428 118.88855503541849 128.16 26.16 49.40000000000001 155.84 168.08 231.3 16526412.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
-46.0 137.60112879330944 143.848454958184 102.43412903225807 124.88 68.48 49.40000000000001 143.28 158.48 225.2 83700.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-47.0 135.41065199186505 113.4351190333772 138.4417035530566 130.96 88.48 86.4 143.28 149.68 172.10000000000005 16718.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-48.0 134.3805002908668 96.9257242582897 173.7996218731821 131.52 95.12 172.60000000000005 136.24 100.0 174.9 3438.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-49.0 138.64344191763982 167.32506199266686 141.6212788504334 131.28 164.72 138.8 144.96 171.20000000000005 144.5 47183.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-50.0 136.65472822378894 95.90322492608598 228.72365249033436 133.6 93.52 226.8 140.96 99.52 230.6 8794.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-51.0 138.9016847256204 160.19201677735055 80.29217056973087 135.12 157.68 78.4 141.52 164.24 82.10000000000002 5722.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-52.0 160.56399301216388 84.7636948706983 232.86393339355698 134.24 26.56 219.0 186.24 103.92 245.7 22712037.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-53.0 155.67539955212024 102.02580625586941 167.92204760528787 135.68 30.96 113.5 186.24 150.08 238.4 276860.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-54.0 154.9512100207855 130.1035863654507 123.4334694947734 139.20000000000002 112.96 52.40000000000001 174.08 145.04 148.5 18364230.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
-55.0 145.94204452320298 63.35105105770295 225.80843947280982 139.92000000000002 55.52 54.6 150.16 133.28 235.3 18058.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-56.0 162.52148649326642 200.6838244590498 104.40585423191877 141.12 118.96 80.0 190.16 213.2 139.70000000000002 279046.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-57.0 152.33912517506656 84.25113280694981 167.49795705035825 142.08 70.96000000000002 159.60000000000005 157.84 100.96 172.9 295602.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-58.0 153.05821876622244 54.78435672261645 218.26258273490228 142.4 40.56 210.7 160.64 75.2 227.9 369856.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-59.0 146.95122012381472 77.89932450434921 163.9777094271609 144.56 74.48 159.60000000000005 153.6 81.28 168.70000000000005 25522.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-60.0 158.87240620442796 63.36010139201903 239.09346626010867 145.28 42.64 236.5 174.0 73.2 240.9 188575.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-61.0 153.9285350543782 127.50815251172207 146.34779653553088 145.12 48.4 139.9 162.56 138.96 239.7 277041.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-62.0 157.5597522493154 177.63744425609593 46.22148911200938 146.16 170.32 40.7 162.96 187.6 51.3 30676.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-63.0 164.9602285518104 149.1612331661701 226.3080479501045 147.84 120.8 211.6 179.84 165.92000000000004 240.3 244671.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-64.0 156.29177994496743 95.1689032813582 126.36853415195705 148.08 87.52 48.0 170.16 187.28 135.0 53423.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-65.0 152.0022234053881 84.10277925673519 207.91449792354385 148.72 78.0 204.6 157.36 91.28 213.3 37564.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-66.0 189.87714450512001 141.6613530312171 157.97695868852622 150.24 78.24 124.1 209.52 195.28 190.6 4086395.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-67.0 154.36772005383582 113.4557631224765 118.02678331090176 150.96 88.56 117.0 195.84 115.28 185.3 3715.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-68.0 172.2024426273725 107.94862427626072 220.846281633 153.12 96.88 209.0 182.64 121.6 240.4 270643.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-69.0 175.5015921214865 68.02619134703149 161.60071784812294 153.52 26.72 148.20000000000005 190.48 84.48 173.0 4810349.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-70.0 158.5970736431976 108.72755560936407 138.11145756518485 154.8 26.72 130.4 189.52 118.88 150.1 90857.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-71.0 187.96219173022786 167.10326073278674 8.845422290572785 155.44 73.68 1.3 214.8 192.08 158.70000000000005 1981941.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-72.0 164.4752823745731 109.49591279222471 117.37050170738114 157.28 104.96 116.2 169.92000000000004 114.32 119.4 15228.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-73.0 162.46209539823445 60.43883979130091 233.42464825301056 158.96 50.16 222.2 165.36 67.12 237.7 80882.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-74.0 167.81198935013313 195.9327348408145 122.06046799415009 162.4 194.08 116.2 175.68 198.24 126.3 26667.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-75.0 163.9545669957687 102.63372073342748 253.2270803949224 162.4 100.64 251.4 165.84 105.84 255.2 14180.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-76.0 183.3531535919833 125.04469528371521 90.71662862608332 164.16 106.72 79.80000000000003 207.44 138.32 104.4 1025659.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-77.0 170.80756071805703 79.62986272439281 236.31816261879624 167.04 76.16 235.4 174.4 85.68 237.7 26516.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-78.0 186.74247809520455 119.96362012958514 122.4281296640972 164.96 106.56 116.6 206.32 138.48 130.8 455608.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-79.0 181.71359573146216 94.94621203815355 143.13076904335534 172.16 81.2 139.1 189.44 106.4 147.70000000000005 82089.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-80.0 179.01022166778614 118.62634296132809 79.49983686786298 177.20000000000005 116.96 78.0 180.88 120.8 81.10000000000002 10421.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-81.0 180.34760127783687 90.11223482568636 233.6319922218621 178.72 85.52 231.4 182.08 94.72 236.4 21599.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-82.0 198.84361336319589 92.1530082820987 164.84832334379098 180.16 40.88 143.8 241.68 112.8 184.5 22200049.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
-83.0 196.93263200703683 124.07919911475126 105.25274630437045 181.28 44.24 96.9 237.04 139.44 157.3 422932.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-84.0 187.91466520135003 59.233877070413676 160.2959690713557 186.0 52.8 156.20000000000005 190.8 65.68 164.10000000000005 25478.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-85.0 190.9351367302774 215.2153019870155 59.349267165060006 188.32 212.16 56.8 193.68 218.0 61.7 20332.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-86.0 198.0166320379633 115.6992176478133 115.85219528878628 189.28 112.16 112.6 207.28 120.48 119.9 46782.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-87.0 207.22834466559533 76.65577034439231 164.44898074708343 189.84 52.56 156.20000000000005 218.88 90.96 172.9 461073.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-88.0 205.21601557054746 98.64442761142016 97.9210347049618 195.04 85.04 93.5 213.52 111.12 103.0 152082.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-89.0 212.0230957417456 163.28301296790215 103.8358203807597 210.4 155.44 100.7 213.84 170.96 108.1 43492.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-90.0 217.47300745128288 129.52113738401675 65.79442177728069 213.76 125.12 61.90000000000001 221.52 132.56 68.9 126958.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-91.0 0.0 0.0 0.0 1.4381545078898533e+307 1.4381545078898533e+307 1.7976931348623168e+307 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0
-92.0 163.9316056372069 151.7256710280716 62.93796717592157 142.8 132.08 36.0 189.84 172.88 144.5 14730345.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-93.0 153.2293184160122 142.89307873187906 58.77492916060809 143.44 131.36 36.6 189.84 172.48 137.20000000000002 1435289.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
-94.0 149.38857142857142 143.62095238095242 66.1857142857143 148.32 135.6 65.9 155.6 145.36 67.0 42.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
diff --git a/scripts/files/for_upload.py b/scripts/files/for_upload.py
new file mode 100644
index 0000000..b80ed91
--- /dev/null
+++ b/scripts/files/for_upload.py
@@ -0,0 +1,280 @@
+import os
+import xml.etree.ElementTree as ET
+import numpy as np
+from shutil import copyfile
+
+from scripts.files.xml_utils import get_h5_path_from_xml
+from glob import glob
+from elf.io import open_file
+from pybdv.converter import copy_dataset
+from pybdv.util import get_key, get_number_of_scales, get_scale_factors
+from pybdv.metadata import write_n5_metadata, get_resolution, indent_xml
+
+
+def normalize_scale_factors(scale_factors, start_scale):
+ if start_scale == 0:
+ return scale_factors
+
+ # we expect scale_factors[0] == [1 1 1]
+ assert np.prod(scale_factors[0]) == 1
+
+ # convert to relative scale factors
+ rel_scales = [scale_factors[0]]
+ for scale in range(1, len(scale_factors)):
+ rel_factor = [sf / prev_sf for sf, prev_sf in zip(scale_factors[scale],
+ scale_factors[scale - 1])]
+ rel_scales.append(rel_factor)
+
+ # start at new scale
+ new_factors = [[1, 1, 1]] + rel_scales[(start_scale + 1):]
+
+ # back to absolute factors
+ for scale in range(1, len(new_factors)):
+ new_factor = [sf * prev_sf for sf, prev_sf in zip(new_factors[scale],
+ new_factors[scale - 1])]
+ new_factors[scale] = new_factor
+
+ return new_factors
+
+
+def copy_file_to_bdv_n5(in_file, out_file, resolution,
+ chunks=None, start_scale=0):
+ # if we have the out-file already, do nothing
+ if os.path.exists(out_file):
+ return
+
+ n_threads = 16
+ n_scales = get_number_of_scales(in_file, 0, 0)
+ scale_factors = get_scale_factors(in_file, 0)
+ # double check newly implemented functions in pybdv
+ assert n_scales == len(scale_factors)
+
+ scale_factors = normalize_scale_factors(scale_factors, start_scale)
+
+ for out_scale, in_scale in enumerate(range(start_scale, n_scales)):
+ in_key = get_key(True, 0, 0, in_scale)
+ out_key = get_key(False, 0, 0, out_scale)
+
+ if chunks is None:
+ with open_file(in_file, 'r') as f:
+ chunks_ = f[in_key].chunks
+ else:
+ chunks_ = chunks
+
+ copy_dataset(in_file, in_key, out_file, out_key, False,
+ chunks_, n_threads)
+
+ write_n5_metadata(out_file, scale_factors, resolution, setup_id=0)
+
+
+# TODO move to pybdv
+def make_xml_s3(in_file, out_file, path_in_bucket,
+ s3_config, shape, resolution=None):
+ nt = 1
+ setup_id = 0
+ setup_name = None
+
+ setup_name = 'Setup%i' % setup_id if setup_name is None else setup_name
+ nz, ny, nx = tuple(shape)
+
+ # check if we have an xml already
+ tree = ET.parse(in_file)
+ root = tree.getroot()
+
+ # load the sequence description
+ seqdesc = root.find('SequenceDescription')
+
+ # update the image loader
+ # remove the old image loader
+ imgload = seqdesc.find('ImageLoader')
+ seqdesc.remove(imgload)
+
+ # write the new image loader
+ imgload = ET.SubElement(seqdesc, 'ImageLoader')
+ bdv_dtype = 'bdv.n5.s3'
+ imgload.set('format', bdv_dtype)
+ el = ET.SubElement(imgload, 'Key')
+ el.text = path_in_bucket
+
+ # TODO read this from the s3 config instead
+ el = ET.SubElement(imgload, 'ServiceEndpoint')
+ el.text = 'https://s3/embl.de'
+ el = ET.SubElement(imgload, 'BucketName')
+ el.text = 'cbb-platybrowser'
+ el = ET.SubElement(imgload, 'SigningRegion')
+ el.text = 'us-west-2'
+
+ # load the view descriptions
+ viewsets = seqdesc.find('ViewSetups')
+ # load the registration decriptions
+ vregs = root.find('ViewRegistrations')
+
+ # write new resolution and shape
+ oz, oy, ox = 0.0, 0.0, 0.0
+ dz, dy, dx = resolution
+ vs = viewsets.find('ViewSetup')
+ vss = vs.find('size')
+ vss.text = '{} {} {}'.format(nx, ny, nz)
+ vox = vs.find('voxelSize')
+ voxs = vox.find('size')
+ voxs.text = '{} {} {}'.format(dx, dy, dz)
+
+ for t in range(nt):
+ vreg = vregs.find('ViewRegistration')
+ vt = vreg.find('ViewTransform')
+ vt.set('type', 'affine')
+ vta = vt.find('affine')
+ vta.text = '{} 0.0 0.0 {} 0.0 {} 0.0 {} 0.0 0.0 {} {}'.format(dx, ox,
+ dy, oy,
+ dz, oz)
+ indent_xml(root)
+ tree = ET.ElementTree(root)
+ tree.write(out_file)
+
+
+def copy_images(in_folder, out_folder, data_out_folder,
+ s3_config, images_to_copy, output_root):
+ os.makedirs(out_folder, exist_ok=True)
+ xml_s3_folder = os.path.join(out_folder, 's3-n5')
+ os.makedirs(xml_s3_folder, exist_ok=True)
+ # TODO make this one as well and copy xml ?
+ # xml_h5_folder = os.path.join(out_folder, 'embl-h5')
+
+ image_names = list(images_to_copy.keys())
+ input_files = glob(os.path.join(in_folder, '*.xml'))
+ input_names = [os.path.splitext(os.path.split(im)[1])[0]
+ for im in input_files]
+ assert all(im in input_names for im in image_names), str(image_names)
+ files_to_copy = [input_files[input_names.index(im)]
+ for im in image_names]
+
+ for im_name, in_file in zip(image_names, files_to_copy):
+ print("Copying", im_name, "...")
+ in_h5 = get_h5_path_from_xml(in_file, True)
+ # TODO we don't want to always copy to rawdata, but instead we
+ # need to copy to the correct path extract from the old path
+ out_file = os.path.join(data_out_folder, im_name + '.n5')
+
+ options = images_to_copy[im_name]
+ start_scale = options.get('start_scale', 0)
+ chunks = options.get('chunks', None)
+ resolution = options.get('resolution', None)
+ # read the resolution from the xml if it is None
+ if resolution is None:
+ resolution = get_resolution(in_file)
+
+ # copy from hdf5 to n5
+ copy_file_to_bdv_n5(in_h5, out_file, resolution, chunks, start_scale)
+ key = get_key(False, 0, 0, start_scale)
+ with open_file(out_file, 'r') as f:
+ shape = f[key].shape
+
+ # update and copy the xml
+ # path in bucket is the relative path from out_file to output_root
+ path_in_bucket = os.path.relpath(out_file, output_root)
+ out_file = os.path.join(xml_s3_folder, im_name + '.xml')
+ make_xml_s3(in_file, out_file, path_in_bucket, s3_config, shape, resolution)
+
+
+def copy_segmentations(in_folder, out_folder, segmentations_to_copy, output_root):
+ # segmentation folders
+ seg_in = os.path.join(in_folder, 'segmentations')
+ seg_out = os.path.join(out_folder, 'segmentations')
+ os.makedirs(seg_out, exist_ok=True)
+ s3_folder = os.path.join(seg_out, 's3-n5')
+ os.makedirs(s3_folder, exist_ok=True)
+ # TODO make this one as well and copy xml ?
+ # xml_h5_folder = os.path.join(out_folder, 'embl-h5')
+
+ # table folders
+ table_in = os.path.join(in_folder, 'tables')
+ table_out = os.path.join(out_folder, 'tables')
+ os.makedirs(table_out, exist_ok=True)
+
+ seg_names = list(segmentations_to_copy.keys())
+ input_files = glob(os.path.join(seg_in, '*.xml'))
+ input_names = [os.path.splitext(os.path.split(im)[1])[0]
+ for im in input_files]
+ assert all(im in input_names for im in seg_names), str(seg_names)
+ files_to_copy = [input_files[input_names.index(im)]
+ for im in seg_names]
+
+ for seg_name, in_file in zip(seg_names, files_to_copy):
+ print("Copying", seg_name, "...")
+ in_h5 = get_h5_path_from_xml(in_file, True)
+ # TODO we don't want to always copy to rawdata, but instead we
+ # need to copy to the correct path extract from the old path
+ out_file = os.path.join(s3_folder, seg_name + '.n5')
+
+ options = segmentations_to_copy[seg_name]
+ start_scale = options.get('start_scale', 0)
+ chunks = options.get('chunks', None)
+ resolution = options.get('resolution', None)
+ # read the resolution from the xml if it is None
+ if resolution is None:
+ resolution = get_resolution(in_file)
+
+ # copy from hdf5 to n5
+ copy_file_to_bdv_n5(in_h5, out_file, resolution, chunks, start_scale)
+ key = get_key(False, 0, 0, start_scale)
+ with open_file(out_file, 'r') as f:
+ shape = f[key].shape
+
+ # update and copy the xml
+ # path in bucket is the relative path from out_file to output_root
+ path_in_bucket = os.path.relpath(out_file, output_root)
+ out_file = os.path.join(s3_folder, seg_name + '.xml')
+ make_xml_s3(in_file, out_file, path_in_bucket, s3_config, shape, resolution)
+
+ # check if we need to copy tables
+ seg_table_in = os.path.join(table_in, seg_name)
+ if not os.path.exists(seg_table_in):
+ continue
+
+ # TODO! we don't want to copy the tables, but just put relative symlinks.
+ # only doing an explicit copy for the test!
+ # copy all tables
+ seg_table_out = os.path.join(table_out, seg_name)
+ os.makedirs(seg_table_out, exist_ok=True)
+ in_tables = glob(os.path.join(seg_table_in, '*'))
+ for in_table in in_tables:
+ table_name = os.path.split(in_table)[1]
+ out_table = os.path.join(seg_table_out, table_name)
+ if os.path.islink(in_table):
+ in_table = os.path.abspath(os.path.realpath(in_table))
+ copyfile(in_table, out_table)
+
+
+# TODO allow changing chunks and lower start scale
+def copy_folder_for_s3(version, images_to_copy, segmentations_to_copy, output_root, s3_config):
+ input_root = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data'
+ version_folder = os.path.join(input_root, version)
+ assert os.path.exists(version_folder), version
+
+ data_folder = os.path.join(output_root, 'rawdata')
+ out_folder = os.path.join(output_root, version)
+
+ os.makedirs(data_folder, exist_ok=True)
+ os.makedirs(out_folder, exist_ok=True)
+
+ # copy images:
+ image_in = os.path.join(version_folder, 'images')
+ image_out = os.path.join(out_folder, 'images')
+ copy_images(image_in, image_out, data_folder,
+ s3_config, images_to_copy, output_root)
+
+ # copy segmentations and tables:
+ copy_segmentations(version_folder, out_folder,
+ segmentations_to_copy, output_root)
+
+
+if __name__ == '__main__':
+ res = [.1, .08, .08]
+ im_names = {'sbem-6dpf-1-whole-raw': {'start_scale': 3, 'resolution': res},
+ 'prospr-6dpf-1-whole-AChE-MED': {'resolution': [.55, .55, .55]}}
+ seg_names = {'sbem-6dpf-1-whole-segmented-cells-labels': {'start_scale': 2,
+ 'resolution': res,
+ 'chunks': [32, 512, 512]}}
+ out = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/test_n5'
+ s3_config = {}
+ copy_folder_for_s3('0.6.5', im_names, seg_names, out, s3_config)
--
GitLab