Implemented batch export chunking

extensions/chaeo/batch_jobs/20231028_Porto_PA.py

@@ -9,7 +9,7 @@ from extensions.ilastik.models import IlastikPixelClassifierModel
 if __name__ == '__main__':
     sample_id = '20231028-porto-PA'
     root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/')
-    where_czi = (root / 'TREC_STOP_26_Porto/Selection').__str__()
+    where_czi = (root / 'TREC_STOP_26_Porto/231028_automic/20231028-134649_successfulrun/Selection').__str__()
     where_output = autonumber_new_directory(
         'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0023/output',
         'batch-output'
@@ -48,9 +48,10 @@ if __name__ == '__main__':
         [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})],
         params,
         catch_and_continue=False,
+        chunk_size=25,
     )
 
-    csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
-    write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900')
+    # csv_path = (Path(where_output) / 'workflow_data.csv').__str__()
+    # write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900')
 
     print('Finished')
\ No newline at end of file

model_server/util.py

+from math import ceil
 from pathlib import Path
 import re
 from time import localtime, strftime
@@ -82,6 +83,7 @@ def loop_workflow(
         export_batch_csvs: bool = True,
         write_intermediate_products: bool = True,
         catch_and_continue: bool = True,
+        chunk_size: int = None,
 ):
     """
     Iteratively call the specified workflow function on each of a list of input files
@@ -92,7 +94,23 @@ def loop_workflow(
     :param export_batch_csvs: if True, write any tabular data returned by workflow_func to CSV files
     :param write_intermediate_products: if True, write any intermediate image products to TIF files
     :param catch_and_continue: if True, catch exceptions returned by workflow_func and keep iterating
+    :param chunk_size: create subdirectories with specified number of input files, or all in top-level directory if None
     """
+    if chunk_size and chunk_size < len(files):
+        for ci in range(0, ceil(len(files) / ceil(chunk_size))):
+            loop_workflow(
+                files[ci * chunk_size: (ci + 1) * chunk_size],
+                (Path(output_folder_path) / f'part-{ci:04d}').__str__(),
+                workflow_func,
+                models,
+                params,
+                export_batch_csvs=export_batch_csvs,
+                write_intermediate_products=write_intermediate_products,
+                catch_and_continue=catch_and_continue,
+                chunk_size=None
+            )
+        return True
+
     failures = []
     for ii, ff in enumerate(files):
         export_kwargs = {