diff --git a/extensions/chaeo/batch_jobs/20231028_Porto_PA.py b/extensions/chaeo/batch_jobs/20231028_Porto_PA.py index a4afbcdd4ecfc1c9c795c01360c00e0361f4a7f1..f41950fdc4ce0d599c6909afbed67f26431ec491 100644 --- a/extensions/chaeo/batch_jobs/20231028_Porto_PA.py +++ b/extensions/chaeo/batch_jobs/20231028_Porto_PA.py @@ -9,7 +9,7 @@ from extensions.ilastik.models import IlastikPixelClassifierModel if __name__ == '__main__': sample_id = '20231028-porto-PA' root = Path('c:/Users/rhodes/projects/proj0012-trec-handoff/owncloud-sync/TREC-HD/Images/') - where_czi = (root / 'TREC_STOP_26_Porto/Selection').__str__() + where_czi = (root / 'TREC_STOP_26_Porto/231028_automic/20231028-134649_successfulrun/Selection').__str__() where_output = autonumber_new_directory( 'c:/Users/rhodes/projects/proj0011-plankton-seg/exp0023/output', 'batch-output' @@ -48,9 +48,10 @@ if __name__ == '__main__': [IlastikPixelClassifierModel(params={'project_file': Path(px_ilp)})], params, catch_and_continue=False, + chunk_size=25, ) - csv_path = (Path(where_output) / 'workflow_data.csv').__str__() - write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900') + # csv_path = (Path(where_output) / 'workflow_data.csv').__str__() + # write_ecotaxa_tsv(csv_path, where_output, sample_id=sample_id, scope_id='EMBL-MS-Zeiss-LSM900') print('Finished') \ No newline at end of file diff --git a/model_server/util.py b/model_server/util.py index 1cc33753dbe477e3ddff686f2a5459a93386b7bb..a88d5e1d0f5b49644372b82c39887bcc28e08f42 100644 --- a/model_server/util.py +++ b/model_server/util.py @@ -1,3 +1,4 @@ +from math import ceil from pathlib import Path import re from time import localtime, strftime @@ -82,6 +83,7 @@ def loop_workflow( export_batch_csvs: bool = True, write_intermediate_products: bool = True, catch_and_continue: bool = True, + chunk_size: int = None, ): """ Iteratively call the specified workflow function on each of a list of input files @@ -92,7 +94,23 @@ def loop_workflow( :param export_batch_csvs: if True, write any tabular data returned by workflow_func to CSV files :param write_intermediate_products: if True, write any intermediate image products to TIF files :param catch_and_continue: if True, catch exceptions returned by workflow_func and keep iterating + :param chunk_size: create subdirectories with specified number of input files, or all in top-level directory if None """ + if chunk_size and chunk_size < len(files): + for ci in range(0, ceil(len(files) / ceil(chunk_size))): + loop_workflow( + files[ci * chunk_size: (ci + 1) * chunk_size], + (Path(output_folder_path) / f'part-{ci:04d}').__str__(), + workflow_func, + models, + params, + export_batch_csvs=export_batch_csvs, + write_intermediate_products=write_intermediate_products, + catch_and_continue=catch_and_continue, + chunk_size=None + ) + return True + failures = [] for ii, ff in enumerate(files): export_kwargs = {