Merge branch 'dev_rel_imports' into 'staging'

Repaired relative imports in scripts

See merge request rhodes/model_server!61

.idea/.name

-model_server
\ No newline at end of file

model_server/conf/servers/extensions.py → model_server/conf/fastapi.py

 import importlib
 
-from model_server.base.api import app
+from ..base.api import app
+
+print(f'PACKAGE NAME IS ' + __package__)
 
 for ex in ['ilastik']:
-    m = importlib.import_module(f'extensions.{ex}.router')
+    m = importlib.import_module(f'..extensions.{ex}.router', package=__package__)
     app.include_router(m.router)

model_server/scripts/correct_ilastik_abspath.py

-from os.path import relpath
-from pathlib import Path
-import shutil
-
-import h5py
-
-def make_abspath_to_relpath(ilp_filename: str, my_root: Path, their_root: Path):
-    pa_ilp_old = my_root / ilp_filename
-    assert pa_ilp_old.exists()
-    pa_ilp_new = pa_ilp_old.parent / f'relpath_{pa_ilp_old.name}'
-    with h5py.File(shutil.copy(pa_ilp_old, pa_ilp_new), 'r+') as h5:
-        infos = h5['Input Data/infos']
-        for lane in infos.keys():
-            for role in infos[lane].keys():
-                if len(infos[lane][role]) == 0:
-                    continue
-                pa_img_abs = Path(infos[lane][role]['filePath'][()].decode())
-                my_ilp_dir = (my_root / ilp_filename).parent
-                their_ilp_dir = (their_root / ilp_filename).parent
-                pa_img_rel = Path(relpath(pa_img_abs, their_ilp_dir))
-                if pa_img_rel.parts[-2].upper().endswith('.H5'):
-                    assert (my_ilp_dir / Path(*pa_img_rel.parts[0:-1])).exists()
-                else:
-                    assert (my_ilp_dir / pa_img_rel).exists()
-                del infos[lane][role]['filePath']
-                infos[lane][role]['filePath'] = str(pa_img_rel)
-    return pa_ilp_new
-
-
-
-
-if __name__ == '__main__':
-    files = [
-        '01_ilastik_files/240301_LSM900_DNA_PC.ilp',
-        '01_ilastik_files/240320_LSM900_DNA_OC_new.ilp',
-        '01_ilastik_files/240301_LSM900_TM_PC.ilp',
-        '01_ilastik_files/240320_LSM900_TM_OC_new.ilp'
-    ]
-    for f in files:
-        new_ilp = make_abspath_to_relpath(
-            f,
-            Path('w:/03_analysis/Trial3_LSM900'),
-            Path('/g/cuylen/01_Share/Filemaker/01_Experiments/Experiments_1100/1156/03_analysis/Trial3_LSM900')
-        )
-        print(f'Finished converting {new_ilp}')
-
-
-

model_server/scripts/verify_multichannel_ilastik_inputs.py

-from pathlib import Path
-
-import h5py
-import numpy as np
-import pandas as pd
-
-from base.accessors import generate_file_accessor, write_accessor_data_to_file, InMemoryDataAccessor
-from extensions.ilastik.models import IlastikPixelClassifierModel, IlastikObjectClassifierFromPixelPredictionsModel
-
-def get_input_files(where_ilp: Path) -> list:
-    files = []
-    with h5py.File(where_ilp, 'r') as h5:
-        infos = h5['Input Data/infos']
-        for lane in infos.keys():
-            lane_dict = {}
-            for role in infos[lane].keys():
-                if len(infos[lane][role]) == 0:
-                    continue
-                rel_path = Path(infos[lane][role]['filePath'][()].decode())
-                lane_dict[role] = where_ilp.parent / rel_path
-            files.append(lane_dict)
-    return files
-
-if __name__ == '__main__':
-    where_out = Path('c:/Users/rhodes/projects/proj0015-model-server/issues/0032_multiple_input_channels/output')
-    root = Path('w:/03_analysis/Trial3_LSM900')
-    max_files = 1
-    ilps = [
-        '01_ilastik_files/relpath_240301_LSM900_DNA_PC.ilp',
-        '01_ilastik_files/relpath_240320_LSM900_DNA_OC_new.ilp',
-        '01_ilastik_files/relpath_240301_LSM900_TM_PC.ilp',
-        '01_ilastik_files/relpath_240320_LSM900_TM_OC_new.ilp'
-    ]
-    records = []
-    for f in ilps:
-        ilp = root / f
-        assert ilp.exists()
-        outdir = where_out / ilp.stem
-        outdir.mkdir(parents=True, exist_ok=True)
-
-        if ilp.stem.upper().endswith('_PC'):
-            mod = IlastikPixelClassifierModel(
-                params={'project_file': str(ilp)},
-                enforce_embedded=False
-            )
-            infiles = get_input_files(ilp)
-            for ln in infiles[0:max_files]:
-                acc_raw = generate_file_accessor(root / ln['Raw Data'])
-                pxmap = mod.infer(acc_raw)[0]
-                pxmap_fn = 'pxmap_' + ln['Raw Data'].stem + '.tif'
-                write_accessor_data_to_file(outdir / pxmap_fn, pxmap)
-                record = {
-                    'classifier': str(ilp.relative_to(root)),
-                    'input_raw_data': str(ln['Raw Data'].relative_to(root)),
-                    'input_raw_data_chroma': acc_raw.chroma,
-                    'input_raw_data_dtype': acc_raw.dtype,
-                    'input_raw_data_shape_dict': acc_raw.shape_dict,
-                    'output_file': pxmap_fn,
-                    'output_dtype': pxmap.dtype,
-                    'output_chroma': pxmap.chroma,
-                    'output_shape_dict': pxmap.shape_dict,
-                }
-                records.append(record)
-
-        elif ilp.stem.upper().endswith('_OC_NEW'):
-            mod = IlastikObjectClassifierFromPixelPredictionsModel(
-                params={'project_file': str(ilp)},
-                enforce_embedded=False
-            )
-            infiles = get_input_files(ilp)
-            for ln in infiles[0:max_files]:
-                acc_raw = generate_file_accessor(root / ln['Raw Data'])
-                pa_pxmap = root / ln['Prediction Maps']
-
-                if pa_pxmap.parts[-2].upper().endswith('.H5'):
-                    pa_h5f = root / Path(*pa_pxmap.parts[0:-1])
-                    h5_key = pa_pxmap.parts[-1]
-                    pxmap_data = h5py.File(pa_h5f)[h5_key][()] # C x Y x X ?
-                    pxmap_yxc = np.moveaxis(
-                        pxmap_data,
-                        [1, 2, 0],
-                        [0, 1, 2]
-                    )
-                    acc_pxmap = InMemoryDataAccessor(np.expand_dims(pxmap_yxc, -1))
-                else:
-                    acc_pxmap = generate_file_accessor(pa_pxmap)
-                obmap = mod.infer(acc_raw, acc_pxmap)[0]
-                obmap_fn = 'obmap_' + ln['Raw Data'].stem + '.tif'
-                write_accessor_data_to_file(outdir / obmap_fn, obmap)
-                record = {
-                    'classifier': str(ilp.relative_to(root)),
-                    'input_raw_data': str(ln['Raw Data'].relative_to(root)),
-                    'input_raw_data_chroma': acc_raw.chroma,
-                    'input_raw_data_dtype': acc_raw.dtype,
-                    'input_raw_data_shape_dict': acc_raw.shape_dict,
-                    'input_pxmap': str(ln['Prediction Maps'].relative_to(root)),
-                    'input_pxmap_chroma': acc_pxmap.chroma,
-                    'input_pxmap_dtype': acc_pxmap.dtype,
-                    'input_pxmap_shape_dict': acc_pxmap.shape_dict,
-                    'output_file': obmap_fn,
-                    'output_dtype': obmap.dtype,
-                    'output_chroma': obmap.chroma,
-                    'output_shape_dict': obmap.shape_dict,
-                }
-                records.append(record)
-
-        else:
-            raise Exception(f'unidentified project file {ilp}')
-
-    pd.DataFrame(records).to_csv(where_out / 'record.csv', index=False)
-    print('Finished')

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "model_server"
 license = {file = "LICENSE"}
-version = "2024.09_25"
+version = "2024.09.30"
 authors = [
   { name="Christopher Rhodes", email="christopher.rhodes@embl.de" },
 ]
@@ -16,13 +16,14 @@ requires-python = ">=3.9"
 dependencies = [
   "czifile",
   "fastapi >=0.101",
-  "ilastik ==1.4.1b15",
+  "ilastik ==1.4.1b6",
   "imagecodecs",
   "jupyterlab",
   "matplotlib",
   "numpy",
   "pandas",
   "pillow",
+  "protobuf ==4.25.3",
   "pydantic ~=1.10.1",
   "pytorch ==1.*",
   "scikit-image >=0.21.0",

model_server/scripts/run_server.py → scripts/run_server.py

 import argparse
 from multiprocessing import Process
+from pathlib import Path
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3 import Retry
 import uvicorn
 import webbrowser
 
-from conf.defaults import server_conf
+from model_server.conf.defaults import server_conf
 
 
 def parse_args():
@@ -15,7 +16,7 @@ def parse_args():
     )
     parser.add_argument(
         '--confpath',
-        default='conf.servers.extensions',
+        default='model_server.conf.fastapi',
         help='path to server startup configuration',
     )
     parser.add_argument(
@@ -49,7 +50,8 @@ def main(args) -> None:
         target=uvicorn.run,
         args=(f'{args.confpath}:app',),
         kwargs={
-            'app_dir': '.',
+            # 'app_dir': Path('..').resolve().__str__(),
+            'app_dir': '..',
             'host': args.host,
             'port': int(args.port),
             'log_level': 'debug',