Merge branch 'dev_prebuild2024.09.25' into 'staging'

Preparation for build

See merge request rhodes/model_server!60

README.md

@@ -18,7 +18,7 @@ and exposes an extensible API to facilitate low-latency analysis.
    `git clone https://almf-staff:KJmFvyPRbpzoVZDqfMzV@git.embl.de/rhodes/model_server.git`
 5. Create the environment: `mamba env create --file requirements.yml --name model_server_env`
 6. Activate the environment: `mamba activate model_server_env`
-7. Add the project source as a Python package: `pip install -e .`
+7. Add the project source as a Python package: `pip install --no-deps -e .`
 
 ## To start the server:
 1. From the Miniforge prompt, run `mamba activate model_server_env`

model_server/clients/ilastik_map_objects.py

-from os.path import basename, dirname
-
-
-def main(request_func, in_abspath, params):
-	"""
-	Execute a sequence of client requests that load ilastik pixel and object classifiers, then infer on an image file
-	:param request_func: (func) function that implements HTTP client, dependent on which environment request are called from
-	:param in_abspath: (str) absolute path to image file to infer
-	:param params:
-		pixel_classifier_path: (str) absolute path to ilastik project file that defines a pixel classifier
-		object_classifier_path: (str) absolute path to ilastik project file that defines an object classifier
-		channel (optional): (int) channel of the input image to process, use all channels if not specified
-	:return: (str) absolute path where a new object map is written
-	"""
-
-	where = dirname(in_abspath)
-	in_file = basename(in_abspath)
-	
-	px_ilp = params['pixel_classifier_path']
-	ob_ilp = params['object_classifier_path']
-	channel = params.get('channel', None)
-	mip = params.get('mip', False)
-	
-	# configure input and output paths
-	resp = request_func(
-		'PUT',
-		'/paths/watch_input',
-		{
-			'path': where,
-		}
-	)
-	assert resp['status'] == 200, 'Error setting up image directory'
-	
-	resp = request_func(
-		'PUT',
-		'/paths/watch_output',
-		{
-			'path': where,
-		}
-	)
-	assert resp['status'] == 200, 'Error setting up image directory'
-	
-	# load pixel classifier
-	resp = request_func(
-		'PUT',
-		'/ilastik/seg/load/',
-		body={
-			'project_file': px_ilp,
-			'duplicate': False,
-		},
-	)
-	assert resp['status'], 'Error loading classifier: ' + px_ilp
-	id_px_mod = resp['content']['model_id']
-	
-	# load object classifier
-	resp = request_func(
-		'PUT', '/ilastik/pxmap_to_obj/load/',
-		body={
-			'project_file': ob_ilp,
-			'duplicate': False,
-		},
-	)
-	assert resp['status'] == 200, 'Error loading object classifier: ' + {ob_ilp}
-	id_ob_mod = resp['content']['model_id']
-	
-	# run inference
-	resp = request_func(
-		'PUT',
-		'/ilastik/pixel_then_object_classification/infer',
-		{
-			'px_model_id': id_px_mod,
-			'ob_model_id': id_ob_mod,
-			'input_filename': in_file,
-			'channel': channel,
-			'mip': mip,
-		}
-	)
-	assert resp['status'] == 200, 'Error calling workfow'
-	return resp['content']['object_map_filepath']
-	

model_server/clients/imagej/adapter.py

-"""
-Functionality needed to run a client request sequence (clients.*.main) in the ImageJ model_server 2.7 script environment
-"""
-
-import httplib
-import json
-import urllib
-
-from ij import IJ
-from ij import ImagePlus
-
-HOST = '127.0.0.1'
-PORT = 6221
-uri = 'http://{}:{}/'.format(HOST, PORT)
-
-def hit_endpoint(method, endpoint, params=None, body=None, drop_none=True):
-    """
-    Python 2.7 implementation of HTTP client
-    :param method: (str) either 'GET' or 'PUT'
-    :param endpoint: (str) endpoint of HTTP request
-    :param params: (dict) of parameters that are embedded in client request URL
-    :param body: (dict) of parameters that JSON-encoded and attached as payload in request
-    :param drop_none: (bool) remove (presumably optional) parameters with value equal to None
-    :return: (dict) of response status and content, formatted as dict if request is successful
-    """
-    connection = httplib.HTTPConnection(HOST, PORT)
-    if not method in ['GET', 'PUT']:
-        raise Exception('Can only handle GET and PUT requests')
-    k_pop = []
-    if drop_none and params is not None:
-        for k, v in params.items():
-            if v is None:
-                k_pop.append(k)
-        for ki in k_pop:
-            params.pop(ki)
-
-    if params:
-        url = endpoint + '?' + urllib.urlencode(params)
-    else:
-        url = endpoint
-    connection.request(method, url, body=json.dumps(body))
-    resp = connection.getresponse()
-    resp_str = resp.read()
-    try:
-        content = json.loads(resp_str)
-    except Exception:
-        content = {'str': str(resp_str)}
-    return {'status': resp.status, 'content': content}
-
-
-def verify_server(popup=True):
-    try:
-        resp = hit_endpoint('GET', '/')
-    except Exception as e:
-        print(e)
-        msg = 'Could not find server at: ' + uri
-        IJ.log(msg)
-        if popup:
-            IJ.error(msg)
-            raise e
-        return False
-    if resp['status'] != 200:
-        msg = 'Unknown error verifying server at: ' + uri
-        if popup:
-            IJ.error(msg)
-            raise Exception(msg)
-        return False
-    else:
-        IJ.log('Verified server is online at: ' + uri)
-        return True
-
-def run_request_sequence(imp, func, params):
-    """
-    Execute a sequence of client requests in the ImageJ scripting environment
-    :param imp: (ij.ImagePlus) input image
-    :param func: (func) function that implements client request sequence
-    :param params: (dict) parameters specific to client request
-    :return: (ij.ImagePlus) output image
-    """
-    in_path = imp.getProp('Location')
-    out_path = func(hit_endpoint, in_path, params)
-    return ImagePlus(out_path)
\ No newline at end of file

model_server/clients/util.py

-import requests
-
-def get_client(host='127.0.0.1', port=8000):
-    """Return a client wrapper for testing in same model_server 3.9 environment as server"""
-    uri = f'http://{host}:{port}'
-
-    def hit_endpoint(method, endpoint, params=None):
-        if method == 'GET':
-            resp = requests.get(uri + endpoint)
-        elif method == 'PUT':
-            resp = requests.put(uri + endpoint, params=params)
-        else:
-            raise Exception('Can only handle GET and PUT requests')
-        if resp.status_code != 200:
-            return {'status': resp.status_code, 'content': resp.text}
-        else:
-            return {'status': resp.status_code, 'content': resp.json()}
-    return hit_endpoint
\ No newline at end of file

model_server/scripts/run_server.py

@@ -15,7 +15,7 @@ def parse_args():
     )
     parser.add_argument(
         '--confpath',
-        default='conf.servers.ilastik',
+        default='conf.servers.extensions',
         help='path to server startup configuration',
     )
     parser.add_argument(

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "model_server"
 license = {file = "LICENSE"}
-version = "2024.07.08"
+version = "2024.09_25"
 authors = [
   { name="Christopher Rhodes", email="christopher.rhodes@embl.de" },
 ]

requirements.yml

+name: model_server_mqtt
+channels:
+  - pytorch
+  - ilastik-forge
+  - conda-forge
+dependencies:
+  - czifile
+  - fastapi>=0.101
+  - ilastik=1.4.1b15
+  - imagecodecs
+  - jupyterlab
+  - matplotlib
+  - numpy
+  - pandas
+  - pillow
+  - pydantic=1.10.*
+  - pytorch=1.*
+  - scikit-image>=0.21.0
+  - scikit-learn>=1.5.0
+  - tifffile
+  - uvicorn>=0.23.0
+  - zstd

tests/base/test_pipelines.py

-from pathlib import Path
 import unittest
 
 from model_server.base.accessors import generate_file_accessor, write_accessor_data_to_file