From de6f2ecd2bf838adcd2426cbc1011072aab0d883 Mon Sep 17 00:00:00 2001
From: Christopher Rhodes <christopher.rhodes@embl.de>
Date: Mon, 25 Sep 2023 12:54:30 +0200
Subject: [PATCH] Moved ilastik to its own extensions package
---
api.py | 8 +--
extensions/__init__.py | 0
extensions/examples/__init__.py | 0
.../examples}/ilastik3d.py | 0
extensions/ilastik/__init__.py | 0
conf/ilastik.py => extensions/ilastik/conf.py | 0
.../ilastik/models.py | 10 ++--
extensions/ilastik/tests/__init__.py | 0
.../ilastik/tests}/test_ilastik.py | 6 +-
.../ilastik/workflows.py | 8 +--
model_server/{image.py => accessors.py} | 0
model_server/{model.py => models.py} | 2 +-
model_server/session.py | 9 +--
model_server/share.py | 5 ++
model_server/{workflow.py => workflows.py} | 4 +-
simulate_automic.py | 55 -------------------
tests/test_api.py | 2 +-
tests/test_image.py | 2 +-
tests/test_model.py | 4 +-
tests/test_session.py | 4 +-
tests/test_workflow.py | 4 +-
21 files changed, 33 insertions(+), 90 deletions(-)
create mode 100644 extensions/__init__.py
create mode 100644 extensions/examples/__init__.py
rename {examples => extensions/examples}/ilastik3d.py (100%)
create mode 100644 extensions/ilastik/__init__.py
rename conf/ilastik.py => extensions/ilastik/conf.py (100%)
rename model_server/ilastik.py => extensions/ilastik/models.py (91%)
create mode 100644 extensions/ilastik/tests/__init__.py
rename {tests => extensions/ilastik/tests}/test_ilastik.py (96%)
rename model_server/workflow_ilastik.py => extensions/ilastik/workflows.py (88%)
rename model_server/{image.py => accessors.py} (100%)
rename model_server/{model.py => models.py} (96%)
rename model_server/{workflow.py => workflows.py} (92%)
delete mode 100644 simulate_automic.py
diff --git a/api.py b/api.py
index abd27bbf..45ee844f 100644
--- a/api.py
+++ b/api.py
@@ -1,10 +1,10 @@
from fastapi import FastAPI, HTTPException
-from model_server.ilastik import IlastikImageToImageModel, IlastikPixelClassifierModel, IlastikObjectClassifierModel
-from model_server.model import DummyImageToImageModel, ParameterExpectedError
+from extensions.ilastik.models import IlastikImageToImageModel, IlastikPixelClassifierModel, IlastikObjectClassifierModel
+from model_server.models import DummyImageToImageModel, ParameterExpectedError
from model_server.session import Session
-from model_server.workflow import infer_image_to_image
-from model_server.workflow_ilastik import infer_px_then_ob_model
+from model_server.workflows import infer_image_to_image
+from extensions.ilastik.workflows import infer_px_then_ob_model
app = FastAPI(debug=True)
session = Session()
diff --git a/extensions/__init__.py b/extensions/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/extensions/examples/__init__.py b/extensions/examples/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/ilastik3d.py b/extensions/examples/ilastik3d.py
similarity index 100%
rename from examples/ilastik3d.py
rename to extensions/examples/ilastik3d.py
diff --git a/extensions/ilastik/__init__.py b/extensions/ilastik/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/conf/ilastik.py b/extensions/ilastik/conf.py
similarity index 100%
rename from conf/ilastik.py
rename to extensions/ilastik/conf.py
diff --git a/model_server/ilastik.py b/extensions/ilastik/models.py
similarity index 91%
rename from model_server/ilastik.py
rename to extensions/ilastik/models.py
index aad0f38c..ddcb0db5 100644
--- a/model_server/ilastik.py
+++ b/extensions/ilastik/models.py
@@ -4,9 +4,9 @@ from pathlib import Path
import numpy as np
import vigra
-import conf.ilastik
-from model_server.image import GenericImageDataAccessor, InMemoryDataAccessor
-from model_server.model import ImageToImageModel, ParameterExpectedError
+import extensions.ilastik.conf
+from model_server.accessors import GenericImageDataAccessor, InMemoryDataAccessor
+from model_server.models import ImageToImageModel, ParameterExpectedError
class IlastikImageToImageModel(ImageToImageModel):
@@ -14,7 +14,7 @@ class IlastikImageToImageModel(ImageToImageModel):
def __init__(self, params, autoload=True):
self.project_file = Path(params['project_file'])
params['project_file'] = self.project_file.__str__()
- pap = conf.ilastik.paths['project_files'] / self.project_file
+ pap = extensions.ilastik.conf.paths['project_files'] / self.project_file
self.project_file_abspath = pap
if not pap.exists():
raise FileNotFoundError(f'Project file does not exist: {pap}')
@@ -53,7 +53,7 @@ class IlastikPixelClassifierModel(IlastikImageToImageModel):
@staticmethod
def get_workflow():
- from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
+ from ilastik.workflows import PixelClassificationWorkflow
return PixelClassificationWorkflow
def infer(self, input_img: GenericImageDataAccessor) -> (np.ndarray, dict):
diff --git a/extensions/ilastik/tests/__init__.py b/extensions/ilastik/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/test_ilastik.py b/extensions/ilastik/tests/test_ilastik.py
similarity index 96%
rename from tests/test_ilastik.py
rename to extensions/ilastik/tests/test_ilastik.py
index 747de17a..f8ec7e9e 100644
--- a/tests/test_ilastik.py
+++ b/extensions/ilastik/tests/test_ilastik.py
@@ -4,9 +4,9 @@ import unittest
import numpy as np
import conf.testing
-from model_server.image import CziImageFileAccessor, InMemoryDataAccessor, write_accessor_data_to_file
-from model_server.ilastik import IlastikObjectClassifierModel, IlastikPixelClassifierModel
-from model_server.workflow import infer_image_to_image
+from model_server.accessors import CziImageFileAccessor, InMemoryDataAccessor, write_accessor_data_to_file
+from extensions.ilastik.models import IlastikObjectClassifierModel, IlastikPixelClassifierModel
+from model_server.workflows import infer_image_to_image
from tests.test_api import TestServerBaseClass
class TestIlastikPixelClassification(unittest.TestCase):
diff --git a/model_server/workflow_ilastik.py b/extensions/ilastik/workflows.py
similarity index 88%
rename from model_server/workflow_ilastik.py
rename to extensions/ilastik/workflows.py
index db412bb4..45763ad7 100644
--- a/model_server/workflow_ilastik.py
+++ b/extensions/ilastik/workflows.py
@@ -2,13 +2,11 @@
Implementation of image analysis work behind API endpoints, without knowledge of persistent data in server session.
"""
from pathlib import Path
-from time import perf_counter
from typing import Dict
-from model_server.ilastik import IlastikPixelClassifierModel, IlastikObjectClassifierModel
-from model_server.image import generate_file_accessor, write_accessor_data_to_file
-from model_server.model import Model
-from model_server.workflow import Timer
+from extensions.ilastik.models import IlastikPixelClassifierModel, IlastikObjectClassifierModel
+from model_server.accessors import generate_file_accessor, write_accessor_data_to_file
+from model_server.workflows import Timer
from pydantic import BaseModel
diff --git a/model_server/image.py b/model_server/accessors.py
similarity index 100%
rename from model_server/image.py
rename to model_server/accessors.py
diff --git a/model_server/model.py b/model_server/models.py
similarity index 96%
rename from model_server/model.py
rename to model_server/models.py
index a2d75c7b..4285b49c 100644
--- a/model_server/model.py
+++ b/model_server/models.py
@@ -4,7 +4,7 @@ import os
import numpy as np
-from model_server.image import GenericImageDataAccessor, InMemoryDataAccessor
+from model_server.accessors import GenericImageDataAccessor, InMemoryDataAccessor
class Model(ABC):
diff --git a/model_server/session.py b/model_server/session.py
index eb82d1b6..3d5db0bc 100644
--- a/model_server/session.py
+++ b/model_server/session.py
@@ -6,9 +6,9 @@ from time import strftime, localtime
from typing import Dict
import conf.defaults
-from model_server.model import Model
+from model_server.models import Model
from model_server.share import SharedImageDirectory
-from model_server.workflow import WorkflowRunRecord
+from model_server.workflows import WorkflowRunRecord
def create_manifest_json():
pass
@@ -17,9 +17,6 @@ class Session(object):
"""
Singleton class for a server session that persists data between API calls
"""
- # inbound = SharedImageDirectory(conf.defaults.paths['inbound_images'])
- # outbound = SharedImageDirectory(conf.defaults.paths['outbound_images'])
- # where_records = conf.defaults.paths['logs']
def __new__(cls):
if not hasattr(cls, 'instance'):
@@ -31,8 +28,6 @@ class Session(object):
self.models = {} # model_id : model object
self.manifest = [] # paths to data as well as other metadata from each inference run
self.paths = self.make_paths(root)
- # self.session_id = self.create_session_id(self.paths['logs'])
- # self.session_id = self.create_session_id(self.paths)
self.session_log = self.paths['logs'] / f'session.log'
self.log_event('Initialized session')
self.manifest_json = self.paths['logs'] / f'manifest.json'
diff --git a/model_server/share.py b/model_server/share.py
index e270dea6..fad2b2f0 100644
--- a/model_server/share.py
+++ b/model_server/share.py
@@ -1,3 +1,8 @@
+"""
+Classes that manage data sharing between server and outside processes. Currently just a directory on a shared
+filesystem; but may expand to include data transfer over API, shared memory objects, etc.
+"""
+
import os
import pathlib
diff --git a/model_server/workflow.py b/model_server/workflows.py
similarity index 92%
rename from model_server/workflow.py
rename to model_server/workflows.py
index 7c09bc46..c81d2afd 100644
--- a/model_server/workflow.py
+++ b/model_server/workflows.py
@@ -5,8 +5,8 @@ from pathlib import Path
from time import perf_counter
from typing import Dict
-from model_server.image import generate_file_accessor, write_accessor_data_to_file
-from model_server.model import Model
+from model_server.accessors import generate_file_accessor, write_accessor_data_to_file
+from model_server.models import Model
from pydantic import BaseModel
diff --git a/simulate_automic.py b/simulate_automic.py
deleted file mode 100644
index 94e042c8..00000000
--- a/simulate_automic.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from multiprocessing import Process
-import os
-import requests
-import unittest
-
-import uvicorn
-
-import conf.server
-
-# run server in subprocess
-
-host = '127.0.0.1'
-port = 5000
-server_process = Process(
- target=uvicorn.run,
- args=('api:app', ),
- kwargs={'host': host, 'port': port, 'log_level': 'debug'},
- daemon=True
-)
-uri = f'http://{host}:{port}/'
-server_process.start()
-
-# configure and validate folders etc.
-paths = conf.server.paths
-paths['automic_watch_folder'] = paths['images']['inbound']
-paths['test_source_files'] = ''
-
-# load models by API
-
-# prompt user start AutoMic loop
-
-# move files from one folder to another
-def copy_one_input_image_files(copy=True):
- pattern = ''
- os.listdir(paths['test_source_files'])
-
- # order by increaseing alpha order? timestamp?
-
-
- if copy:
- # copy image from test source files to automic watch folder
- else:
- # move them
-
- # recursive repeat until nothign is left
-
-# point to server session logs
-
-
-# shut down server
-server_process.terminate()
-
-
-# do something with the data?
-
diff --git a/tests/test_api.py b/tests/test_api.py
index 40ecfd29..778c378f 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -4,7 +4,7 @@ import requests
import unittest
from conf.testing import czifile
-from model_server.model import DummyImageToImageModel
+from model_server.models import DummyImageToImageModel
class TestServerBaseClass(unittest.TestCase):
def setUp(self) -> None:
diff --git a/tests/test_image.py b/tests/test_image.py
index 6c085b6c..3d590990 100644
--- a/tests/test_image.py
+++ b/tests/test_image.py
@@ -3,7 +3,7 @@ import unittest
import numpy as np
from conf.testing import czifile, output_path
-from model_server.image import CziImageFileAccessor, DataShapeError, InMemoryDataAccessor, write_accessor_data_to_file
+from model_server.accessors import CziImageFileAccessor, DataShapeError, InMemoryDataAccessor, write_accessor_data_to_file
class TestCziImageFileAccess(unittest.TestCase):
diff --git a/tests/test_model.py b/tests/test_model.py
index a265cbe4..661aeadf 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -1,7 +1,7 @@
import unittest
from conf.testing import czifile
-from model_server.image import CziImageFileAccessor
-from model_server.model import DummyImageToImageModel, CouldNotLoadModelError
+from model_server.accessors import CziImageFileAccessor
+from model_server.models import DummyImageToImageModel, CouldNotLoadModelError
class TestCziImageFileAccess(unittest.TestCase):
def setUp(self) -> None:
diff --git a/tests/test_session.py b/tests/test_session.py
index a03e1ad1..fd4b79c9 100644
--- a/tests/test_session.py
+++ b/tests/test_session.py
@@ -1,5 +1,5 @@
import unittest
-from model_server.model import DummyImageToImageModel
+from model_server.models import DummyImageToImageModel
from model_server.session import Session
class TestGetSessionObject(unittest.TestCase):
@@ -38,7 +38,7 @@ class TestGetSessionObject(unittest.TestCase):
def test_session_records_workflow(self):
import json
- from model_server.workflow import WorkflowRunRecord
+ from model_server.workflows import WorkflowRunRecord
sesh = Session()
di = WorkflowRunRecord(
model_id='test_model',
diff --git a/tests/test_workflow.py b/tests/test_workflow.py
index 6bc8d56d..2cab499d 100644
--- a/tests/test_workflow.py
+++ b/tests/test_workflow.py
@@ -1,8 +1,8 @@
import unittest
from conf.testing import czifile, output_path
-from model_server.model import DummyImageToImageModel
-from model_server.workflow import infer_image_to_image
+from model_server.models import DummyImageToImageModel
+from model_server.workflows import infer_image_to_image
class TestGetSessionObject(unittest.TestCase):
--
GitLab