API test cases now call PUT through an internal method with re-attempts built...

API test cases now call PUT through an internal method with re-attempts built in; trying to eliminate lost connections during testing but some still happen

model_server/extensions/ilastik/tests/test_ilastik.py

@@ -130,17 +130,17 @@ class TestIlastikPixelClassification(unittest.TestCase):
 class TestIlastikOverApi(TestServerBaseClass):
 
     def test_httpexception_if_incorrect_project_file_loaded(self):
-        resp_load = requests.put(
-            self.uri + 'ilastik/seg/load/',
-            params={'project_file': 'improper.ilp'},
+        resp_load = self._put(
+            'ilastik/seg/load/',
+            {'project_file': 'improper.ilp'},
         )
         self.assertEqual(resp_load.status_code, 404)
 
 
     def test_load_ilastik_pixel_model(self):
-        resp_load = requests.put(
-            self.uri + 'ilastik/seg/load/',
-            params={'project_file': str(ilastik_classifiers['px'])},
+        resp_load = self._put(
+            'ilastik/seg/load/',
+            {'project_file': str(ilastik_classifiers['px'])},
         )
         self.assertEqual(resp_load.status_code, 200, resp_load.json())
         model_id = resp_load.json()['model_id']
@@ -154,21 +154,15 @@ class TestIlastikOverApi(TestServerBaseClass):
         model_id = self.test_load_ilastik_pixel_model()
         resp_list_1st = requests.get(self.uri + 'models').json()
         self.assertEqual(len(resp_list_1st), 1, resp_list_1st)
-        resp_load_2nd = requests.put(
-            self.uri + 'ilastik/seg/load/',
-            params={
-                'project_file': str(ilastik_classifiers['px']),
-                'duplicate': True,
-            },
+        resp_load_2nd = self._put(
+            'ilastik/seg/load/',
+            {'project_file': str(ilastik_classifiers['px']), 'duplicate': True, },
         )
         resp_list_2nd = requests.get(self.uri + 'models').json()
         self.assertEqual(len(resp_list_2nd), 2, resp_list_2nd)
-        resp_load_3rd = requests.put(
-            self.uri + 'ilastik/seg/load/',
-            params={
-                'project_file': str(ilastik_classifiers['px']),
-                'duplicate': False,
-            },
+        resp_load_3rd = self._put(
+            'ilastik/seg/load/',
+            {'project_file': str(ilastik_classifiers['px']), 'duplicate': False},
         )
         resp_list_3rd = requests.get(self.uri + 'models').json()
         self.assertEqual(len(resp_list_3rd), 2, resp_list_3rd)
@@ -189,19 +183,13 @@ class TestIlastikOverApi(TestServerBaseClass):
         self.assertEqual(pathlib.Path(ilp_win), pathlib.Path(ilp_posx))
 
         # load models with these paths
-        resp1 = requests.put(
-            self.uri + 'ilastik/seg/load/',
-            params={
-                'project_file': ilp_win,
-                'duplicate': False,
-            },
+        resp1 = self._put(
+            'ilastik/seg/load/',
+            {'project_file': ilp_win, 'duplicate': False },
         )
-        resp2 = requests.put(
-            self.uri + 'ilastik/seg/load/',
-            params={
-                'project_file': ilp_posx,
-                'duplicate': False,
-            },
+        resp2 = self._put(
+            'ilastik/seg/load/',
+            {'project_file': ilp_posx, 'duplicate': False},
         )
         self.assertEqual(resp1.json(), resp2.json())
 
@@ -212,9 +200,9 @@ class TestIlastikOverApi(TestServerBaseClass):
 
 
     def test_load_ilastik_pxmap_to_obj_model(self):
-        resp_load = requests.put(
-            self.uri + 'ilastik/pxmap_to_obj/load/',
-            params={'project_file': str(ilastik_classifiers['pxmap_to_obj'])},
+        resp_load = self._put(
+            'ilastik/pxmap_to_obj/load/',
+            {'project_file': str(ilastik_classifiers['pxmap_to_obj'])},
         )
         model_id = resp_load.json()['model_id']
 
@@ -226,9 +214,9 @@ class TestIlastikOverApi(TestServerBaseClass):
         return model_id
 
     def test_load_ilastik_seg_to_obj_model(self):
-        resp_load = requests.put(
-            self.uri + 'ilastik/seg_to_obj/load/',
-            params={'project_file': str(ilastik_classifiers['seg_to_obj'])},
+        resp_load = self._put(
+            'ilastik/seg_to_obj/load/',
+            {'project_file': str(ilastik_classifiers['seg_to_obj'])},
         )
         model_id = resp_load.json()['model_id']
 
@@ -243,13 +231,9 @@ class TestIlastikOverApi(TestServerBaseClass):
         self.copy_input_file_to_server()
         model_id = self.test_load_ilastik_pixel_model()
 
-        resp_infer = requests.put(
-            self.uri + f'workflows/segment',
-            params={
-                'model_id': model_id,
-                'input_filename': czifile['filename'],
-                'channel': 0,
-            },
+        resp_infer = self._put(
+            f'workflows/segment',
+            {'model_id': model_id, 'input_filename': czifile['filename'], 'channel': 0},
         )
         self.assertEqual(resp_infer.status_code, 200, resp_infer.content.decode())
 
@@ -258,9 +242,9 @@ class TestIlastikOverApi(TestServerBaseClass):
         px_model_id = self.test_load_ilastik_pixel_model()
         ob_model_id = self.test_load_ilastik_pxmap_to_obj_model()
 
-        resp_infer = requests.put(
-            self.uri + f'ilastik/pixel_then_object_classification/infer/',
-            params={
+        resp_infer = self._put(
+            'ilastik/pixel_then_object_classification/infer/',
+            {
                 'px_model_id': px_model_id,
                 'ob_model_id': ob_model_id,
                 'input_filename': czifile['filename'],

tests/test_api.py

@@ -2,6 +2,7 @@ from multiprocessing import Process
 from pathlib import Path
 import requests
 import unittest
+from urllib3 import Retry
 
 from model_server.conf.testing import czifile
 
@@ -24,6 +25,15 @@ class TestServerBaseClass(unittest.TestCase):
         self.server_process.start()
         requests.get(self.uri + 'restart')
 
+    def _put(self, endpoint, params=None):
+        sesh = requests.Session()
+        retries = Retry(
+            total=5,
+            backoff_factor=0.1,
+        )
+        sesh.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
+        return sesh.put(self.uri + endpoint, params=params)
+
     def copy_input_file_to_server(self):
         from shutil import copyfile
 
@@ -44,7 +54,7 @@ class TestApiFromAutomatedClient(TestServerBaseClass):
         self.assertEqual(resp.status_code, 200)
 
     def test_bounceback_parameters(self):
-        resp = requests.put(self.uri + 'bounce_back', params={'par1': 'hello'})
+        resp = self._put('bounce_back', {'par1': 'hello'})
         self.assertEqual(resp.status_code, 200, resp.json())
         self.assertEqual(resp.json()['params']['par1'], 'hello', resp.json())
         self.assertEqual(resp.json()['params']['par2'], None, resp.json())
@@ -64,9 +74,7 @@ class TestApiFromAutomatedClient(TestServerBaseClass):
         self.assertEqual(resp.content, b'{}')
 
     def test_load_dummy_model(self):
-        resp_load = requests.put(
-            self.uri + f'models/dummy/load',
-        )
+        resp_load = self._put(f'models/dummy/load')
         model_id = resp_load.json()['model_id']
         self.assertEqual(resp_load.status_code, 200, resp_load.json())
         resp_list = requests.get(self.uri + 'models')
@@ -78,33 +86,27 @@ class TestApiFromAutomatedClient(TestServerBaseClass):
     def test_respond_with_error_when_invalid_filepath_requested(self):
         model_id = self.test_load_dummy_model()
 
-        resp = requests.put(
-            self.uri + f'infer/from_image_file',
-            params={
-                'model_id': model_id,
-                'input_filename': 'not_a_real_file.name'
-            }
+        resp = self._put(
+            f'infer/from_image_file',
+            {'model_id': model_id, 'input_filename': 'not_a_real_file.name'}
         )
         self.assertEqual(resp.status_code, 404, resp.content.decode())
 
 
     def test_i2i_inference_errors_when_model_not_found(self):
         model_id = 'not_a_real_model'
-        resp = requests.put(
-            self.uri + f'workflows/segment',
-            params={
-                'model_id': model_id,
-                'input_filename': 'not_a_real_file.name'
-            }
+        resp = self._put(
+            f'workflows/segment',
+            {'model_id': model_id, 'input_filename': 'not_a_real_file.name'}
         )
         self.assertEqual(resp.status_code, 409, resp.content.decode())
 
     def test_i2i_dummy_inference_by_api(self):
         model_id = self.test_load_dummy_model()
         self.copy_input_file_to_server()
-        resp_infer = requests.put(
-            self.uri + f'workflows/segment',
-            params={
+        resp_infer = self._put(
+            f'workflows/segment',
+            {
                 'model_id': model_id,
                 'input_filename': czifile['filename'],
                 'channel': 2,
@@ -113,9 +115,7 @@ class TestApiFromAutomatedClient(TestServerBaseClass):
         self.assertEqual(resp_infer.status_code, 200, resp_infer.content.decode())
 
     def test_restarting_session_clears_loaded_models(self):
-        resp_load = requests.put(
-            self.uri + f'models/dummy/load',
-        )
+        resp_load = self._put(f'models/dummy/load',)
         self.assertEqual(resp_load.status_code, 200, resp_load.json())
         resp_list_0 = requests.get(self.uri + 'models')
         self.assertEqual(resp_list_0.status_code, 200)
@@ -130,11 +130,9 @@ class TestApiFromAutomatedClient(TestServerBaseClass):
         resp_inpath = requests.get(
             self.uri + 'paths'
         )
-        resp_change = requests.put(
-            self.uri + f'paths/watch_output',
-            params={
-                'path': resp_inpath.json()['inbound_images']
-            }
+        resp_change = self._put(
+            f'paths/watch_output',
+            {'path': resp_inpath.json()['inbound_images']}
         )
         self.assertEqual(resp_change.status_code, 200)
         resp_check = requests.get(
@@ -147,11 +145,9 @@ class TestApiFromAutomatedClient(TestServerBaseClass):
             self.uri + 'paths'
         )
         fakepath = 'c:/fake/path/to/nowhere'
-        resp_change = requests.put(
-            self.uri + f'paths/watch_output',
-            params={
-                'path': fakepath,
-            }
+        resp_change = self._put(
+            f'paths/watch_output',
+            {'path': fakepath}
         )
         self.assertEqual(resp_change.status_code, 404)
         self.assertIn(fakepath, resp_change.json()['detail'])
@@ -164,11 +160,9 @@ class TestApiFromAutomatedClient(TestServerBaseClass):
         resp_inpath = requests.get(
             self.uri + 'paths'
         )
-        resp_change = requests.put(
-            self.uri + f'paths/watch_output',
-            params={
-                'path': resp_inpath.json()['outbound_images']
-            }
+        resp_change = self._put(
+            f'paths/watch_output',
+            {'path': resp_inpath.json()['outbound_images']}
         )
         self.assertEqual(resp_change.status_code, 200)
         resp_check = requests.get(