Merge branch 'master' of https://github.com/superannotateai/superannotate-python-sdk

Author: agyurjin

Makefile

@@ -1,12 +1,14 @@
 .PHONY: clean tests stress-tests test_coverage install lint docs dist check_formatting
 
+MAKEFLAGS += -j1
+
 PYTHON=python3
 PYLINT=pylint
 PYTESTS=pytest
 COVERAGE=coverage
 
 tests: check_formatting docs
-	$(PYTESTS) -n auto tests
+	$(PYTESTS) -n 8 --full-trace tests
 
 stress-tests: SA_STRESS_TESTS=1
 stress-tests: tests

superannotate/__init__.py

@@ -89,6 +89,11 @@ def consensus(*args, **kwargs):
     coco_split_dataset, convert_project_type, export_annotation,
     import_annotation, convert_json_version
 )
+from .ml.ml_funcs import (
+    delete_model, download_model, plot_model_metrics, run_prediction,
+    run_segmentation, run_training, stop_model_training
+)
+from .ml.ml_models import search_models
 
 from .old_to_new_format_convertor import update_json_format
 from .version import __version__

superannotate/api.py

@@ -123,6 +123,11 @@ def send_request(self, req_type, path, params=None, json_req=None):
             )
         url = self._main_endpoint + path
 
+        if params is not None:
+            for key, value in params.items():
+                if isinstance(value, str):
+                    params[key] = value.replace("\\", "\\\\")
+
         req = requests.Request(
             method=req_type, url=url, json=json_req, params=params
         )

superannotate/common.py

@@ -6,13 +6,14 @@
 import time
 from tqdm import tqdm
 from pathlib import Path
-
+from enum import IntEnum
 import numpy as np
 from PIL import Image
 
 logger = logging.getLogger("superannotate-python-sdk")
 
 _PROJECT_TYPES = {"Vector": 1, "Pixel": 2}
+
 _ANNOTATION_STATUSES = {
     "NotStarted": 1,
     "InProgress": 2,
@@ -22,6 +23,44 @@
     "Skipped": 6
 }
 _USER_ROLES = {"Admin": 2, "Annotator": 3, "QA": 4, "Customer": 5, "Viewer": 6}
+_AVAILABLE_SEGMENTATION_MODELS = ['autonomous', 'generic']
+_MODEL_TRAINING_STATUSES = {
+    "NotStarted": 1,
+    "InProgress": 2,
+    "Completed": 3,
+    "FailedBeforeEvaluation": 4,
+    "FailedAfterEvaluation": 5,
+    "FailedAfterEvaluationWithSavedModel": 6
+}
+
+_PREDICTION_SEGMENTATION_STATUSES = {
+    "NotStarted": 1,
+    "InProgress": 2,
+    "Completed": 3,
+    "Failed": 4
+}
+
+
+def prediction_segmentation_status_from_str_to_int(status):
+    return _PREDICTION_SEGMENTATION_STATUSES[status]
+
+
+def prediction_segmentation_status_from_int_to_str(status):
+    for idx, item in _PREDICTION_SEGMENTATION_STATUSES.items():
+        if item == status:
+            return idx
+    raise RuntimeError("NA segmentation/prediction status")
+
+
+def model_training_status_int_to_str(project_status):
+    for item in _MODEL_TRAINING_STATUSES:
+        if _MODEL_TRAINING_STATUSES[item] == project_status:
+            return item
+    raise RuntimeError("NA training status")
+
+
+def model_training_status_str_to_int(project_status):
+    return _MODEL_TRAINING_STATUSES[project_status]
 
 
 def image_path_to_annotation_paths(image_path, project_type):
@@ -257,6 +296,14 @@ def write_to_json(output_path, json_data):
 }  # Resolution limit
 
 
+def process_api_response(data):
+    logger.info("Maintenance operations are being performed")
+    if 'metadata' not in data:
+        return data
+
+    return data['data']
+
+
 def tqdm_converter(
     total_num, images_converted, images_not_converted, finish_event
 ):

superannotate/db/images.py

@@ -24,6 +24,8 @@
     search_annotation_classes
 )
 from .project_api import get_project_metadata_bare
+from ..common import process_api_response
+from ..parameter_decorators import project_metadata
 
 logger = logging.getLogger("superannotate-python-sdk")
 
@@ -43,7 +45,10 @@ def _get_project_root_folder_id(project):
     )
     if not response.ok:
         raise SABaseException(response.status_code, response.text)
-    return response.json()['folder_id']
+
+    response = process_api_response(response.json())
+
+    return response['folder_id']
 
 
 def search_images(
@@ -94,14 +99,16 @@ def search_images(
         )
         if response.ok:
             # print(response.json())
-            results = response.json()["data"]
+            response = process_api_response(response.json())
+            results = response["data"]
             total_got += len(results)
             for r in results:
                 if return_metadata:
                     result_list.append(r)
                 else:
                     result_list.append(r["name"])
-            if response.json()["count"] <= total_got:
+
+            if response["count"] <= total_got:
                 break
             params["offset"] = total_got
             # print(
@@ -126,7 +133,8 @@ def process_result(x):
         return result_list
 
 
-def get_image_metadata(project, image_name):
+@project_metadata
+def get_image_metadata(project, image_names):
     """Returns image metadata
 
     :param project: project name or metadata of the project
@@ -137,11 +145,42 @@ def get_image_metadata(project, image_name):
     :return: metadata of image
     :rtype: dict
     """
-    images = search_images(project, image_name, return_metadata=True)
-    for image in images:
-        if image["name"] == image_name:
-            return image
-    raise SABaseException(0, "Image " + image_name + " doesn't exist.")
+    if isinstance(image_names, str):
+        image_names = [image_names]
+
+    json_req = {
+        'project_id': project['id'],
+        'team_id': _api.team_id,
+        'names': image_names
+    }
+    response = _api.send_request(
+        req_type='POST',
+        path='/images/getBulk',
+        json_req=json_req,
+    )
+
+    metadata = response.json()
+    if len(metadata) == 0:
+        raise SABaseException(
+            0,
+            f"None of the images in {image_names} exist in the provided project"
+        )
+    for item in metadata:
+        item['annotation_status'] = common.annotation_status_int_to_str(
+            item['annotation_status']
+        )
+        item['prediction_status'
+            ] = common.prediction_segmentation_status_from_int_to_str(
+                item['prediction_status']
+            )
+        item['segmentation_status'
+            ] = common.prediction_segmentation_status_from_int_to_str(
+                item['segmentation_status']
+            )
+
+    if len(metadata) == 1:
+        return metadata[0]
+    return metadata
 
 
 def set_image_annotation_status(project, image_name, annotation_status):
@@ -174,7 +213,10 @@ def set_image_annotation_status(project, image_name, annotation_status):
     )
     if not response.ok:
         raise SABaseException(response.status_code, response.text)
-    return response.json()
+
+    response = process_api_response(response.json())
+
+    return response
 
 
 def add_annotation_comment_to_image(

superannotate/db/project_api.py

@@ -12,7 +12,7 @@
 _api = API.get_instance()
 
 
-def get_project_metadata_bare(project_name):
+def get_project_metadata_bare(project_name, include_complete_image_count=False):
     """Returns project metadata
 
     :param project_name: project name
@@ -21,7 +21,11 @@ def get_project_metadata_bare(project_name):
     :return: metadata of project
     :rtype: dict
     """
-    projects = search_projects(project_name, return_metadata=True)
+    projects = search_projects(
+        project_name,
+        return_metadata=True,
+        include_complete_image_count=include_complete_image_count
+    )
     results = []
     for project in projects:
         if project["name"] == project_name:

superannotate/db/project_images.py

@@ -181,6 +181,7 @@ def copy_image(
             else:
                 new_name = new_name[:m.start() +
                                     2] + str(num + 1) + ")" + extension
+
     upload_image_to_project(destination_project, img_b, new_name)
     if include_annotations:
         annotations = get_image_annotations(source_project, image_name)

superannotate/db/project_metadata.py

@@ -15,7 +15,8 @@ def get_project_metadata(
     include_annotation_classes=False,
     include_settings=False,
     include_workflow=False,
-    include_contributors=False
+    include_contributors=False,
+    include_complete_image_count=False
 ):
     """Returns project metadata
 
@@ -38,7 +39,9 @@ def get_project_metadata(
     :rtype: dict
     """
     if not isinstance(project, dict):
-        project = get_project_metadata_bare(project)
+        project = get_project_metadata_bare(
+            project, include_complete_image_count
+        )
     result = copy.deepcopy(project)
     if include_annotation_classes:
         result["annotation_classes"] = search_annotation_classes(project)

superannotate/db/search_projects.py

@@ -2,12 +2,14 @@
 
 from ..api import API
 from ..exceptions import (SABaseException)
-
+from ..common import process_api_response
 logger = logging.getLogger("superannotate-python-sdk")
 _api = API.get_instance()
 
 
-def search_projects(name=None, return_metadata=False):
+def search_projects(
+    name=None, return_metadata=False, include_complete_image_count=False
+):
     """Project name based case-insensitive search for projects.
     If **name** is None, all the projects will be returned.
 
@@ -20,17 +22,21 @@ def search_projects(name=None, return_metadata=False):
     :rtype: list of strs or dicts
     """
     result_list = []
-    params = {'team_id': str(_api.team_id), 'offset': 0}
+    params = {
+        'team_id': str(_api.team_id),
+        'offset': 0,
+        'completeImagesCount': include_complete_image_count
+    }
     if name is not None:
         params['name'] = name
     while True:
         response = _api.send_request(
             req_type='GET', path='/projects', params=params
         )
         if response.ok:
-            new_results = response.json()
+            new_results = process_api_response(response.json())
             result_list += new_results["data"]
-            if response.json()["count"] <= len(result_list):
+            if new_results["count"] <= len(result_list):
                 break
             params["offset"] = len(result_list)
         else: