Merge pull request #643 from superannotateai/develop

Develop

Author: VaghinakDev

CHANGELOG.rst

@@ -9,6 +9,14 @@ All release highlights of this project will be documented in this file.
 4.4.15 - August 20, 2023
 _______________________
 
+**Added**
+
+    - Support for `relationship` class types in the document project.
+
+
+4.4.14 - August 20, 2023
+_______________________
+
 **Added**
 
     - New project type support `CustomEditor`.

requirements.txt

@@ -8,9 +8,9 @@ email-validator~=2.0
 pandas~=1.3
 ffmpeg-python~=0.2
 pillow~=9.5
+tqdm~=4.66.1
+requests~=2.31.0
 aiofiles==23.1.0
-requests==2.31.0
-tqdm==4.64.0
 fire==0.4.0
 mixpanel==4.8.3
 jsonschema==3.2.0

src/superannotate/__init__.py

@@ -3,7 +3,7 @@
 import sys
 
 
-__version__ = "4.4.14"
+__version__ = "4.4.15"
 
 sys.path.append(os.path.split(os.path.realpath(__file__))[0])
 

src/superannotate/lib/app/interface/sdk_interface.py

@@ -55,6 +55,8 @@
 from lib.core.entities.classes import AttributeGroup
 from lib.core.entities.integrations import IntegrationEntity
 from lib.core.enums import ImageQuality
+from lib.core.enums import ProjectType
+from lib.core.enums import ClassTypeEnum
 from lib.core.exceptions import AppException
 from lib.core.types import MLModel
 from lib.core.types import PriorityScoreEntity
@@ -72,11 +74,16 @@
 PROJECT_STATUS = Literal["NotStarted", "InProgress", "Completed", "OnHold"]
 
 PROJECT_TYPE = Literal[
-    "Vector", "Pixel", "Video", "Document", "Tiled", "Other", "PointCloud", "CustomEditor"
+    "Vector",
+    "Pixel",
+    "Video",
+    "Document",
+    "Tiled",
+    "Other",
+    "PointCloud",
+    "CustomEditor",
 ]
 
-CLASS_TYPE = Literal["object", "tag"]
-
 ANNOTATION_STATUS = Literal[
     "NotStarted", "InProgress", "QualityCheck", "Returned", "Completed", "Skipped"
 ]
@@ -1310,7 +1317,7 @@ def create_annotation_class(
         name: NotEmptyStr,
         color: NotEmptyStr,
         attribute_groups: Optional[List[AttributeGroup]] = None,
-        class_type: CLASS_TYPE = "object",
+        class_type: str = "object",
     ):
         """Create annotation class in project
 
@@ -1331,7 +1338,7 @@ def create_annotation_class(
               -  "name"
         :type attribute_groups: list of dicts
 
-        :param class_type: class type. Should be either "object" or "tag"
+        :param class_type: class type. Should be either "object" or "tag". Document project type can also have "relationship" type of classes.
         :type class_type: str
 
         :return: new class metadata
@@ -1405,6 +1412,13 @@ def create_annotation_class(
         except ValidationError as e:
             raise AppException(wrap_error(e))
         project = self.controller.projects.get_by_name(project).data
+        if (
+            project.type != ProjectType.DOCUMENT
+            and annotation_class.type == ClassTypeEnum.RELATIONSHIP
+        ):
+            raise AppException(
+                f"{annotation_class.type.name} class type is not supported in {project.type.name} project."
+            )
         response = self.controller.annotation_classes.create(
             project=project, annotation_class=annotation_class
         )
@@ -1514,6 +1528,7 @@ def download_export(
         to_s3_bucket=None,
     ):
         """Download prepared export.
+
         :param project: project name
         :type project: str
         :param export: export name
@@ -2520,6 +2535,9 @@ def attach_items(
                                    “Completed”
                                    “Skipped”
         :type annotation_status: str
+
+        :return: uploaded, failed and duplicated item names
+        :rtype: tuple of list of strs
         """
 
         project_name, folder_name = extract_project_folder(project)

src/superannotate/lib/core/enums.py

@@ -157,6 +157,7 @@ class AnnotationStatus(BaseTitledEnum):
 class ClassTypeEnum(BaseTitledEnum):
     OBJECT = "object", 1
     TAG = "tag", 2
+    RELATIONSHIP = "relationship", 3
 
     @classmethod
     def get_value(cls, name):

src/superannotate/lib/core/serviceproviders.py

@@ -367,7 +367,7 @@ async def upload_small_annotations(
         self,
         project: entities.ProjectEntity,
         folder: entities.FolderEntity,
-        items_name_file_map: Dict[str, io.StringIO],
+        items_name_data_map: Dict[str, dict],
     ) -> UploadAnnotationsResponse:
         raise NotImplementedError
 

src/superannotate/lib/core/usecases/annotations.py

@@ -47,6 +47,7 @@
 from lib.core.types import PriorityScoreEntity
 from lib.core.usecases.base import BaseReportableUseCase
 from lib.core.video_convertor import VideoFrameGenerator
+from lib.infrastructure.utils import divide_to_chunks
 from pydantic import BaseModel
 
 logger = logging.getLogger("sa")
@@ -119,11 +120,6 @@ def get_or_raise(response: ServiceResponse):
         raise AppException(response.error)
 
 
-def divide_to_chunks(it, size):
-    it = iter(it)
-    return iter(lambda: tuple(islice(it, size)), ())
-
-
 def log_report(
     report: Report,
 ):
@@ -148,7 +144,6 @@ class ItemToUpload(BaseModel):
     item: BaseItemEntity
     annotation_json: Optional[dict]
     path: Optional[str]
-    file: Optional[io.StringIO]
     file_size: Optional[int]
     mask: Optional[io.BytesIO]
 
@@ -186,7 +181,7 @@ async def upload_small_annotations(
     report: Report,
     callback: Callable = None,
 ):
-    async def upload(_chunk):
+    async def upload(_chunk: List[ItemToUpload]):
         failed_annotations, missing_classes, missing_attr_groups, missing_attrs = (
             [],
             [],
@@ -197,7 +192,7 @@ async def upload(_chunk):
             response = await service_provider.annotations.upload_small_annotations(
                 project=project,
                 folder=folder,
-                items_name_file_map={i.item.name: i.file for i in chunk},
+                items_name_data_map={i.item.name: i.annotation_json for i in chunk},
             )
             if response.ok:
                 if response.data.failed_items:  # noqa
@@ -221,9 +216,9 @@ async def upload(_chunk):
             reporter.update_progress(len(chunk))
 
     _size = 0
-    chunk = []
+    chunk: List[ItemToUpload] = []
     while True:
-        item_data = await queue.get()
+        item_data: ItemToUpload = await queue.get()
         queue.task_done()
         if not item_data:
             queue.put_nowait(None)
@@ -253,11 +248,14 @@ async def upload_big_annotations(
 ):
     async def _upload_big_annotation(item_data: ItemToUpload) -> Tuple[str, bool]:
         try:
+            buff = io.StringIO()
+            json.dump(item_data.annotation_json, buff, allow_nan=False)
+            buff.seek(0)
             is_uploaded = await service_provider.annotations.upload_big_annotation(
                 project=project,
                 folder=folder,
                 item_id=item_data.item.id,
-                data=item_data.file,
+                data=buff,
                 chunk_size=5 * 1024 * 1024,
             )
             if is_uploaded and callback:
@@ -335,15 +333,14 @@ async def distribute_queues(self, items_to_upload: List[ItemToUpload]):
             for idx, (item_to_upload, processed) in enumerate(data):
                 if not processed:
                     try:
-                        item_to_upload.file = io.StringIO()
+                        file = io.StringIO()
                         json.dump(
                             item_to_upload.annotation_json,
-                            item_to_upload.file,
+                            file,
                             allow_nan=False,
                         )
-                        item_to_upload.file.seek(0, os.SEEK_END)
-                        item_to_upload.file_size = item_to_upload.file.tell()
-                        item_to_upload.file.seek(0)
+                        file.seek(0, os.SEEK_END)
+                        item_to_upload.file_size = file.tell()
                         while True:
                             if item_to_upload.file_size > BIG_FILE_THRESHOLD:
                                 if self._big_files_queue.qsize() > 32:
@@ -723,13 +720,10 @@ async def distribute_queues(self, items_to_upload: List[ItemToUpload]):
                 if not processed:
                     try:
                         (
-                            annotation,
+                            item_to_upload.annotation_json,
                             item_to_upload.mask,
                             item_to_upload.file_size,
                         ) = await self.get_annotation(item_to_upload.path)
-                        item_to_upload.file = io.StringIO()
-                        json.dump(annotation, item_to_upload.file, allow_nan=False)
-                        item_to_upload.file.seek(0)
                         while True:
                             if item_to_upload.file_size > BIG_FILE_THRESHOLD:
                                 if self._big_files_queue.qsize() > 32:
@@ -1018,7 +1012,7 @@ def execute(self):
                         self._service_provider.annotations.upload_small_annotations(
                             project=self._project,
                             folder=self._folder,
-                            items_name_file_map={self._image.name: annotation_file},
+                            items_name_data_map={self._image.name: annotation_json},
                         )
                     )
                     if response.ok: