Merge pull request #672 from superannotateai/develop

Develop

Author: VaghinakDev

CHANGELOG.rst

@@ -6,6 +6,15 @@ History
 
 All release highlights of this project will be documented in this file.
 
+4.4.19 - February 08, 2024
+__________________________
+
+
+**Updated**
+
+    - ``SAClient.attach_items()`` added the ability to attach items from custom integrated storage.
+
+
 4.4.18 - January 18, 2024
 __________________________
 

src/superannotate/__init__.py

@@ -3,7 +3,7 @@
 import sys
 
 
-__version__ = "4.4.18"
+__version__ = "4.4.19"
 
 sys.path.append(os.path.split(os.path.realpath(__file__))[0])
 

src/superannotate/lib/app/helpers.py

@@ -6,6 +6,7 @@
 from typing import Union
 
 import boto3
+import numpy as np
 import pandas as pd
 from superannotate.lib.app.exceptions import AppException
 from superannotate.lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX
@@ -78,6 +79,7 @@ def get_s3_annotation_paths(folder_path, s3_bucket, annotation_paths, recursive)
 
 def get_name_url_duplicated_from_csv(csv_path):
     image_data = pd.read_csv(csv_path, dtype=str)
+    image_data.replace({pd.NA: None}, inplace=True)
     if "url" not in image_data.columns:
         raise AppException("Column 'url' is required")
     image_data = image_data[~image_data["url"].isnull()]
@@ -90,7 +92,8 @@ def get_name_url_duplicated_from_csv(csv_path):
     else:
         image_data["name"] = [str(uuid.uuid4()) for _ in range(len(image_data.index))]
 
-    image_data = pd.DataFrame(image_data, columns=["name", "url"])
+    image_data = pd.DataFrame(image_data, columns=["name", "url", "integration"])
+    image_data = image_data.replace(np.nan, None)
     img_names_urls = image_data.to_dict(orient="records")
     duplicate_images = []
     seen = []

src/superannotate/lib/app/interface/sdk_interface.py

@@ -50,6 +50,7 @@
 from lib.core.entities.classes import AnnotationClassEntity
 from lib.core.entities.classes import AttributeGroup
 from lib.core.entities.integrations import IntegrationEntity
+from lib.core.entities.integrations import IntegrationTypeEnum
 from lib.core.enums import ImageQuality
 from lib.core.enums import ProjectType
 from lib.core.enums import ClassTypeEnum
@@ -64,7 +65,6 @@
 from lib.infrastructure.utils import extract_project_folder
 from lib.infrastructure.validators import wrap_error
 
-
 logger = logging.getLogger("sa")
 
 NotEmptyStr = TypeVar("NotEmptyStr", bound=constr(strict=True, min_length=1))
@@ -77,7 +77,6 @@
     "Video",
     "Document",
     "Tiled",
-    "Other",
     "PointCloud",
     "GenAI",
 ]
@@ -110,6 +109,7 @@ class PriorityScore(TypedDict):
 class Attachment(TypedDict, total=False):
     url: Required[str]  # noqa
     name: NotRequired[str]  # noqa
+    integration: NotRequired[str]  # noqa
 
 
 class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta):
@@ -311,7 +311,7 @@ def create_project(
         :param project_description: the new project's description
         :type project_description: str
 
-        :param project_type: the new project type, Vector, Pixel, Video, Document, Tiled, PointCloud, Other.
+        :param project_type: the new project type, Vector, Pixel, Video, Document, Tiled, PointCloud, GenAI.
         :type project_type: str
 
         :param settings: list of settings objects
@@ -2559,6 +2559,20 @@ def attach_items(
                 attachments = [{"name": "item", "url": "https://..."}]
             )
 
+        Example of attaching items from custom integration:
+        ::
+            client = SAClient()
+            client.attach_items(
+                project = "Medical Annotations",
+                attachments =  [
+                    {
+                        "name": "item",
+                        "url": "https://sa-public-files.s3.../text_file_example_1.jpeg"
+                        "integration": "custom-integration"
+                        }
+                    ]
+            )
+
         """
 
         project_name, folder_name = extract_project_folder(project)
@@ -2579,25 +2593,51 @@ def attach_items(
             logger.info("Dropping duplicates.")
         unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments)
         uploaded, fails, duplicated = [], [], []
-        if unique_attachments:
+        _unique_attachments = []
+        if any(i.integration for i in unique_attachments):
+            integtation_item_map = {
+                i.name: i
+                for i in self.controller.integrations.list().data
+                if i.type == IntegrationTypeEnum.CUSTOM
+            }
+            invalid_integrations = set()
+            for attachment in unique_attachments:
+                if attachment.integration:
+                    if attachment.integration in integtation_item_map:
+                        attachment.integration_id = integtation_item_map[
+                            attachment.integration
+                        ].id
+                    else:
+                        invalid_integrations.add(attachment.integration)
+                        continue
+                _unique_attachments.append(attachment)
+            if invalid_integrations:
+                logger.error(
+                    f"The ['{','.join(invalid_integrations)}'] integrations specified for the items doesn't exist in the "
+                    "list of integrations on the platform. Any associated items will be skipped."
+                )
+        else:
+            _unique_attachments = unique_attachments
+
+        if _unique_attachments:
             logger.info(
-                f"Attaching {len(unique_attachments)} file(s) to project {project}."
+                f"Attaching {len(_unique_attachments)} file(s) to project {project}."
             )
             project, folder = self.controller.get_project_folder(
                 project_name, folder_name
             )
             response = self.controller.items.attach(
                 project=project,
                 folder=folder,
-                attachments=unique_attachments,
+                attachments=_unique_attachments,
                 annotation_status=annotation_status,
             )
             if response.errors:
                 raise AppException(response.errors)
             uploaded, duplicated = response.data
             fails = [
                 attachment.name
-                for attachment in unique_attachments
+                for attachment in _unique_attachments
                 if attachment.name not in uploaded and attachment.name not in duplicated
             ]
         return uploaded, fails, duplicated

src/superannotate/lib/core/__init__.py

@@ -99,7 +99,6 @@ def setup_logging(level=DEFAULT_LOGGING_LEVEL, file_path=LOG_FILE_LOCATION):
 LIMITED_FUNCTIONS = {
     ProjectType.VIDEO: DEPRECATED_VIDEO_PROJECTS_MESSAGE,
     ProjectType.DOCUMENT: DEPRECATED_DOCUMENT_PROJECTS_MESSAGE,
-    ProjectType.OTHER: DEPRECATED_PROJECTS_MESSAGE,
     ProjectType.POINT_CLOUD: DEPRECATED_PROJECTS_MESSAGE,
     ProjectType.TILED: DEPRECATED_PROJECTS_MESSAGE,
 }

src/superannotate/lib/core/entities/project.py

@@ -40,6 +40,8 @@ class TimedBaseModel(BaseModel):
 class AttachmentEntity(BaseModel):
     name: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4()))
     url: str
+    integration: Optional[str] = None
+    integration_id: Optional[int] = None
 
     class Config:
         extra = Extra.ignore

src/superannotate/lib/core/enums.py

@@ -171,6 +171,7 @@ class IntegrationTypeEnum(BaseTitledEnum):
     AWS = "aws", 1
     GCP = "gcp", 2
     AZURE = "azure", 3
+    CUSTOM = "custom", 4
 
 
 class TrainingStatus(BaseTitledEnum):

src/superannotate/lib/core/types.py

@@ -33,8 +33,10 @@ class PriorityScoreEntity(BaseModel):
 class Attachment(BaseModel):
     name: str
     path: str
+    integration_id: Optional[int] = None
 
 
 class AttachmentMeta(BaseModel):
     width: Optional[float] = None
     height: Optional[float] = None
+    integration_id: Optional[int] = None

src/superannotate/lib/core/usecases/items.py

@@ -410,8 +410,8 @@ def validate_upload_state(self):
             raise AppValidationException(constants.ATTACHING_UPLOAD_STATE_ERROR)
 
     @staticmethod
-    def generate_meta() -> AttachmentMeta:
-        return AttachmentMeta(width=None, height=None)
+    def generate_meta(integration_id=None) -> AttachmentMeta:
+        return AttachmentMeta(width=None, height=None, integration_id=integration_id)
 
     def execute(self) -> Response:
         if self.is_valid():
@@ -436,7 +436,9 @@ def execute(self) -> Response:
                         to_upload.append(
                             Attachment(name=attachment.name, path=attachment.url)
                         )
-                        to_upload_meta[attachment.name] = self.generate_meta()
+                        to_upload_meta[attachment.name] = self.generate_meta(
+                            attachment.integration_id
+                        )
                 if to_upload:
                     backend_response = self._service_provider.items.attach(
                         project=self._project,

src/superannotate/lib/infrastructure/services/item.py

@@ -40,7 +40,7 @@ class ItemService(BaseItemService):
         ProjectType.PIXEL: ImageResponse,
         ProjectType.DOCUMENT: DocumentResponse,
         ProjectType.POINT_CLOUD: PointCloudResponse,
-        ProjectType.GEN_AI: ImageResponse
+        ProjectType.GEN_AI: ImageResponse,
     }
 
     def get_by_id(self, item_id, project_id, project_type):