Added integration id supperot for attach_items

Author: VaghinakDev

pytest.ini

@@ -3,5 +3,5 @@ minversion = 3.7
 log_cli=true
 python_files = test_*.py
 ;pytest_plugins = ['pytest_profiling']
-addopts = -n auto --dist=loadscope
+;addopts = -n auto --dist=loadscope
 

src/superannotate/__init__.py

@@ -3,7 +3,7 @@
 import sys
 
 
-__version__ = "4.4.18"
+__version__ = "4.4.19dev1"
 
 sys.path.append(os.path.split(os.path.realpath(__file__))[0])
 

src/superannotate/lib/app/helpers.py

@@ -78,6 +78,7 @@ def get_s3_annotation_paths(folder_path, s3_bucket, annotation_paths, recursive)
 
 def get_name_url_duplicated_from_csv(csv_path):
     image_data = pd.read_csv(csv_path, dtype=str)
+    image_data.replace({pd.NA: None}, inplace=True)
     if "url" not in image_data.columns:
         raise AppException("Column 'url' is required")
     image_data = image_data[~image_data["url"].isnull()]
@@ -90,7 +91,7 @@ def get_name_url_duplicated_from_csv(csv_path):
     else:
         image_data["name"] = [str(uuid.uuid4()) for _ in range(len(image_data.index))]
 
-    image_data = pd.DataFrame(image_data, columns=["name", "url"])
+    image_data = pd.DataFrame(image_data, columns=["name", "url", "integration"])
     img_names_urls = image_data.to_dict(orient="records")
     duplicate_images = []
     seen = []

src/superannotate/lib/app/interface/sdk_interface.py

@@ -50,6 +50,7 @@
 from lib.core.entities.classes import AnnotationClassEntity
 from lib.core.entities.classes import AttributeGroup
 from lib.core.entities.integrations import IntegrationEntity
+from lib.core.entities.integrations import IntegrationTypeEnum
 from lib.core.enums import ImageQuality
 from lib.core.enums import ProjectType
 from lib.core.enums import ClassTypeEnum
@@ -64,7 +65,6 @@
 from lib.infrastructure.utils import extract_project_folder
 from lib.infrastructure.validators import wrap_error
 
-
 logger = logging.getLogger("sa")
 
 NotEmptyStr = TypeVar("NotEmptyStr", bound=constr(strict=True, min_length=1))
@@ -110,6 +110,7 @@ class PriorityScore(TypedDict):
 class Attachment(TypedDict, total=False):
     url: Required[str]  # noqa
     name: NotRequired[str]  # noqa
+    integration: NotRequired[str]  # noqa
 
 
 class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta):
@@ -2579,25 +2580,48 @@ def attach_items(
             logger.info("Dropping duplicates.")
         unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments)
         uploaded, fails, duplicated = [], [], []
-        if unique_attachments:
+        _unique_attachments = []
+        if any(i.integration for i in unique_attachments):
+            integtation_item_map = {
+                i.name: i
+                for i in self.controller.integrations.list().data
+                if i.type == IntegrationTypeEnum.CUSTOM
+            }
+            invalid_integrations = set()
+            for attachment in unique_attachments:
+                if attachment.integration:
+                    if attachment.integration in integtation_item_map:
+                        attachment.integration_id = integtation_item_map[
+                            attachment.integration
+                        ].id
+                    else:
+                        invalid_integrations.add(attachment.integration)
+                        continue
+                _unique_attachments.append(attachment)
+            if invalid_integrations:
+                logger.error(
+                    f"The ['{','.join(invalid_integrations)}'] integrations specified for the items doesn't exist in the "
+                    "list of integrations on the platform. Any associated items will be skipped."
+                )
+        if _unique_attachments:
             logger.info(
-                f"Attaching {len(unique_attachments)} file(s) to project {project}."
+                f"Attaching {len(_unique_attachments)} file(s) to project {project}."
             )
             project, folder = self.controller.get_project_folder(
                 project_name, folder_name
             )
             response = self.controller.items.attach(
                 project=project,
                 folder=folder,
-                attachments=unique_attachments,
+                attachments=_unique_attachments,
                 annotation_status=annotation_status,
             )
             if response.errors:
                 raise AppException(response.errors)
             uploaded, duplicated = response.data
             fails = [
                 attachment.name
-                for attachment in unique_attachments
+                for attachment in _unique_attachments
                 if attachment.name not in uploaded and attachment.name not in duplicated
             ]
         return uploaded, fails, duplicated

src/superannotate/lib/core/entities/project.py

@@ -40,6 +40,8 @@ class TimedBaseModel(BaseModel):
 class AttachmentEntity(BaseModel):
     name: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4()))
     url: str
+    integration: Optional[str] = None
+    integration_id: Optional[int] = None
 
     class Config:
         extra = Extra.ignore

src/superannotate/lib/core/enums.py

@@ -171,6 +171,7 @@ class IntegrationTypeEnum(BaseTitledEnum):
     AWS = "aws", 1
     GCP = "gcp", 2
     AZURE = "azure", 3
+    CUSTOM = "custom", 4
 
 
 class TrainingStatus(BaseTitledEnum):

src/superannotate/lib/core/types.py

@@ -33,8 +33,10 @@ class PriorityScoreEntity(BaseModel):
 class Attachment(BaseModel):
     name: str
     path: str
+    integration_id: Optional[int] = None
 
 
 class AttachmentMeta(BaseModel):
     width: Optional[float] = None
     height: Optional[float] = None
+    integration_id: Optional[int] = None

src/superannotate/lib/core/usecases/items.py

@@ -410,8 +410,8 @@ def validate_upload_state(self):
             raise AppValidationException(constants.ATTACHING_UPLOAD_STATE_ERROR)
 
     @staticmethod
-    def generate_meta() -> AttachmentMeta:
-        return AttachmentMeta(width=None, height=None)
+    def generate_meta(integration_id=None) -> AttachmentMeta:
+        return AttachmentMeta(width=None, height=None, integration_id=integration_id)
 
     def execute(self) -> Response:
         if self.is_valid():
@@ -436,7 +436,9 @@ def execute(self) -> Response:
                         to_upload.append(
                             Attachment(name=attachment.name, path=attachment.url)
                         )
-                        to_upload_meta[attachment.name] = self.generate_meta()
+                        to_upload_meta[attachment.name] = self.generate_meta(
+                            attachment.integration_id
+                        )
                 if to_upload:
                     backend_response = self._service_provider.items.attach(
                         project=self._project,

src/superannotate/lib/infrastructure/services/item.py

@@ -40,7 +40,7 @@ class ItemService(BaseItemService):
         ProjectType.PIXEL: ImageResponse,
         ProjectType.DOCUMENT: DocumentResponse,
         ProjectType.POINT_CLOUD: PointCloudResponse,
-        ProjectType.GEN_AI: ImageResponse
+        ProjectType.GEN_AI: ImageResponse,
     }
 
     def get_by_id(self, item_id, project_id, project_type):

tests/data_set/attach_urls_integration.csv

@@ -0,0 +1,6 @@
+url,name,integration
+/600x800,ex_1,custom1
+/250x420,ex_2,custom1
+/1024x768,ex_3,custom4
+/300x200,ex_4,custom2
+fjvnvjsvnsdjcndsjcndjcs,,