Merge pull request #678 from superannotateai/FRIDAY-2706

fix upload_images_to_project

Author: nareksa

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2048,25 +2048,21 @@ def upload_images_to_project(
             from_s3_bucket=from_s3_bucket,
         )
 
-        images_to_upload, duplicates = use_case.images_to_upload
-        if len(duplicates):
-            logger.warning(
-                f"{len(duplicates)} duplicated images found that won't be uploaded."
-            )
+        images_to_upload, existing_items = use_case.images_to_upload
         logger.info(f"Uploading {len(images_to_upload)} images to project {project}.")
-        uploaded, failed_images, duplications = [], [], duplicates
+        uploaded, failed_images = [], []
         if not images_to_upload:
-            return uploaded, failed_images, duplications
+            return uploaded, failed_images, existing_items
         if use_case.is_valid():
             with tqdm(
                 total=len(images_to_upload), desc="Uploading images"
             ) as progress_bar:
                 for _ in use_case.execute():
                     progress_bar.update(1)
-            uploaded, failed_images, duplications = use_case.data
-            if duplications:
-                logger.info(f"Duplicated images {', '.join(duplications)}")
-            return uploaded, failed_images, duplications
+            uploaded, failed_images, existing_items = use_case.data
+            if existing_items:
+                logger.info(f"Existing images {', '.join(existing_items)}")
+            return uploaded, failed_images, existing_items
         raise AppException(use_case.response.errors)
 
     def aggregate_annotations_as_df(

src/superannotate/lib/core/usecases/images.py

@@ -1023,11 +1023,17 @@ def filter_paths(self, paths: List[str]):
 
         filtered_paths = []
         duplicated_paths = []
+        existing_items = []
         for file_name in name_path_map:
             if len(name_path_map[file_name]) > 1:
                 duplicated_paths.extend(name_path_map[file_name][1:])
             filtered_paths.append(name_path_map[file_name][0])
 
+        if duplicated_paths:
+            logger.warning(
+                f"{len(duplicated_paths)} duplicate paths found that won't be uploaded."
+            )
+
         image_list = []
         for i in range(0, len(filtered_paths), CHUNK_SIZE):
             response = self._service_provider.items.list_by_names(
@@ -1046,11 +1052,12 @@ def filter_paths(self, paths: List[str]):
         images_to_upload = []
 
         for path in filtered_paths:
-            if Path(path).name not in image_list:
+            image_name = Path(path).name
+            if image_name not in image_list:
                 images_to_upload.append(path)
             else:
-                duplicated_paths.append(path)
-        return list(set(images_to_upload)), duplicated_paths
+                existing_items.append(image_name)
+        return list(set(images_to_upload)), existing_items
 
     @property
     def images_to_upload(self):
@@ -1060,7 +1067,7 @@ def images_to_upload(self):
 
     def execute(self):
         if self.is_valid():
-            images_to_upload, _ = self.images_to_upload
+            images_to_upload, existing_items = self.images_to_upload
             images_to_upload = images_to_upload[: self.auth_data["availableImageCount"]]
             if not images_to_upload:
                 return self._response
@@ -1083,7 +1090,7 @@ def execute(self):
                     yield
 
             uploaded = []
-            duplications = []  # existing items
+            attach_duplications_list = []
             for i in range(0, len(uploaded_images), 100):
                 response = AttachFileUrlsUseCase(
                     project=self._project,
@@ -1101,10 +1108,14 @@ def execute(self):
                     continue
                 attachments, attach_duplications = response.data
                 uploaded.extend(attachments)
-                duplications.extend(attach_duplications)
+                attach_duplications_list.extend(attach_duplications)
+            if attach_duplications_list:
+                logger.debug(
+                    f"{len(attach_duplications_list)} item attachments duplicates found."
+                )
             uploaded = [image["name"] for image in uploaded]
             failed_images = [image.split("/")[-1] for image in failed_images]
-            self._response.data = uploaded, failed_images, duplications
+            self._response.data = uploaded, failed_images, existing_items
         return self._response
 
 

tests/integration/test_image_upload.py

@@ -67,11 +67,10 @@ def test_multiple_image_upload(self):
                 f"{self.folder_path}/example_image_1.jpg",
                 f"{self.folder_path}/example_image_2.jpg",
                 f"{self.folder_path}/example_image_3.jpg",
-                f"{self.folder_path}/example_image_4.jpg",
             ],
             annotation_status="InProgress",
         )
-        self.assertEqual(len(uploaded), 4)
+        self.assertEqual(len(uploaded), 3)
         self.assertEqual(len(could_not_upload), 0)
         self.assertEqual(len(existing_images), 0)
 
@@ -85,9 +84,9 @@ def test_multiple_image_upload(self):
             ],
             annotation_status="InProgress",
         )
-        self.assertEqual(len(uploaded), 0)
+        self.assertEqual(len(uploaded), 1)
         self.assertEqual(len(could_not_upload), 0)
-        self.assertEqual(len(existing_images), 4)
+        self.assertEqual(len(existing_images), 3)
 
         uploaded, could_not_upload, existing_images = sa.upload_images_to_project(
             self.PROJECT_NAME, [f"{self.folder_path}/dd.jpg"]
@@ -115,12 +114,12 @@ def test_multiple_image_upload_with_duplicates(self):
             )
             assert (
                 mock_log.records[0].msg
-                == "2 duplicated images found that won't be uploaded."
+                == "2 duplicate paths found that won't be uploaded."
             )
             assert (
                 mock_log.records[1].msg
                 == "Uploading 3 images to project test_multiple_image_upload."
             )
             self.assertEqual(len(uploaded), 3)
             self.assertEqual(len(could_not_upload), 0)
-            self.assertEqual(len(existing_images), 2)
+            self.assertEqual(len(existing_images), 0)