Merge pull request #676 from superannotateai/FRIDAY-2706

fix upload_images_to_project in case of duplicate paths

Author: VaghinakDev

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2051,8 +2051,7 @@ def upload_images_to_project(
         images_to_upload, duplicates = use_case.images_to_upload
         if len(duplicates):
             logger.warning(
-                "%s already existing images found that won't be uploaded.",
-                len(duplicates),
+                f"{len(duplicates)} duplicated images found that won't be uploaded."
             )
         logger.info(f"Uploading {len(images_to_upload)} images to project {project}.")
         uploaded, failed_images, duplications = [], [], duplicates

src/superannotate/lib/core/usecases/images.py

@@ -1020,11 +1020,12 @@ def filter_paths(self, paths: List[str]):
             name_path_map[Path(path).name].append(path)
 
         CHUNK_SIZE = UploadImagesToProject.LIST_NAME_CHUNK_SIZE
+
         filtered_paths = []
         duplicated_paths = []
         for file_name in name_path_map:
             if len(name_path_map[file_name]) > 1:
-                duplicated_paths.append(name_path_map[file_name][1:])
+                duplicated_paths.extend(name_path_map[file_name][1:])
             filtered_paths.append(name_path_map[file_name][0])
 
         image_list = []
@@ -1059,7 +1060,7 @@ def images_to_upload(self):
 
     def execute(self):
         if self.is_valid():
-            images_to_upload, duplications = self.images_to_upload
+            images_to_upload, _ = self.images_to_upload
             images_to_upload = images_to_upload[: self.auth_data["availableImageCount"]]
             if not images_to_upload:
                 return self._response
@@ -1082,6 +1083,7 @@ def execute(self):
                     yield
 
             uploaded = []
+            duplications = []  # existing items
             for i in range(0, len(uploaded_images), 100):
                 response = AttachFileUrlsUseCase(
                     project=self._project,

src/superannotate/logger.py

@@ -0,0 +1,126 @@
+import io
+import os
+from os.path import dirname
+
+from src.superannotate import SAClient
+from tests.integration.base import BaseTestCase
+
+sa = SAClient()
+
+
+class TestSingleImageUpload(BaseTestCase):
+    PROJECT_NAME = "single_image_upload"
+    PROJECT_DESCRIPTION = "Desc"
+    PROJECT_TYPE = "Vector"
+    TEST_FOLDER_PTH = "data_set"
+    TEST_FOLDER_PATH = "data_set/sample_project_vector"
+
+    @property
+    def folder_path(self):
+        return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH)
+
+    @property
+    def classes_path(self):
+        return os.path.join(
+            dirname(dirname(__file__)), self.TEST_FOLDER_PATH, "classes/classes.json"
+        )
+
+    def test_single_image_upload(self):
+        sa.upload_image_to_project(
+            self.PROJECT_NAME,
+            self.folder_path + "/example_image_1.jpg",
+            annotation_status="InProgress",
+        )
+        assert len(sa.search_items(self.PROJECT_NAME)) == 1
+
+        with open(self.folder_path + "/example_image_1.jpg", "rb") as f:
+            img = io.BytesIO(f.read())
+
+        sa.upload_image_to_project(
+            self.PROJECT_NAME, img, image_name="rr.jpg", annotation_status="InProgress"
+        )
+
+        assert len(sa.search_items(self.PROJECT_NAME)) == 2
+
+
+class TestMultipleImageUpload(BaseTestCase):
+    PROJECT_NAME = "test_multiple_image_upload"
+    PROJECT_DESCRIPTION = "Desc"
+    PROJECT_TYPE = "Vector"
+    TEST_FOLDER_PTH = "data_set"
+    TEST_FOLDER_PATH = "data_set/sample_project_vector"
+
+    @property
+    def folder_path(self):
+        return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH)
+
+    @property
+    def classes_path(self):
+        return os.path.join(
+            dirname(dirname(__file__)), self.TEST_FOLDER_PATH, "classes/classes.json"
+        )
+
+    def test_multiple_image_upload(self):
+        (uploaded, could_not_upload, existing_images,) = sa.upload_images_to_project(
+            self.PROJECT_NAME,
+            [
+                f"{self.folder_path}/example_image_1.jpg",
+                f"{self.folder_path}/example_image_2.jpg",
+                f"{self.folder_path}/example_image_3.jpg",
+                f"{self.folder_path}/example_image_4.jpg",
+            ],
+            annotation_status="InProgress",
+        )
+        self.assertEqual(len(uploaded), 4)
+        self.assertEqual(len(could_not_upload), 0)
+        self.assertEqual(len(existing_images), 0)
+
+        (uploaded, could_not_upload, existing_images,) = sa.upload_images_to_project(
+            self.PROJECT_NAME,
+            [
+                f"{self.folder_path}/example_image_1.jpg",
+                f"{self.folder_path}/example_image_2.jpg",
+                f"{self.folder_path}/example_image_3.jpg",
+                f"{self.folder_path}/example_image_4.jpg",
+            ],
+            annotation_status="InProgress",
+        )
+        self.assertEqual(len(uploaded), 0)
+        self.assertEqual(len(could_not_upload), 0)
+        self.assertEqual(len(existing_images), 4)
+
+        uploaded, could_not_upload, existing_images = sa.upload_images_to_project(
+            self.PROJECT_NAME, [f"{self.folder_path}/dd.jpg"]
+        )
+        self.assertEqual(len(uploaded), 0)
+        self.assertEqual(len(could_not_upload), 1)
+        self.assertEqual(len(existing_images), 0)
+
+    def test_multiple_image_upload_with_duplicates(self):
+        with self.assertLogs("sa", level="INFO") as mock_log:
+            (
+                uploaded,
+                could_not_upload,
+                existing_images,
+            ) = sa.upload_images_to_project(
+                self.PROJECT_NAME,
+                [
+                    f"{self.folder_path}/example_image_1.jpg",
+                    f"{self.folder_path}/example_image_1.jpg",
+                    f"{self.folder_path}/example_image_1.jpg",
+                    f"{self.folder_path}/example_image_3.jpg",
+                    f"{self.folder_path}/example_image_4.jpg",
+                ],
+                annotation_status="InProgress",
+            )
+            assert (
+                mock_log.records[0].msg
+                == "2 duplicated images found that won't be uploaded."
+            )
+            assert (
+                mock_log.records[1].msg
+                == "Uploading 3 images to project test_multiple_image_upload."
+            )
+            self.assertEqual(len(uploaded), 3)
+            self.assertEqual(len(could_not_upload), 0)
+            self.assertEqual(len(existing_images), 2)