removed unnecessary prints

Author: VahagnTKA

pytest.ini

@@ -3,4 +3,4 @@ minversion = 3.7
 log_cli=true
 python_files = test_*.py
 ;pytest_plugins = ['pytest_profiling']
-;addopts = -n auto --dist=loadscope
+addopts = -n auto --dist=loadscope

src/superannotate/__init__.py

@@ -1,7 +1,7 @@
 import os
 import sys
 
-__version__ = "4.4.2dev12"
+__version__ = "4.4.2dev13"
 
 sys.path.append(os.path.split(os.path.realpath(__file__))[0])
 

src/superannotate/lib/app/analytics/aggregators.py

@@ -173,17 +173,13 @@ def aggregate_annotations_as_df(self):
             return self.aggregate_document_annotations_as_df(annotation_paths)
 
     def __add_attributes_to_raws(self, raws, attributes, element_raw):
-        if element_raw.type == 'tag':
-            print(attributes)
         for attribute_id, attribute in enumerate(attributes):
             attribute_raw = copy.copy(element_raw)
             attribute_raw.attributeId = attribute_id
             attribute_raw.attributeGroupName = attribute.get("groupName")
             attribute_raw.attributeName = attribute.get("name")
             raws.append(attribute_raw)
         if not attributes:
-            if element_raw.type == 'tag':
-                print(element_raw)
             raws.append(element_raw)
 
         return raws

src/superannotate/lib/app/input_converters/converters/coco_converters/coco_converter.py

@@ -305,7 +305,6 @@ def make_anno_json_generator(self):
             jsons = list(Path(self.export_root).glob("*objects.json"))
 
         self.set_num_total_images(len(jsons))
-        print()
         for fpath in jsons:
             with open(fpath) as fp:
                 json_data = json.load(fp)

src/superannotate/lib/core/usecases/annotations.py

@@ -169,9 +169,9 @@ def get_annotation_from_s3(bucket, path: str):
         file.seek(0)
         return file
 
-    def prepare_annotation(self, annotation: dict) -> dict:
+    def prepare_annotation(self, annotation: dict, size) -> dict:
         errors = None
-        if sys.getsizeof(annotation) < BIG_FILE_THRESHOLD:
+        if size < BIG_FILE_THRESHOLD:
             use_case = ValidateAnnotationUseCase(
                 reporter=self.reporter,
                 team_id=self._project.team_id,
@@ -192,12 +192,12 @@ def get_annotation(
         self, path: str
     ) -> (Optional[Tuple[io.StringIO]], Optional[io.BytesIO]):
         mask = None
+
         if self._client_s3_bucket:
-            annotation = json.load(
-                self.get_annotation_from_s3(self._client_s3_bucket, path)
-            )
+            file = self.get_annotation_from_s3(self._client_s3_bucket, path)
+
         else:
-            annotation = json.load(open(path))
+            file = open(path)
             if self._project.type == constants.ProjectType.PIXEL.value:
                 mask = open(
                     path.replace(
@@ -206,11 +206,18 @@ def get_annotation(
                     ),
                     "rb",
                 )
-        annotation = self.prepare_annotation(annotation)
+        _tmp = file.read()
+        if not isinstance(_tmp, bytes):
+            _tmp = _tmp.encode("utf8")
+        file = io.BytesIO(_tmp)
+        file.seek(0)
+        size = file.getbuffer().nbytes
+        annotation = json.load(file)
+        annotation = self.prepare_annotation(annotation, size)
         if not annotation:
             self.reporter.store_message("invalid_jsons", path)
-            return None, None
-        return annotation, mask
+            raise AppException("Invalid json")
+        return annotation, mask, size
 
     @staticmethod
     def chunks(data, size: int = 10000):
@@ -299,25 +306,29 @@ async def _upload_small_annotations(self, chunk) -> Report:
             [],
             [],
         )
-        response = await self._backend_service.upload_annotations(
-            team_id=self._project.team_id,
-            project_id=self._project.id,
-            folder_id=self._folder.id,
-            items_name_file_map={i.name: i.data for i in chunk},
-        )
-        self.reporter.update_progress(len(chunk))
-        if response.ok:
-            if response.data.failed_items:  # noqa
-                failed_annotations = response.data.failed_items
-            missing_classes = response.data.missing_resources.classes
-            missing_attr_groups = response.data.missing_resources.attribute_groups
-            missing_attrs = response.data.missing_resources.attributes
-            if self._project.type == constants.ProjectType.PIXEL.value:
-                for i in chunk:
-                    if i.mask:
-                        self._upload_mask(i)
-        else:
+        try:
+            response = await self._backend_service.upload_annotations(
+                team_id=self._project.team_id,
+                project_id=self._project.id,
+                folder_id=self._folder.id,
+                items_name_file_map={i.name: i.data for i in chunk},
+            )
+            if response.ok:
+                if response.data.failed_items:  # noqa
+                    failed_annotations = response.data.failed_items
+                missing_classes = response.data.missing_resources.classes
+                missing_attr_groups = response.data.missing_resources.attribute_groups
+                missing_attrs = response.data.missing_resources.attributes
+                if self._project.type == constants.ProjectType.PIXEL.value:
+                    for i in chunk:
+                        if i.mask:
+                            self._upload_mask(i)
+            else:
+                failed_annotations.extend([i.name for i in chunk])
+        except Exception:  # noqa
             failed_annotations.extend([i.name for i in chunk])
+        finally:
+            self.reporter.update_progress(len(chunk))
         return Report(
             failed_annotations, missing_classes, missing_attr_groups, missing_attrs
         )
@@ -350,7 +361,6 @@ async def upload(_chunk):
             ):
                 await upload(chunk)
                 chunk = []
-
         if chunk:
             await upload(chunk)
 
@@ -364,24 +374,27 @@ async def _upload_big_annotation(self, item) -> Tuple[str, bool]:
                 data=item.data,
                 chunk_size=5 * 1024 * 1024,
             )
-            self.reporter.update_progress()
             if is_uploaded and (
                 self._project.type == constants.ProjectType.PIXEL.value and item.mask
             ):
                 self._upload_mask(item.mask)
             return item.name, is_uploaded
-        except AppException as e:
-            self.reporter.log_error(str(e))
+        except Exception as e:
+            self.reporter.log_debug(str(e))
             self._report.failed_annotations.append(item.name)
+            raise
+        finally:
+            self.reporter.update_progress()
 
     async def upload_big_annotations(self):
         while True:
             item = await self._big_files_queue.get()
             self._big_files_queue.task_done()
-            if not item:
-                self._big_files_queue.put_nowait(None)
-                return
-            await self._upload_big_annotation(item)
+            if item:
+                await self._upload_big_annotation(item)
+            else:
+                await self._big_files_queue.put(None)
+                break
 
     async def distribute_queues(self, items_to_upload: list):
         data: List[List[Any, bool]] = [[i, False] for i in items_to_upload]
@@ -390,12 +403,12 @@ async def distribute_queues(self, items_to_upload: list):
             for idx, (item, processed) in enumerate(data):
                 if not processed:
                     try:
-                        annotation, mask = self.get_annotation(item.path)
+                        annotation, mask, size = self.get_annotation(item.path)
                         t_item = copy.copy(item)
-                        size = sys.getsizeof(annotation)
                         annotation_file = io.StringIO()
                         json.dump(annotation, annotation_file)
                         annotation_file.seek(0)
+                        del annotation
                         t_item.data = annotation_file
                         t_item.mask = mask
                         if size > BIG_FILE_THRESHOLD:
@@ -407,10 +420,9 @@ async def distribute_queues(self, items_to_upload: list):
                             self._small_files_queue.put_nowait(t_item)
                     except Exception as e:
                         self.reporter.log_debug(str(e))
-                        data[idx][1] = True
-                        self.reporter.update_progress()
                         self._report.failed_annotations.append(item.name)
                     finally:
+                        self.reporter.update_progress()
                         data[idx][1] = True
                         processed_count += 1
         self._big_files_queue.put_nowait(None)
@@ -426,7 +438,6 @@ async def run_workers(self, items_to_upload):
                 self.upload_big_annotations(),
                 self.upload_big_annotations(),
                 self.upload_big_annotations(),
-                self.upload_big_annotations(),
                 return_exceptions=True,
             )
         except Exception as e:
@@ -466,10 +477,16 @@ def execute(self):
             if not statuses_changed:
                 self._response.errors = AppException("Failed to change status.")
 
+        if missing_annotations:
+            logger.warning(
+                f"Couldn't find {len(missing_annotations)}/{len(name_path_mappings.keys())} "
+                "items on the platform that match the annotations you want to upload."
+            )
         if self._report.failed_annotations:
             self.reporter.log_warning(
                 f"Couldn't validate annotations. {constants.USE_VALIDATE_MESSAGE}"
             )
+
         self._response.data = (
             uploaded_annotations,
             self._report.failed_annotations,
@@ -1007,7 +1024,12 @@ def download_annotation_classes(self, path: str):
             classes_path.mkdir(parents=True, exist_ok=True)
             with open(classes_path / "classes.json", "w+") as file:
                 json.dump(
-                    [i.dict(exclude_unset=True) for i in response.data], file, indent=4
+                    [
+                        i.dict(exclude_unset=True, by_alias=True, fill_enum_values=True)
+                        for i in response.data
+                    ],
+                    file,
+                    indent=4,
                 )
         else:
             self._response.errors = AppException("Cant download classes.")
@@ -1056,9 +1078,9 @@ def execute(self):
                 loop = asyncio.new_event_loop()
                 if not self._item_names:
                     condition = (
-                            Condition("team_id", self._project.team_id, EQ)
-                            & Condition("project_id", self._project.id, EQ)
-                            & Condition("folder_id", self._folder.uuid, EQ)
+                        Condition("team_id", self._project.team_id, EQ)
+                        & Condition("project_id", self._project.id, EQ)
+                        & Condition("folder_id", self._folder.uuid, EQ)
                     )
                     item_names = [item.name for item in self._images.get_all(condition)]
                 else:
@@ -1081,11 +1103,13 @@ def execute(self):
                     for folder in folders:
                         if not self._item_names:
                             condition = (
-                                    Condition("team_id", self._project.team_id, EQ)
-                                    & Condition("project_id", self._project.id, EQ)
-                                    & Condition("folder_id", folder.uuid, EQ)
+                                Condition("team_id", self._project.team_id, EQ)
+                                & Condition("project_id", self._project.id, EQ)
+                                & Condition("folder_id", folder.uuid, EQ)
                             )
-                            item_names = [item.name for item in self._images.get_all(condition)]
+                            item_names = [
+                                item.name for item in self._images.get_all(condition)
+                            ]
                         else:
                             item_names = self._item_names
                         coroutines.append(

src/superannotate/lib/core/usecases/classes.py

@@ -197,7 +197,7 @@ def execute(self):
         )
         if response.ok:
             classes = [
-                entity.dict(by_alias=True, fill_enum_values=True)
+                entity.dict(by_alias=True, fill_enum_values=True, exclude_unset=True)
                 for entity in response.data
             ]
             json_path = f"{self._download_path}/classes.json"

src/superannotate/lib/infrastructure/controller.py

@@ -167,7 +167,7 @@ def get_default_reporter(
         log_info: bool = True,
         log_warning: bool = True,
         disable_progress_bar: bool = False,
-        log_debug: bool = False,
+        log_debug: bool = True,
     ) -> Reporter:
         return Reporter(log_info, log_warning, disable_progress_bar, log_debug)