fix upload

Author: VaghinakDev

src/superannotate/lib/core/usecases/annotations.py

@@ -120,8 +120,8 @@ def __init__(
         self._annotation_upload_data = None
         self._item_ids = []
         self._s3_bucket = None
-        self._big_files_queue = asyncio.Queue()
-        self._small_files_queue = asyncio.Queue()
+        self._big_files_queue = None
+        self._small_files_queue = None
         self._report = Report([], [], [], [])
 
     @staticmethod
@@ -327,16 +327,14 @@ async def upload_small_annotations(self):
 
         async def upload(_chunk):
             try:
-                report = await self._upload_small_annotations(chunk)
+                report = await self._upload_small_annotations(_chunk)
                 self._report.failed_annotations.extend(report.failed_annotations)
                 self._report.missing_classes.extend(report.missing_classes)
                 self._report.missing_attr_groups.extend(report.missing_attr_groups)
                 self._report.missing_attrs.extend(report.missing_attrs)
-            except Exception:
-                import traceback
-
-                traceback.print_exc()
-                self._report.failed_annotations.extend([i.name for i in chunk])
+            except Exception as e:
+                self.reporter.log_debug(str(e))
+                self._report.failed_annotations.extend([i.name for i in _chunk])
 
         while True:
             item = await self._small_files_queue.get()
@@ -407,7 +405,10 @@ async def distribute_queues(self, items_to_upload: list):
                                 self._big_files_queue.put_nowait(t_item)
                         else:
                             self._small_files_queue.put_nowait(t_item)
-                    except (ValidationError, AppException):
+                    except Exception as e:
+                        self.reporter.log_debug(str(e))
+                        data[idx][1] = True
+                        self.reporter.update_progress()
                         self._report.failed_annotations.append(item.name)
                     finally:
                         data[idx][1] = True
@@ -417,6 +418,8 @@ async def distribute_queues(self, items_to_upload: list):
 
     async def run_workers(self, items_to_upload):
         try:
+            self._big_files_queue = asyncio.Queue()
+            self._small_files_queue = asyncio.Queue()
             await asyncio.gather(
                 self.distribute_queues(items_to_upload),
                 self.upload_small_annotations(),
@@ -449,7 +452,6 @@ def execute(self):
                 items_to_upload.append(self.AnnotationToUpload(item.uuid, name, path))
             except KeyError:
                 missing_annotations.append(name)
-
         asyncio.run(self.run_workers(items_to_upload))
         self.reporter.finish_progress()
         self._log_report()
@@ -1228,8 +1230,6 @@ def iter_errors(self, instance, _schema=None):
             validators.extend(jsonschema.validators.iteritems(_schema))
 
             for k, v in validators:
-                if k == "createdBy":
-                    print()
                 validator = self.VALIDATORS.get(k)
                 if validator is None:
                     continue

src/superannotate/lib/infrastructure/controller.py

@@ -167,7 +167,7 @@ def get_default_reporter(
         log_info: bool = True,
         log_warning: bool = True,
         disable_progress_bar: bool = False,
-        log_debug: bool = True,
+        log_debug: bool = False,
     ) -> Reporter:
         return Reporter(log_info, log_warning, disable_progress_bar, log_debug)
 

src/superannotate/lib/infrastructure/services.py

@@ -1402,7 +1402,8 @@ async def upload_annotations(
                 data=data,
             )
             if not _response.ok:
-                raise AppException(_response.json())
+                self.logger.debug(str(await _response.text()))
+                raise AppException("Can't upload annotations.")
             data_json = await _response.json()
             response = ServiceResponse()
             response.status = _response.status

tests/integration/annotations/test_.py

@@ -0,0 +1,35 @@
+import os
+
+import tempfile
+from pathlib import Path
+import json
+import sys
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+
+LIB_PATH = Path(__file__).parent.parent.parent.parent / "src"
+print(LIB_PATH)
+sys.path.insert(0, str(LIB_PATH))
+sys.path.append(str( Path(__file__).parent.parent.parent.parent))
+from src.superannotate import SAClient
+
+# # def test_():
+# os.environ.update(
+#     {
+#         "SA_VERSION_CHECK": "False",
+#         "SA_URL": "https://api.devsuperannotate.com",
+#         "SA_TOKEN": "b0677090e55b8fe5d7448e459a13b0aa3d0fdf8be71fb64c3f10cdde129703de1300ea0a613a7909t=8084"
+#     }
+# )
+def test_():
+    sa = SAClient()
+    VECTOR_JSON = "example_image_1.jpg___objects.json"
+    with tempfile.TemporaryDirectory() as tmpdir_name:
+        with open(f"{tmpdir_name}/{VECTOR_JSON}", "w") as f:
+            json.dump([{"metadata": {"name": "name"}, "instances": []}], f)
+        print(sa.upload_annotations_from_folder_to_project("testo", f"{tmpdir_name}"))
+    # sa.get_annotations("50k Large Data")
+
+
+