Merge pull request #668 from superannotateai/develop

VaghinakDev · web-flow · commit 5be819a4e631 · 2024-01-18T10:24:17.000+04:00
Develop
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -6,6 +6,18 @@ History
 
 All release highlights of this project will be documented in this file.
 
+4.4.18 - January 18, 2024
+__________________________
+
+
+**Updated**
+
+    - Improved error handling.
+
+**Removed**
+
+    -  dependency from ``jsonschema``.
+
 4.4.17 - December 21, 2023
 __________________________
 
diff --git a/pytest.ini b/pytest.ini
@@ -3,5 +3,5 @@ minversion = 3.7
 log_cli=true
 python_files = test_*.py
 ;pytest_plugins = ['pytest_profiling']
-;addopts = -n auto --dist=loadscope
+addopts = -n auto --dist=loadscope
 
diff --git a/requirements.txt b/requirements.txt
@@ -9,8 +9,8 @@ pandas~=2.0
 ffmpeg-python~=0.2
 pillow>=9.5,~=10.0
 tqdm~=4.66.1
-requests~=2.31.0
-aiofiles==23.1.0
+requests==2.*
+aiofiles==23.*
 fire==0.4.0
 mixpanel==4.8.3
-jsonschema==3.2.0
+superannotate-schemas==1.0.47b5
diff --git a/src/superannotate/__init__.py b/src/superannotate/__init__.py
@@ -3,7 +3,7 @@
 import sys
 
 
-__version__ = "4.4.17"
+__version__ = "4.4.18"
 
 sys.path.append(os.path.split(os.path.realpath(__file__))[0])
 
diff --git a/src/superannotate/lib/app/interface/sdk_interface.py b/src/superannotate/lib/app/interface/sdk_interface.py
@@ -79,7 +79,7 @@
     "Tiled",
     "Other",
     "PointCloud",
-    "CustomEditor",
+    "GenAI",
 ]
 
 ANNOTATION_STATUS = Literal[
diff --git a/src/superannotate/lib/app/serializers.py b/src/superannotate/lib/app/serializers.py
@@ -122,7 +122,10 @@ def serialize(
         if exclude:
             for field in exclude:
                 to_exclude[field] = True
-
+        if self._entity.classes:
+            self._entity.classes = [
+                i.dict(by_alias=True, exclude_unset=True) for i in self._entity.classes
+            ]
         data = super().serialize(fields, by_alias, flat, to_exclude)
         if data.get("settings"):
             data["settings"] = [
diff --git a/src/superannotate/lib/core/entities/classes.py b/src/superannotate/lib/core/entities/classes.py
@@ -73,6 +73,7 @@ class AttributeGroup(TimedBaseModel):
     group_type: Optional[GroupTypeEnum]
     class_id: Optional[StrictInt]
     name: Optional[StrictStr]
+    isRequired: bool = Field(default=False)
     attributes: Optional[List[Attribute]]
     default_value: Any
 
diff --git a/src/superannotate/lib/core/enums.py b/src/superannotate/lib/core/enums.py
@@ -93,7 +93,7 @@ class ProjectType(BaseTitledEnum):
     TILED = "Tiled", 5
     OTHER = "Other", 6
     POINT_CLOUD = "PointCloud", 7
-    CUSTOM_EDITOR = "CustomEditor", 8
+    GEN_AI = "GenAI", 8
     UNSUPPORTED_TYPE_1 = "UnsupportedType", 9
     UNSUPPORTED_TYPE_2 = "UnsupportedType", 10
 
diff --git a/src/superannotate/lib/core/usecases/annotations.py b/src/superannotate/lib/core/usecases/annotations.py
@@ -25,10 +25,8 @@
 
 import aiofiles
 import boto3
-import jsonschema.validators
 import lib.core as constants
-from jsonschema import Draft7Validator
-from jsonschema import ValidationError
+import superannotate_schemas
 from lib.core.conditions import Condition
 from lib.core.conditions import CONDITION_EQ as EQ
 from lib.core.entities import BaseItemEntity
@@ -309,7 +307,7 @@ def __init__(
 
     def validate_project_type(self):
         if self._project.type == constants.ProjectType.PIXEL.value:
-            raise ValidationError("Unsupported project type.")
+            raise AppException("Unsupported project type.")
 
     def _validate_json(self, json_data: dict) -> list:
         if self._project.type >= constants.ProjectType.PIXEL.value:
@@ -1227,7 +1225,7 @@ def execute(self):
 
 class ValidateAnnotationUseCase(BaseReportableUseCase):
     DEFAULT_VERSION = "V1.00"
-    SCHEMAS: Dict[str, Draft7Validator] = {}
+    SCHEMAS: Dict[str, superannotate_schemas.Draft7Validator] = {}
     PATTERN_MAP = {
         "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d(?:\\.\\d{3})Z": "does not match YYYY-MM-DDTHH:MM:SS.fffZ",
         "^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$": "invalid email",
@@ -1278,15 +1276,17 @@ def oneOf(validator, oneOf, instance, schema):  # noqa
                     const_key, instance
                 )
                 if not instance_type:
-                    yield ValidationError("type required")
+                    yield superannotate_schemas.ValidationError("type required")
                     return
                 if const_key and instance_type == _type:
                     errs = list(
                         validator.descend(instance, sub_schema, schema_path=index)
                     )
                     if not errs:
                         return
-                    yield ValidationError("invalid instance", context=errs)
+                    yield superannotate_schemas.ValidationError(
+                        "invalid instance", context=errs
+                    )
                     return
             else:
                 subschemas = enumerate(oneOf)
@@ -1299,24 +1299,25 @@ def oneOf(validator, oneOf, instance, schema):  # noqa
                         break
                     all_errors.extend(errs)
                 else:
-                    yield ValidationError(
+                    yield superannotate_schemas.ValidationError(
                         f"{instance!r} is not valid under any of the given schemas",
                         context=all_errors[:1],
                     )
-                # yield from jsonschema._validators.oneOf(  # noqa
-                #     validator, oneOf, instance, schema
-                # )
         if const_key:
-            yield ValidationError(f"invalid {'.'.join(const_key)}")
+            yield superannotate_schemas.ValidationError(
+                f"invalid {'.'.join(const_key)}"
+            )
 
     @staticmethod
     def _pattern(validator, patrn, instance, schema):
         if validator.is_type(instance, "string") and not re.search(patrn, instance):
             _patrn = ValidateAnnotationUseCase.PATTERN_MAP.get(patrn)
             if _patrn:
-                yield ValidationError(f"{instance}  {_patrn}")
+                yield superannotate_schemas.ValidationError(f"{instance}  {_patrn}")
             else:
-                yield ValidationError(f"{instance} does not match {patrn}")
+                yield superannotate_schemas.ValidationError(
+                    f"{instance} does not match {patrn}"
+                )
 
     @staticmethod
     def iter_errors(self, instance, _schema=None):
@@ -1325,7 +1326,7 @@ def iter_errors(self, instance, _schema=None):
         if _schema is True:
             return
         elif _schema is False:
-            yield jsonschema.exceptions.ValidationError(
+            yield superannotate_schemas.ValidationError(
                 f"False schema does not allow {instance!r}",
                 validator=None,
                 validator_value=None,
@@ -1334,7 +1335,7 @@ def iter_errors(self, instance, _schema=None):
             )
             return
 
-        scope = jsonschema.validators._id_of(_schema)  # noqa
+        scope = superannotate_schemas.validators._id_of(_schema)  # noqa
         _schema = copy.copy(_schema)
         if scope:
             self.resolver.push_scope(scope)
@@ -1344,7 +1345,7 @@ def iter_errors(self, instance, _schema=None):
                 ref = _schema.pop("$ref")
                 validators.append(("$ref", ref))
 
-            validators.extend(jsonschema.validators.iteritems(_schema))
+            validators.extend(superannotate_schemas.validators.iteritems(_schema))
 
             for k, v in validators:
                 validator = self.VALIDATORS.get(k)
@@ -1381,7 +1382,7 @@ def extract_path(path):
                     real_path.append(item)
         return real_path
 
-    def _get_validator(self, version: str) -> Draft7Validator:
+    def _get_validator(self, version: str) -> superannotate_schemas.Draft7Validator:
         key = f"{self._project_type}__{version}"
         validator = ValidateAnnotationUseCase.SCHEMAS.get(key)
         if not validator:
@@ -1393,7 +1394,7 @@ def _get_validator(self, version: str) -> Draft7Validator:
             if not schema_response.data:
                 ValidateAnnotationUseCase.SCHEMAS[key] = lambda x: x
                 return ValidateAnnotationUseCase.SCHEMAS[key]
-            validator = jsonschema.Draft7Validator(schema_response.data)
+            validator = superannotate_schemas.Draft7Validator(schema_response.data)
             from functools import partial
 
             iter_errors = partial(self.iter_errors, validator)
diff --git a/src/superannotate/lib/infrastructure/services/annotation.py b/src/superannotate/lib/infrastructure/services/annotation.py
@@ -80,9 +80,9 @@ async def _sync_large_annotation(self, team_id, project_id, item_id):
         async with AIOHttpSession(
             connector=aiohttp.TCPConnector(ssl=False),
             headers=self.client.default_headers,
+            raise_for_status=True,
         ) as session:
             _response = await session.request("post", sync_url, params=sync_params)
-            _response.raise_for_status()
             sync_params.pop("current_source")
             sync_params.pop("desired_source")
 
@@ -123,9 +123,9 @@ async def get_big_annotation(
         async with AIOHttpSession(
             connector=aiohttp.TCPConnector(ssl=False),
             headers=self.client.default_headers,
+            raise_for_status=True,
         ) as session:
             start_response = await session.request("post", url, params=query_params)
-            start_response.raise_for_status()
             large_annotation = await start_response.json()
 
         reporter.update_progress()
@@ -206,9 +206,9 @@ async def download_big_annotation(
         async with AIOHttpSession(
             connector=aiohttp.TCPConnector(ssl=False),
             headers=self.client.default_headers,
+            raise_for_status=True,
         ) as session:
             start_response = await session.request("post", url, params=query_params)
-            start_response.raise_for_status()
             res = await start_response.json()
             Path(download_path).mkdir(exist_ok=True, parents=True)
 
diff --git a/src/superannotate/lib/infrastructure/services/http_client.py b/src/superannotate/lib/infrastructure/services/http_client.py
@@ -1,4 +1,5 @@
 import asyncio
+import io
 import json
 import logging
 import platform
@@ -13,7 +14,6 @@
 
 import aiohttp
 import requests
-from aiohttp.client_exceptions import ClientError
 from lib.core.exceptions import AppException
 from lib.core.service_types import ServiceResponse
 from lib.core.serviceproviders import BaseClient
@@ -229,19 +229,35 @@ class AIOHttpSession(aiohttp.ClientSession):
     RETRY_LIMIT = 3
     BACKOFF_FACTOR = 0.3
 
+    @staticmethod
+    def _copy_form_data(data: aiohttp.FormData) -> aiohttp.FormData:
+        form_data = aiohttp.FormData(quote_fields=False)
+        for field in data._fields:  # noqa
+            if isinstance(field[2], io.IOBase):
+                field[2].seek(0)
+            form_data.add_field(
+                value=field[2],
+                content_type=field[1].get("Content-Type", ""),
+                **field[0],
+            )
+        return form_data
+
     async def request(self, *args, **kwargs) -> aiohttp.ClientResponse:
         attempts = self.RETRY_LIMIT
         delay = 0
         for _ in range(attempts):
             delay += self.BACKOFF_FACTOR
-            attempts -= 1
             try:
                 response = await super()._request(*args, **kwargs)
-            except ClientError:
-                if not attempts:
+                if attempts <= 1 or response.status not in self.RETRY_STATUS_CODES:
+                    return response
+                if isinstance(kwargs["data"], aiohttp.FormData):
+                    raise RuntimeError(await response.text())
+            except (aiohttp.ClientError, RuntimeError) as e:
+                if attempts <= 1:
                     raise
-                await asyncio.sleep(delay)
-                continue
-            if response.status not in self.RETRY_STATUS_CODES or not attempts:
-                return response
+                data = kwargs["data"]
+                if isinstance(data, aiohttp.FormData):
+                    kwargs["data"] = self._copy_form_data(data)
+            attempts -= 1
             await asyncio.sleep(delay)
diff --git a/src/superannotate/lib/infrastructure/services/item.py b/src/superannotate/lib/infrastructure/services/item.py
@@ -40,6 +40,7 @@ class ItemService(BaseItemService):
         ProjectType.PIXEL: ImageResponse,
         ProjectType.DOCUMENT: DocumentResponse,
         ProjectType.POINT_CLOUD: PointCloudResponse,
+        ProjectType.GEN_AI: ImageResponse
     }
 
     def get_by_id(self, item_id, project_id, project_type):
diff --git a/tests/integration/classes/test_create_annotation_class.py b/tests/integration/classes/test_create_annotation_class.py
@@ -44,6 +44,7 @@ def test_create_annotation_class_with_attr_and_default_value(self):
             attribute_groups=[
                 {
                     "name": "test",
+                    "isRequired:": False,
                     "attributes": [{"name": "Car"}, {"name": "Track"}, {"name": "Bus"}],
                     "default_value": "Bus",
                 }
diff --git a/tests/integration/export/test_export.py b/tests/integration/export/test_export.py
@@ -106,4 +106,4 @@ def test_upload_s3(self):
             Bucket=self.TEST_S3_BUCKET, Prefix=self.TMP_DIR
         ).get("Contents", []):
             files.append(object_data["Key"])
-        self.assertEqual(33, len(files))
+        self.assertEqual(25, len(files))
diff --git a/tests/integration/projects/test_basic_project.py b/tests/integration/projects/test_basic_project.py
@@ -11,6 +11,45 @@
 sa = SAClient()
 
 
+class TestGenAIProjectBasic(BaseTestCase):
+    PROJECT_NAME = "TestGenAICreate"
+    PROJECT_TYPE = "GenAI"
+    PROJECT_DESCRIPTION = "DESCRIPTION"
+    ANNOTATION_PATH = (
+        "data_set/sample_project_vector/example_image_1.jpg___objects.json"
+    )
+
+    @property
+    def annotation_path(self):
+        return os.path.join(Path(__file__).parent.parent.parent, self.ANNOTATION_PATH)
+
+    def test_search(self):
+        projects = sa.search_projects(self.PROJECT_NAME, return_metadata=True)
+        assert projects
+
+        sa.create_annotation_class(
+            self.PROJECT_NAME,
+            "class1",
+            "#FFAAFF",
+            [
+                {
+                    "name": "Human",
+                    "attributes": [{"name": "yes"}, {"name": "no"}],
+                },
+                {
+                    "name": "age",
+                    "attributes": [{"name": "young"}, {"name": "old"}],
+                },
+            ],
+        )
+        sa.attach_items(self.PROJECT_NAME, attachments=[{"url": "", "name": "name"}])
+        annotation = json.load(open(self.annotation_path))
+        annotation["metadata"]["name"] = "name"
+        sa.upload_annotations(self.PROJECT_NAME, annotations=[annotation])
+        data = sa.get_annotations(self.PROJECT_NAME)
+        assert data
+
+
 class TestProjectBasic(BaseTestCase):
     PROJECT_NAME = "TestWorkflowGet"
     PROJECT_TYPE = "Vector"
diff --git a/tests/unit/test_classes_serialization.py b/tests/unit/test_classes_serialization.py
@@ -43,7 +43,7 @@ def test_empty_multiselect_excluded(self):
             "type": 1,
             "name": "asd",
             "color": "#0000FF",
-            "attribute_groups": [{"name": "sad"}],
+            "attribute_groups": [{"name": "sad", "isRequired": False}],
         } == serializer_data
 
     def test_empty_multiselect_bool_serializer(self):
@@ -59,7 +59,7 @@ def test_empty_multiselect_bool_serializer(self):
             "type": 1,
             "name": "asd",
             "color": "#0000FF",
-            "attribute_groups": [{"name": "sad"}],
+            "attribute_groups": [{"name": "sad", "isRequired": False}],
         } == serializer_data
 
     def test_group_type_wrong_arg(self):

Original file line number	Diff line number	Diff line change
`@@ -79,7 +79,7 @@`
`79`	`79`	`"Tiled",`
`80`	`80`	`"Other",`
`81`	`81`	`"PointCloud",`
`82`		`- "CustomEditor",`
	`82`	`+ "GenAI",`
`83`	`83`	`]`
`84`	`84`
`85`	`85`	`ANNOTATION_STATUS = Literal[`