Merge pull request #16 from superannotateai/develop

Attach Public URLs

Author: rcmanga

Makefile

@@ -9,7 +9,7 @@ PYTESTS=pytest
 COVERAGE=coverage
 
 tests: check_formatting docs
-	$(PYTESTS) -n auto --full-trace --verbose tests
+	$(PYTESTS) -n auto --full-trace tests
 
 stress-tests: SA_STRESS_TESTS=1
 stress-tests: tests

docs/source/cli.rst

@@ -79,6 +79,19 @@ to look for. If the argument is not given then value *jpg,jpeg,png,tif,tiff,webp
 
 ----------
 
+.. _ref_attach_image_urls:
+
+Attaching image URLs
+~~~~~~~~~~~~~~~~~~~~
+
+To attach image URLs to project use:
+
+.. code-block:: bash
+
+   superannotatecli attach-image-urls --project <project_name/folder_name> --attachments <csv_path> [--annotation_status <annotation_status>]
+
+----------
+
 .. _ref_upload_videos:
 
 Uploading videos

docs/source/superannotate.sdk.rst

@@ -37,6 +37,7 @@ ________
 .. autofunction:: superannotate.delete_folders
 .. autofunction:: superannotate.rename_folder
 .. autofunction:: superannotate.upload_images_to_project
+.. autofunction:: superannotate.attach_image_urls_to_project
 .. autofunction:: superannotate.upload_images_from_public_urls_to_project
 .. autofunction:: superannotate.upload_images_from_google_cloud_to_project
 .. autofunction:: superannotate.upload_images_from_azure_blob_to_project

docs/source/tutorial.sdk.rst

@@ -98,6 +98,10 @@ To create a new "Vector" project with name "Example Project 1" and description
 
     sa.create_project(project, "test", "Vector")
 
+.. warning::
+
+   In general, SDK functions are not thread-safe.
+
 Creating a folder in a project
 ______________________________
 

superannotate/__init__.py

@@ -72,8 +72,8 @@ def consensus(*args, **kwargs):
     upload_images_from_google_cloud_to_project,
     upload_images_from_public_urls_to_project,
     upload_images_from_s3_bucket_to_project, upload_images_to_project,
-    upload_preannotations_from_folder_to_project, upload_video_to_project,
-    upload_videos_from_folder_to_project
+    attach_image_urls_to_project, upload_preannotations_from_folder_to_project,
+    upload_video_to_project, upload_videos_from_folder_to_project
 )
 from .db.search_projects import search_projects
 from .db.teams import (

superannotate/__main__.py

@@ -62,6 +62,8 @@ def main():
         create_folder(command, further_args)
     elif command == "upload-images":
         image_upload(command, further_args)
+    elif command == "attach-image-urls":
+        attach_image_urls(command, further_args)
     elif command == "upload-videos":
         video_upload(command, further_args)
     elif command in ["upload-preannotations", "upload-annotations"]:
@@ -288,6 +290,31 @@ def image_upload(command_name, args):
     )
 
 
+def attach_image_urls(command_name, args):
+    parser = argparse.ArgumentParser(prog=_CLI_COMMAND + " " + command_name)
+    parser.add_argument(
+        '--project', required=True, help='Project name to upload'
+    )
+    parser.add_argument(
+        '--attachments',
+        required=True,
+        help='path to csv file on attachments metadata'
+    )
+    parser.add_argument(
+        '--annotation_status',
+        required=False,
+        default="NotStarted",
+        help=
+        'Set images\' annotation statuses after upload. Default is NotStarted'
+    )
+    args = parser.parse_args(args)
+    sa.attach_image_urls_to_project(
+        project=args.project,
+        attachments=args.attachments,
+        annotation_status=args.annotation_status
+    )
+
+
 def export_project(command_name, args):
     parser = argparse.ArgumentParser(prog=_CLI_COMMAND + " " + command_name)
     parser.add_argument(

superannotate/common.py

@@ -29,6 +29,10 @@
     "Completed": 5,
     "Skipped": 6
 }
+
+_UPLOAD_STATES_STR_TO_CODES = {"Initial": 1, "Basic": 2, "External": 3}
+_UPLOAD_STATES_CODES_TO_STR = {1: "Initial", 2: "Basic", 3: "External"}
+
 _USER_ROLES = {"Admin": 2, "Annotator": 3, "QA": 4, "Customer": 5, "Viewer": 6}
 _AVAILABLE_SEGMENTATION_MODELS = ['autonomous', 'generic']
 _MODEL_TRAINING_STATUSES = {
@@ -118,6 +122,14 @@ def annotation_status_str_to_int(annotation_status):
     return _ANNOTATION_STATUSES[annotation_status]
 
 
+def upload_state_str_to_int(upload_state):
+    return _UPLOAD_STATES_STR_TO_CODES[upload_state]
+
+
+def upload_state_int_to_str(upload_state):
+    return _UPLOAD_STATES_CODES_TO_STR[upload_state]
+
+
 def annotation_status_int_to_str(annotation_status):
     """Converts metadata annotation_status int value to a string
 

superannotate/db/exports.py

@@ -6,13 +6,14 @@
 import zipfile
 from datetime import datetime
 from pathlib import Path
+import shutil
 
 import boto3
 import requests
 from tqdm import tqdm
 
 from ..api import API
-from ..common import annotation_status_str_to_int
+from ..common import annotation_status_str_to_int, upload_state_int_to_str
 from ..exceptions import (
     SABaseException, SAExistingExportNameException,
     SANonExistingExportNameException
@@ -123,6 +124,12 @@ def prepare_export(
     """
     if not isinstance(project, dict):
         project = get_project_metadata_bare(project)
+    upload_state = upload_state_int_to_str(project.get("upload_state"))
+    if upload_state == "External" and include_fuse == True:
+        logger.info(
+            "Include fuse functionality is not supported for  projects containing images attached with URLs"
+        )
+        include_fuse = False
     team_id, project_id = project["team_id"], project["id"]
     if annotation_statuses is None:
         annotation_statuses = [2, 3, 4, 5]
@@ -203,6 +210,15 @@ def __upload_files_to_aws_thread(
             already_uploaded[i] = True
 
 
+def _download_file(url, local_filename):
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(local_filename, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+    return local_filename
+
+
 def download_export(
     project, export, folder_path, extract_zip_contents=True, to_s3_bucket=None
 ):
@@ -237,25 +253,26 @@ def download_export(
         break
 
     filename = Path(res['path']).name
-    r = requests.get(res['download'], allow_redirects=True)
-    if to_s3_bucket is None:
-        filepath = Path(folder_path) / filename
-        open(filepath, 'wb').write(r.content)
-        if extract_zip_contents:
-            with zipfile.ZipFile(filepath, 'r') as f:
-                f.extractall(folder_path)
-            Path.unlink(filepath)
-            logger.info("Extracted %s to folder %s", filepath, folder_path)
-        else:
-            logger.info("Downloaded export ID %s to %s", res['id'], filepath)
-    else:
-        with tempfile.TemporaryDirectory() as tmpdirname:
-            filepath = Path(tmpdirname) / filename
-            open(filepath, 'wb').write(r.content)
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        temp_filepath = Path(tmpdirname) / filename
+        _download_file(res['download'], temp_filepath)
+        if to_s3_bucket is None:
+            filepath = Path(folder_path) / filename
+            shutil.copyfile(temp_filepath, filepath)
             if extract_zip_contents:
                 with zipfile.ZipFile(filepath, 'r') as f:
-                    f.extractall(tmpdirname)
+                    f.extractall(folder_path)
                 Path.unlink(filepath)
+                logger.info("Extracted %s to folder %s", filepath, folder_path)
+            else:
+                logger.info(
+                    "Downloaded export ID %s to %s", res['id'], filepath
+                )
+        else:
+            if extract_zip_contents:
+                with zipfile.ZipFile(temp_filepath, 'r') as f:
+                    f.extractall(tmpdirname)
+                Path.unlink(temp_filepath)
             files_to_upload = []
             for file in Path(tmpdirname).rglob("*.*"):
                 if not file.is_file():
@@ -290,4 +307,4 @@ def download_export(
                     t.join()
             finish_event.set()
             tqdm_thread.join()
-        logger.info("Exported to AWS %s/%s", to_s3_bucket, folder_path)
+            logger.info("Exported to AWS %s/%s", to_s3_bucket, folder_path)

superannotate/db/images.py

@@ -621,6 +621,11 @@ def download_image(
     :return: paths of downloaded image and annotations if included
     :rtype: tuple
     """
+    if (include_fuse or include_overlay) and not include_annotations:
+        raise SABaseException(
+            0,
+            "To download fuse or overlay image need to set include_annotations=True in download_image"
+        )
     if not Path(local_dir_path).is_dir():
         raise SABaseException(
             0, f"local_dir_path {local_dir_path} is not an existing directory"
@@ -631,6 +636,12 @@ def download_image(
         )
 
     project, project_folder = get_project_and_folder_metadata(project)
+    upload_state = common.upload_state_int_to_str(project.get("upload_state"))
+    if upload_state == "External":
+        raise SABaseException(
+            0,
+            "The function does not support projects containing images attached with URLs"
+        )
     img = get_image_bytes(
         (project, project_folder), image_name, variant=variant
     )
@@ -698,6 +709,13 @@ def get_image_bytes(project, image_name, variant='original'):
     :return: io.BytesIO() of the image
     :rtype: io.BytesIO()
     """
+    project, project_folder = get_project_and_folder_metadata(project)
+    upload_state = common.upload_state_int_to_str(project.get("upload_state"))
+    if upload_state == "External":
+        raise SABaseException(
+            0,
+            "The function does not support projects containing images attached with URLs"
+        )
     if variant not in ["original", "lores"]:
         raise SABaseException(
             0, "Image download variant should be either original or lores"
@@ -818,7 +836,7 @@ def _get_image_pre_or_annotations(project, image_name, pre):
     fill_class_and_attribute_names(res_json, annotation_classes_dict)
     result = {
         f"{pre}annotation_json":
-            response.json(),
+            res_json,
         f"{pre}annotation_json_filename":
             common.get_annotation_json_name(image_name, project_type)
     }
@@ -1041,12 +1059,11 @@ def create_fuse_image(
             (image_size[1], image_size[0], 4), [0, 0, 0, 255], np.uint8
         )
         fi_ovl[:, :, :3] = np.array(pil_image)
+        fi_pil_ovl = Image.fromarray(fi_ovl)
+        draw_ovl = ImageDraw.Draw(fi_pil_ovl)
     if project_type == "Vector":
         fi_pil = Image.fromarray(fi)
         draw = ImageDraw.Draw(fi_pil)
-        if output_overlay:
-            fi_pil_ovl = Image.fromarray(fi_ovl)
-            draw_ovl = ImageDraw.Draw(fi_pil_ovl)
         for annotation in annotation_json["instances"]:
             if "className" not in annotation:
                 continue
@@ -1159,6 +1176,11 @@ def create_fuse_image(
                 temp_mask = np.alltrue(annotation_mask == part_color, axis=2)
                 fi[temp_mask] = fill_color
         fi_pil = Image.fromarray(fi)
+        alpha = 0.5  # transparency measure
+        if output_overlay:
+            fi_pil_ovl = Image.fromarray(
+                cv2.addWeighted(fi, alpha, fi_ovl, 1 - alpha, 0)
+            )
 
     if in_memory:
         if output_overlay: