Merge branch 'master' of https://github.com/superannotateai/superannotate-python-sdk

Author: agyurjin

Makefile

@@ -4,7 +4,6 @@ PYTHON=python3
 PYLINT=pylint
 PYTESTS=pytest
 COVERAGE=coverage
-BROWSER=google-chrome
 
 all: coverage tests
 	$(PYTHON) setup.py build_ext --inplace
@@ -25,15 +24,17 @@ clean:
 coverage: test_coverage
 
 test_coverage:
-	$(COVERAGE) run -m  --source=./superannotate/  $(PYTESTS)
+	$(PYTESTS) --cov=superannotate -n 8
 	$(COVERAGE) html
 	@echo "\033[95m\n\nCoverage successful! View the output at file://htmlcov/index.html.\n\033[0m"
 
 install:
 	pip install .
 
 lint:
-	-$(PYLINT) superannotate/
+	-$(PYLINT) --output-format=json superannotate/ | pylint-json2html -o pylint.html
+
+lint_tests:
 	-$(PYLINT) tests/*
 
 docs:

README.md

@@ -29,7 +29,13 @@ pip install 'git+https://github.com/philferriere/cocoapi.git#egg=pycocotools&sub
 ```
 
 
-The package officially supports Python 3.6+.
+The package officially supports Python 3.6+ and was tested under Linux platform.
+For Windows based Anaconda distribution 
+you might also need to install *shapely* package separately:
+
+```console
+conda install shapely
+```
 
 For further steps please have a look at the [tutorial](https://superannotate.readthedocs.io/en/latest/tutorial.sdk.html).
 

docs/source/benchmark_annotators_box.png

@@ -50,7 +50,8 @@ To upload images from folder to project use:
 If optional argument *recursive* is given then subfolders of :file:`<folder_path>` are also recursively
 scanned for available images.
 
-Optional argument *extensions* accepts comma separated list of image extensions to look for. If the argument is not given then value *jpg,png* is assumed.
+Optional argument *extensions* accepts comma separated list of image extensions
+to look for. If the argument is not given then value *jpg,png* is assumed.
 
 ----------
 
@@ -69,7 +70,7 @@ To upload videos from folder to project use:
                                [--end-time <float>]
 
 If optional argument *recursive* is given then subfolders of :file:`<folder_path>` are also recursively
-scanned for available images.
+scanned for available videos.
 
 Optional argument *extensions* accepts comma separated list of image extensions
 to look for. If the argument is not given then value *mp4,avi,mov,webm,flv,mpg,ogg* is assumed.
@@ -83,6 +84,42 @@ default is 0.0.
 *end-time* specifies time (in seconds) up to which to extract frames. 
 If it is not specified, then up to end is assumed.
 
+----------
+
+.. _ref_upload_preannotations:
+
+Uploading preannotations
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To upload preannotations from folder to project use:
+
+.. code-block:: bash
+
+   superannotate upload-preannotations --project <project_name> --folder <folder_path> 
+                                       [--format "COCO" or "SA"] [--dataset-name "<dataset_name_for_COCO_projects>"]
+                                       [--task "<task_type_for_COCO_projects>]
+
+
+Optional argument *format* accepts input preannotation format. It can have COCO or SuperAnnotate values.
+If the argument is not given then SuperAnnotate (the native preannotation format) assumed.
+
+----------
+
+.. _ref_upload_annotations:
+
+Uploading annotations
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+To upload annotations from folder to project use:
+
+.. code-block:: bash
+
+   superannotate upload-annotations --project <project_name> --folder <folder_path> 
+                                    [--format "COCO" or "SA"] [--dataset-name "<dataset_name_for_COCO_projects>"]
+                                    [--task "<task_type_for_COCO_projects>]
+
+Optional argument *format* accepts input preannotation format. It can have COCO or SuperAnnotate values.
+If the argument is not given then SuperAnnotate (the native preannotation format) assumed.
 
 ----------
 

docs/source/benchmark_projects_box.png

@@ -54,7 +54,13 @@ for COCO annotation format converters support also need to install:
    pip install 'git+https://github.com/cocodataset/panopticapi.git'
    pip install 'git+https://github.com/philferriere/cocoapi.git#egg=pycocotools&subdirectory=PythonAPI'
 
-The package officially supports Python 3.6+.
+The package officially supports Python 3.6+ and was tested under Linux platform.
+For Windows based Anaconda distribution 
+you might also need to install :py:obj:`shapely` package separately:
+
+.. code-block:: bash
+
+   conda install shapely
 
 For further steps please have a look at the :ref:`tutorial <ref_tutorial>`.
 

docs/source/benchmark_scatter.png

@@ -23,6 +23,7 @@ ________
 .. _ref_search_projects:
 .. autofunction:: superannotate.search_projects
 .. autofunction:: superannotate.create_project
+.. autofunction:: superannotate.clone_project
 .. autofunction:: superannotate.create_project_like_project
 .. autofunction:: superannotate.delete_project
 .. autofunction:: superannotate.rename_project
@@ -322,3 +323,5 @@ Utility functions
 --------------------------------
 
 .. autofunction:: superannotate.dicom_to_rgb_sequence
+.. autofunction:: superannotate.consensus
+.. autofunction:: superannotate.benchmark

docs/source/cli.rst

@@ -24,7 +24,13 @@ for COCO annotation format converters support also need to install:
    pip install 'git+https://github.com/cocodataset/panopticapi.git'
    pip install 'git+https://github.com/philferriere/cocoapi.git#egg=pycocotools&subdirectory=PythonAPI'
 
-The package officially supports Python 3.6+.
+The package officially supports Python 3.6+ and was tested under Linux platform.
+For Windows based Anaconda distribution 
+you might also need to install :py:obj:`shapely` package separately:
+
+.. code-block:: bash
+
+   conda install shapely
 
 ----------
 
@@ -538,4 +544,45 @@ Analogically the box plots of consensus scores for each project are colored acco
 Scatter plot of consensus score vs instance area is separated by projects. Hovering on a point reveals its annotator and image name. 
 The points are colored according to class name. Each annotator is represented with separate symbol.
 
-.. image:: consensus_scatter.png
+.. image:: consensus_scatter.png
+
+----------
+
+
+Computing benchmark scores for instances between ground truth project and given project list
+____________________________________________________________________________________________
+
+
+Benchmark is a tool to compare the quallity of the annotations of the same image that is present in several projects with 
+the ground truth annotation of the same image that is in a separate project.
+
+To compute the benchmark scores:
+
+.. code-block:: python
+
+   res_df = sa.benchmark("<ground_truth_project_name>",[project_names], "<path_to_export_folder>", [image_list], "<annotation_type>")
+
+Here pandas DataFrame with exactly same structure as in case of consensus computation is returned.
+
+Besides the pandas DataFrame there is an option to get the following plots by setting the show_plots flag to True:
+
+* Box plot of benchmark scores for each annotators
+* Box plot of benchmark scores for each project
+* Scatter plots of benchmark score vs instance area for each project
+
+.. code-block:: python
+
+   sa.benchmark("<ground_truth_project_name>", [project_names], "<path_to_export_folder>", [image_list], "<annotation_type>", show_plots=True)
+
+To the left of each box plot the original score points of that annotator is depicted, the box plots are colored by annotator.
+
+.. image:: benchmark_annotators_box.png
+
+Analogically the box plots of benchmark scores for each project are colored according to project name.
+
+.. image:: benchmark_projects_box.png
+
+Scatter plot of benchmark score vs instance area is separated by projects. Hovering on a point reveals its annotator and image name. 
+The points are colored according to class name. Each annotator is represented with separate symbol.
+
+.. image:: benchmark_scatter.png

docs/source/index.rst

@@ -16,9 +16,10 @@ pip install 'git+https://github.com/cocodataset/panopticapi.git'
 pip install 'git+https://github.com/philferriere/cocoapi.git#egg=pycocotools&subdirectory=PythonAPI'
 
 pip install -e .
+# pip install superannotate
 
 # for testing
-pip install pytest pytest-xdist
+pip install pytest pytest-xdist coverage pytest-cov pylint pylint-json2html
 
 # for docs
 pip install sphinx sphinx_rtd_theme