[PTDT-2854] MAL and GT support for pdf relationships

[lgluszek]

Description

Added MAL and GT support for connecting classifications to PDF annotations (DocumentRectangle and DocumentEntity) using relationships.

How to test

• MAL and GT import work for relationships connecting classifications and bounding boxes/named entities

import labelbox as lb
import uuid
import labelbox.types as lb_types
from labelbox.data.annotation_types import RectangleUnit

client = lb.Client(
    api_key="",
    endpoint="https://app.lb-stage.xyz/api/_gql",
    rest_endpoint="https://app.lb-stage.xyz/api/api/v1",
)

radio_annotation = lb_types.ClassificationAnnotation(
    name="radio_pdf",
    value=lb_types.Radio(
        answer=lb_types.ClassificationAnswer(name="second_radio_answer")
    ),
)

checklist_annotation = lb_types.ClassificationAnnotation(
    name="checklist_pdf",  # must match your ontology feature's name
    value=lb_types.Checklist(
        answer=[
            lb_types.ClassificationAnswer(name="first_checklist_answer"),
            lb_types.ClassificationAnswer(name="second_checklist_answer"),
        ]
    ),
)

text_annotation = lb_types.ClassificationAnnotation(
    name="text_pdf", value=lb_types.Text(answer="sample text")
)

bbox_annotation = lb_types.ObjectAnnotation(
    name="bounding_box",
    value=lb_types.DocumentRectangle(
        start=lb_types.Point(x=0.5, y=0.5),  #  x = left, y = top
        end=lb_types.Point(x=0.7, y=0.7),  # x= left + width , y = top + height
        page=1,
        unit=RectangleUnit.PERCENT,
    ),
)

bbox_annotation_2 = lb_types.ObjectAnnotation(
    name="bounding_box",
    value=lb_types.DocumentRectangle(
        start=lb_types.Point(x=0.5, y=0.5),
        end=lb_types.Point(x=0.7, y=0.7),
        page=2,
        unit=RectangleUnit.PERCENT,
    ),
)

entity_annotation = lb_types.ObjectAnnotation(
    name="named_entity",
    value=lb_types.DocumentEntity(
        name="named_entity",
        textSelections=[
            lb_types.DocumentTextSelection(
                token_ids=["9763b68b-9ea7-472d-af40-20ab69fbee71"],
                group_id="ee108a07-059a-4302-910f-b16eb148f123",
                page=1,
            )
        ],
    ),
)

text_bbox_relationship = lb_types.RelationshipAnnotation(
    name="relationship",
    value=lb_types.Relationship(
        source=text_annotation,
        target=bbox_annotation,
        type=lb_types.Relationship.Type.UNIDIRECTIONAL,
    ),
)

radio_bbox_relationship = lb_types.RelationshipAnnotation(
    name="relationship",
    value=lb_types.Relationship(
        source=radio_annotation,
        target=bbox_annotation,
        type=lb_types.Relationship.Type.UNIDIRECTIONAL,
    ),
)

checklist_bbox_relationship = lb_types.RelationshipAnnotation(
    name="relationship",
    value=lb_types.Relationship(
        source=checklist_annotation,
        target=bbox_annotation,
        type=lb_types.Relationship.Type.UNIDIRECTIONAL,
    ),
)


text_entity_relationship = lb_types.RelationshipAnnotation(
    name="relationship",
    value=lb_types.Relationship(
        source=text_annotation,
        target=entity_annotation,
        type=lb_types.Relationship.Type.UNIDIRECTIONAL,
    ),
)

bbox_bbox2_relationship = lb_types.RelationshipAnnotation(
    name="relationship",
    value=lb_types.Relationship(
        source=bbox_annotation,
        target=bbox_annotation_2,
        type=lb_types.Relationship.Type.UNIDIRECTIONAL,
    ),
)

global_key = f"pdf_{uuid.uuid4()}"

test_img_url = {
    "row_data": {
        "pdf_url": "https://storage.googleapis.com/labelbox-developer-testing-assets/pdf-with-percents.pdf",
        "text_layer_url": "https://storage.googleapis.com/labelbox-developer-testing-assets/text-layer-percent-unit.json",
    },
    "global_key": global_key,
}

dataset = client.create_dataset(name="demo_dataset_pdf")
task = dataset.create_data_rows([test_img_url])
task.wait_till_done()
print("Errors:", task.errors)
print("Failed data rows:", task.failed_data_rows)
ontology_builder = lb.OntologyBuilder(
    classifications=[
        lb.Classification(class_type=lb.Classification.Type.TEXT, name="text_pdf"),
        lb.Classification(
            class_type=lb.Classification.Type.CHECKLIST,
            name="checklist_pdf",
            options=[
                lb.Option(value="first_checklist_answer"),
                lb.Option(value="second_checklist_answer"),
            ],
        ),
        lb.Classification(
            class_type=lb.Classification.Type.RADIO,
            name="radio_pdf",
            options=[
                lb.Option(value="first_radio_answer"),
                lb.Option(value="second_radio_answer"),
            ],
        ),
    ],
    tools=[  # List of tools
        lb.Tool(tool=lb.Tool.Type.BBOX, name="bounding_box"),
        lb.Tool(tool=lb.Tool.Type.RELATIONSHIP, name="relationship"),
        lb.Tool(tool=lb.Tool.Type.NER, name="named_entity"),
    ],
)

ontology = client.create_ontology(
    "Ontology PDF Annotations", ontology_builder.asdict(), media_type=lb.MediaType.Pdf
)
project = client.create_project(name="pdf_project 2", media_type=lb.MediaType.Pdf)

# Setup your ontology
project.connect_ontology(ontology)
batch = project.create_batch(
    "first-batch-pdf-demo",  # Each batch in a project must have a unique name
    global_keys=[
        global_key
    ],  # Paginated collection of data row objects, list of data row ids or global keys
    priority=5,  # priority between 1(highest) - 5(lowest)
)

print("Batch: ", batch)
label = []
label.append(
    lb_types.Label(
        data={"global_key": global_key},
        annotations=[
            text_annotation,
            checklist_annotation,
            radio_annotation,
            bbox_annotation,
            bbox_annotation_2,
            entity_annotation,
            text_bbox_relationship,
            radio_bbox_relationship,
            checklist_bbox_relationship,
            text_entity_relationship,
            bbox_bbox2_relationship,
        ],
    )
)
# MAL
upload_job = lb.MALPredictionImport.create_from_objects(
    client=client,
    project_id=project.uid,
    name=f"mal_job-{str(uuid.uuid4())}",
    predictions=label,
)

# GT
# upload_job = lb.LabelImport.create_from_objects(
#     client=client,
#     project_id=project.uid,
#     name="label_import_job" + str(uuid.uuid4()),
#     labels=label,
# )

upload_job.wait_until_done()
print("Errors:", upload_job.errors)
print("Status of uploads: ", upload_job.statuses)

• MAL and GT import of relationships works in image

Type of change

Please delete options that are not relevant.

• New feature (non-breaking change which adds functionality)

All Submissions

• Have you followed the guidelines in our Contributing document?
• Have you provided a description?
• Are your changes properly formatted?

New Feature Submissions

• Does your submission pass tests?
• Have you added thorough tests for your new feature?
• Have you commented your code, particularly in hard-to-understand areas?
• Have you added a Docstring?

Changes to Core Features

• Have you written new tests for your core changes, as applicable?
• Have you successfully run tests with your changes locally?
• Have you updated any code comments, as applicable?

[lgluszek]

The relationships are currently included twice in the result when calling NDJsonConverter.serialize.

Because we already generate NDRelationship in yield from cls._create_relationship_annotations(label), there is no need to also do it in yield from cls._create_non_video_annotations(label)

[Gabefire]

LGTM!