LCORE-1179: Update metadata extraction from chunks

docs/openapi.json

@@ -3664,7 +3664,7 @@
                     "rlsapi-v1"
                 ],
                 "summary": "Infer Endpoint",
-                "description": "Handle rlsapi v1 /infer requests for stateless inference.\n\nThis endpoint serves requests from the RHEL Lightspeed Command Line Assistant (CLA).\n\nAccepts a question with optional context (stdin, attachments, terminal output,\nsystem info) and returns an LLM-generated response.\n\nArgs:\n    infer_request: The inference request containing question and context.\n    auth: Authentication tuple from the configured auth provider.\n\nReturns:\n    RlsapiV1InferResponse containing the generated response text and request ID.\n\nRaises:\n    HTTPException: 503 if the LLM service is unavailable.",
+                "description": "Handle rlsapi v1 /infer requests for stateless inference.\n\nThis endpoint serves requests from the RHEL Lightspeed Command Line Assistant (CLA).\n\nAccepts a question with optional context (stdin, attachments, terminal output,\nsystem info) and returns an LLM-generated response.\n\nArgs:\n    infer_request: The inference request containing question and context.\n    request: The FastAPI request object for accessing headers and state.\n    background_tasks: FastAPI background tasks for async Splunk event sending.\n    auth: Authentication tuple from the configured auth provider.\n\nReturns:\n    RlsapiV1InferResponse containing the generated response text and request ID.\n\nRaises:\n    HTTPException: 503 if the LLM service is unavailable.",
                 "operationId": "infer_endpoint_v1_infer_post",
                 "requestBody": {
                     "content": {
@@ -8431,11 +8431,23 @@
                         ],
                         "title": "Doc Title",
                         "description": "Title of the referenced document"
+                    },
+                    "doc_id": {
+                        "anyOf": [
+                            {
+                                "type": "string"
+                            },
+                            {
+                                "type": "null"
+                            }
+                        ],
+                        "title": "Doc Id",
+                        "description": "ID of the referenced document"
                     }
                 },
                 "type": "object",
                 "title": "ReferencedDocument",
-                "description": "Model representing a document referenced in generating a response.\n\nAttributes:\n    doc_url: Url to the referenced doc.\n    doc_title: Title of the referenced doc."
+                "description": "Model representing a document referenced in generating a response.\n\nAttributes:\n    doc_url: Url to the referenced doc.\n    doc_title: Title of the referenced doc.\n    doc_id: ID of the referenced doc."
             },
             "RlsapiV1Attachment": {
                 "properties": {

docs/openapi.md

@@ -3200,6 +3200,8 @@ system info) and returns an LLM-generated response.
 
 Args:
     infer_request: The inference request containing question and context.
+    request: The FastAPI request object for accessing headers and state.
+    background_tasks: FastAPI background tasks for async Splunk event sending.
     auth: Authentication tuple from the configured auth provider.
 
 Returns:
@@ -5316,12 +5318,14 @@ Model representing a document referenced in generating a response.
 Attributes:
     doc_url: Url to the referenced doc.
     doc_title: Title of the referenced doc.
+    doc_id: ID of the referenced doc.
 
 
 | Field | Type | Description |
 |-------|------|-------------|
 | doc_url |  | URL of the referenced document |
 | doc_title |  | Title of the referenced document |
+| doc_id |  | ID of the referenced document |
 
 
 ## RlsapiV1Attachment

src/app/endpoints/query.py

@@ -1,8 +1,6 @@
 """Handler for REST API call to provide answer to query."""
 
-import ast
 import logging
-import re
 from datetime import UTC, datetime
 from typing import Annotated, Any, Optional
 
@@ -14,7 +12,6 @@
     RateLimitError,  # type: ignore
 )
 from llama_stack_client.types.model_list_response import ModelListResponse
-from llama_stack_client.types.shared.interleaved_content_item import TextContentItem
 from sqlalchemy.exc import SQLAlchemyError
 
 import constants
@@ -36,7 +33,6 @@
     PromptTooLongResponse,
     QueryResponse,
     QuotaExceededResponse,
-    ReferencedDocument,
     ServiceUnavailableResponse,
     UnauthorizedResponse,
     UnprocessableEntityResponse,
@@ -553,39 +549,6 @@ def is_input_shield(shield: Shield) -> bool:
     return _is_inout_shield(shield) or not is_output_shield(shield)
 
 
-def parse_metadata_from_text_item(
-    text_item: TextContentItem,
-) -> Optional[ReferencedDocument]:
-    """
-    Parse a single TextContentItem to extract referenced documents.
-
-    Args:
-        text_item (TextContentItem): The TextContentItem containing metadata.
-
-    Returns:
-        ReferencedDocument: A ReferencedDocument object containing 'doc_url' and 'doc_title'
-        representing the referenced documents found in the metadata.
-    """
-    docs: list[ReferencedDocument] = []
-    if not isinstance(text_item, TextContentItem):
-        return docs
-
-    metadata_blocks = re.findall(
-        r"Metadata:\s*({.*?})(?:\n|$)", text_item.text, re.DOTALL
-    )
-    for block in metadata_blocks:
-        try:
-            data = ast.literal_eval(block)
-            url = data.get("docs_url")
-            title = data.get("title")
-            if url and title:
-                return ReferencedDocument(doc_url=url, doc_title=title)
-            logger.debug("Invalid metadata block (missing url or title): %s", block)
-        except (ValueError, SyntaxError) as e:
-            logger.debug("Failed to parse metadata block: %s | Error: %s", block, e)
-    return None
-
-
 def validate_attachments_metadata(attachments: list[Attachment]) -> None:
     """Validate the attachments metadata provided in the request.
 

src/app/endpoints/query_v2.py

@@ -541,11 +541,11 @@ def parse_referenced_documents_from_responses_api(
         response: The OpenAI Response API response object
 
     Returns:
-        list[ReferencedDocument]: List of referenced documents with doc_url and doc_title
+        list[ReferencedDocument]: List of referenced documents with doc_url, doc_title and doc_id
     """
     documents: list[ReferencedDocument] = []
-    # Use a set to track unique documents by (doc_url, doc_title) tuple
-    seen_docs: set[tuple[Optional[str], Optional[str]]] = set()
+    # Use a set to track unique documents by (doc_url, doc_title, doc_id) tuple
+    seen_docs: set[tuple[Optional[str], Optional[str], Optional[str]]] = set()
 
     # Handle None response (e.g., when agent fails)
     if response is None or not response.output:
@@ -560,74 +560,31 @@ def parse_referenced_documents_from_responses_api(
             for result in results:
                 # Handle both object and dict access
                 if isinstance(result, dict):
-                    filename = result.get("filename")
                     attributes = result.get("attributes", {})
                 else:
-                    filename = getattr(result, "filename", None)
                     attributes = getattr(result, "attributes", {})
 
                 # Try to get URL from attributes
                 # Look for common URL fields in attributes
                 doc_url = (
-                    attributes.get("link")
+                    attributes.get("doc_url")
+                    or attributes.get("docs_url")
                     or attributes.get("url")
-                    or attributes.get("doc_url")
+                    or attributes.get("link")
                 )
+                doc_title = attributes.get("title")
+                doc_id = attributes.get("document_id") or attributes.get("doc_id")
 
-                # If we have at least a filename or url
-                if filename or doc_url:
+                if doc_title or doc_url:
                     # Treat empty string as None for URL to satisfy Optional[AnyUrl]
                     final_url = doc_url if doc_url else None
-                    if (final_url, filename) not in seen_docs:
+                    if (final_url, doc_title, doc_id) not in seen_docs:
                         documents.append(
-                            ReferencedDocument(doc_url=final_url, doc_title=filename)
-                        )
-                        seen_docs.add((final_url, filename))
-
-        # 2. Parse from message content annotations
-        elif item_type == "message":
-            content = getattr(output_item, "content", None)
-            if isinstance(content, list):
-                for part in content:
-                    # Skip if part is a string or doesn't have annotations
-                    if isinstance(part, str):
-                        continue
-
-                    annotations = getattr(part, "annotations", []) or []
-                    for annotation in annotations:
-                        # Handle both object and dict access for annotations
-                        if isinstance(annotation, dict):
-                            anno_type = annotation.get("type")
-                            anno_url = annotation.get("url")
-                            anno_title = annotation.get("title") or annotation.get(
-                                "filename"
-                            )
-                        else:
-                            anno_type = getattr(annotation, "type", None)
-                            anno_url = getattr(annotation, "url", None)
-                            anno_title = getattr(annotation, "title", None) or getattr(
-                                annotation, "filename", None
+                            ReferencedDocument(
+                                doc_url=final_url, doc_title=doc_title, doc_id=doc_id
                             )
-
-                        if anno_type == "url_citation":
-                            # Treat empty string as None
-                            final_url = anno_url if anno_url else None
-                            if (final_url, anno_title) not in seen_docs:
-                                documents.append(
-                                    ReferencedDocument(
-                                        doc_url=final_url, doc_title=anno_title
-                                    )
-                                )
-                                seen_docs.add((final_url, anno_title))
-
-                        elif anno_type == "file_citation":
-                            if (None, anno_title) not in seen_docs:
-                                documents.append(
-                                    ReferencedDocument(
-                                        doc_url=None, doc_title=anno_title
-                                    )
-                                )
-                                seen_docs.add((None, anno_title))
+                        )
+                        seen_docs.add((final_url, doc_title, doc_id))
 
     return documents
 

src/models/responses.py

@@ -8,8 +8,8 @@
 from pydantic import AnyUrl, BaseModel, Field
 from pydantic_core import SchemaError
 
-from quota.quota_exceed_error import QuotaExceedError
 from models.config import Action, Configuration
+from quota.quota_exceed_error import QuotaExceedError
 from utils.types import RAGChunk, ToolCallSummary, ToolResultSummary
 
 SUCCESSFUL_RESPONSE_DESCRIPTION = "Successful response"
@@ -369,6 +369,7 @@ class ReferencedDocument(BaseModel):
     Attributes:
         doc_url: Url to the referenced doc.
         doc_title: Title of the referenced doc.
+        doc_id: ID of the referenced doc.
     """
 
     doc_url: Optional[AnyUrl] = Field(
@@ -379,6 +380,8 @@ class ReferencedDocument(BaseModel):
         None, description="Title of the referenced document"
     )
 
+    doc_id: Optional[str] = Field(None, description="ID of the referenced document")
+
 
 class QueryResponse(AbstractSuccessfulResponse):
     """Model representing LLM response to a query.

tests/unit/app/endpoints/test_query.py

@@ -8,22 +8,18 @@
 
 import pytest
 from fastapi import HTTPException, Request, status
-from llama_stack_client.types.shared.interleaved_content_item import TextContentItem
-from pydantic import AnyUrl
 from pytest_mock import MockerFixture
 
 from app.endpoints.query import (
     evaluate_model_hints,
     is_transcripts_enabled,
-    parse_metadata_from_text_item,
     select_model_and_provider_id,
     validate_attachments_metadata,
 )
 from configuration import AppConfig
 from models.config import Action
 from models.database.conversations import UserConversation
 from models.requests import Attachment, QueryRequest
-from models.responses import ReferencedDocument
 from utils.token_counter import TokenCounter
 
 # User ID must be proper UUID
@@ -392,48 +388,6 @@ def test_validate_attachments_metadata_invalid_content_type() -> None:
     )
 
 
-def test_parse_metadata_from_text_item_valid(mocker: MockerFixture) -> None:
-    """Test parsing metadata from a TextContentItem."""
-    text = """
-    Some text...
-    Metadata: {"docs_url": "https://redhat.com", "title": "Example Doc"}
-    """
-    mock_item = mocker.Mock(spec=TextContentItem)
-    mock_item.text = text
-
-    doc = parse_metadata_from_text_item(mock_item)
-
-    assert isinstance(doc, ReferencedDocument)
-    assert doc.doc_url == AnyUrl("https://redhat.com")
-    assert doc.doc_title == "Example Doc"
-
-
-def test_parse_metadata_from_text_item_missing_title(mocker: MockerFixture) -> None:
-    """Test parsing metadata from a TextContentItem with missing title."""
-    mock_item = mocker.Mock(spec=TextContentItem)
-    mock_item.text = """Metadata: {"docs_url": "https://redhat.com"}"""
-    doc = parse_metadata_from_text_item(mock_item)
-    assert doc is None
-
-
-def test_parse_metadata_from_text_item_missing_url(mocker: MockerFixture) -> None:
-    """Test parsing metadata from a TextContentItem with missing url."""
-    mock_item = mocker.Mock(spec=TextContentItem)
-    mock_item.text = """Metadata: {"title": "Example Doc"}"""
-    doc = parse_metadata_from_text_item(mock_item)
-    assert doc is None
-
-
-def test_parse_metadata_from_text_item_malformed_url(mocker: MockerFixture) -> None:
-    """Test parsing metadata from a TextContentItem with malformed url."""
-    mock_item = mocker.Mock(spec=TextContentItem)
-    mock_item.text = (
-        """Metadata: {"docs_url": "not a valid url", "title": "Example Doc"}"""
-    )
-    doc = parse_metadata_from_text_item(mock_item)
-    assert doc is None
-
-
 def test_no_tools_parameter_backward_compatibility() -> None:
     """Test that default behavior is unchanged when no_tools parameter is not specified."""
     # This test ensures that existing code that doesn't specify no_tools continues to work