Feature/evaluation metrics (#27)

* feat(evaluation): add simple faithfulness overlap metric with tests

* feat(evaluation): add JSONL trace capture for RAG runs

* feat(evaluation): add JSONL trace capture for RAG runs

* feat(evaluation): add evaluation summary endpoint from trace aggregates

Author: kernelshard

app/api/router.py

@@ -7,6 +7,7 @@
 from app.retrieval.api import router as retrival_router
 from app.ingestion.api import router as ingestion_router
 from app.generation.api import router as generation_router
+from app.evaludation.api import router as evaluation_router
 
 
 router = APIRouter(prefix="/api/v1", tags=["api"])
@@ -22,3 +23,4 @@ async def ping() -> dict[str, Any]:
 router.include_router(retrival_router)
 router.include_router(ingestion_router)
 router.include_router(generation_router)
+router.include_router(evaluation_router)

app/evaludation/aggregate.py

@@ -0,0 +1,48 @@
+import json
+from pathlib import Path
+
+
+TRACE_FILE = Path(__file__).parent.parent.parent / "var/traces" / "rage_trace.jsonl"
+
+
+def aggregate_traces() -> dict:
+    """
+    Aggregate traces from the JSONL trace file.
+    Read the file line by line, parse each JSON record, and compute
+    average metrics like recall_k and faithfulness.
+
+    :return: Aggregated trace records.
+    :rtype: dict
+    """
+
+    total = 0
+    recall_k_sum = 0.0
+    recall_n = 0  # recall_k_count
+    faithfulness_sum = 0.0
+    faithfulness_n = 0  # faithfulness_count
+
+    if not TRACE_FILE.exists():
+        return {
+            "runs": 0,
+            "avg_recall_k": None,
+            "avg_faithfulness": None,
+        }
+
+    with TRACE_FILE.open("r", encoding="utf-8") as f:
+        for line in f:
+            total += 1
+            r = json.loads(line)
+            if r.get("recall_k") is not None:
+                recall_k_sum += r["recall_k"]
+                recall_n += 1
+            if r.get("faithfulness") is not None:
+                faithfulness_sum += r["faithfulness"]
+                faithfulness_n += 1
+
+    return {
+        "runs": total,
+        "avg_recall_k": round(recall_k_sum / recall_n, 4) if recall_n > 0 else None,
+        "avg_faithfulness": round(faithfulness_sum / faithfulness_n, 4)
+        if faithfulness_n > 0
+        else None,
+    }

app/evaludation/api.py

@@ -0,0 +1,11 @@
+from fastapi import APIRouter
+
+from app.evaludation.aggregate import aggregate_traces
+
+
+router = APIRouter(prefix="/eval", tags=["evaluation"])
+
+
+@router.get("/summary")
+async def evaluation_summary() -> dict:
+    return aggregate_traces()

app/evaludation/faithfulness.py

@@ -0,0 +1,37 @@
+import re
+from typing import Iterable
+
+
+def _tokens(text: str) -> set[str]:
+    """
+    Tokenize the input text into a set of normalized words.
+    """
+    return set(re.findall(r"\b\w+\b", text.lower()))
+
+
+def faithfulness_overlap(answer: str, contexts: Iterable[str]) -> float:
+    """
+    Calculate the faithfulness overlap score between the answer and provided contexts.
+    The score is the ratio of overlapping words to total words in the answer.
+
+    Args:
+        answer (str): The generated answer text.
+        contexts (Iterable[str]): A list of context strings.
+
+    Returns:
+        float: The faithfulness overlap score (0.0 to 1.0).
+    """
+    # Normalize and tokenize the answer
+    answer_tokens = _tokens(answer)
+    # means it has no tokens, e.g., empty strings
+    if not answer_tokens:
+        return 0.0
+
+    # Normalize and tokenize all contexts
+    context_tokens = set()
+    for context in contexts:
+        context_tokens.update(_tokens(context))
+
+    # Calculate overlap score
+    # intersection over answer tokens
+    return round(len(answer_tokens & context_tokens) / len(answer_tokens), 4)

app/evaludation/recall.py

@@ -0,0 +1,30 @@
+from typing import Iterable, Set
+
+
+def recall_at_k(retrieved_ids: Iterable[str], relevant_ids: Set[str], k: int) -> float:
+    """
+    Calculate Recall@K for a set of retrieved document IDs against relevant document IDs.
+
+    Args:
+        retrieved_ids (Iterable[str]): An iterable of retrieved document IDs.
+        relevant_ids (Set[str]): A set of relevant document IDs.
+        k (int): The cutoff rank K.
+
+    Returns:
+        float: The Recall@K value.
+    """
+    if k <= 0:
+        raise ValueError("k must be a positive integer")
+
+    relevant_ids_set = set(relevant_ids)
+
+    # Limit retrieved IDs to top K
+    if not relevant_ids_set:
+        return 0.0
+
+    # chunk retrieved_ids to top k
+    top_k_retrieved_ids_set = set(list(retrieved_ids)[:k])
+
+    return round(
+        len(relevant_ids_set & top_k_retrieved_ids_set) / len(relevant_ids_set), 4
+    )

app/evaludation/trace_writer.py

@@ -0,0 +1,43 @@
+import json
+import time
+from pathlib import Path
+from typing import Iterable
+
+
+TRACE_DIR = Path(__file__).parent.parent.parent / "var/traces"
+TRACE_DIR.mkdir(parents=True, exist_ok=True)
+TRACE_FILE = TRACE_DIR / "rage_trace.jsonl"
+
+
+def write_trace(
+    query: str,
+    retrieved_ids: Iterable[str],
+    answer_text: str,
+    recall_k: float | None = None,
+    faithfulness: float | None = None,
+):
+    """
+    write trace of each generation to a JSONL file.
+
+    :param query: Description
+    :type query: str
+    :param retrieved_ids: Description
+    :type retrieved_ids: Iterable[str]
+    :param answer_text: Description
+    :type answer_text: str
+    :param recall_k: Description
+    :type recall_k: float | None
+    :param faithfulness: Description
+    :type faithfulness: float | None
+    """
+    record = {
+        "ts": time.time(),
+        "query": query,
+        "retrieved_ids": list(retrieved_ids),
+        "answer": answer_text,
+        "recall_k": recall_k,
+        "faithfulness": faithfulness,
+    }
+
+    with TRACE_FILE.open("a", encoding="utf-8") as f:
+        f.write(json.dumps(record) + "\n")

app/generation/service.py

@@ -5,6 +5,7 @@
 from app.generation.prompt_builder import build_prompt
 
 from app.core.interfaces import BaseGenerator, BaseRetriever
+from app.evaludation.trace_writer import write_trace
 
 
 logger = get_logger(__name__)
@@ -52,6 +53,13 @@ async def generate_answer(
 
     logger.info(f"Generated answer for query='{req.query}'")
 
+    # Write trace for the generation
+    write_trace(
+        query=req.query,
+        retrieved_ids=[chunk["doc_id"] for chunk in retrieved_chunks],
+        answer_text=answer_text,
+    )
+
     return GenerationResponse(
         query=req.query,
         answer=GenerateAnswer(

tests/evaluation/test_failthfulness.py

@@ -0,0 +1,7 @@
+from app.evaludation.faithfulness import faithfulness_overlap
+
+
+def test_failthfulness_basic():
+    answer = "RAG combines retrieval and generation"
+    contexts = ["retrieval augmented generation uses retrieval"]
+    assert faithfulness_overlap(answer, contexts) == 0.4

tests/evaluation/test_recall.py

@@ -0,0 +1,19 @@
+from app.evaludation.recall import recall_at_k
+
+
+def test_recall_at_k_basic():
+    """
+    Test basic functionality of recall_at_k.
+    it should return correct recall value for simple cases.
+    """
+    retrieved_ids = ["doc1", "doc2", "doc3"]
+    relevant_ids = {
+        "doc2",
+    }
+    k = 3
+    # it shall be 1/1 = 1.0
+    # explaination:
+    # relevant_ids has one document "doc2"
+    # retrieved_ids has "doc2" within top 3
+    #
+    assert recall_at_k(retrieved_ids, relevant_ids, k) == 1.0