How to Test if RAG Is Retrieving the Right Context

"""A thin retrieval wrapper: query string in, ranked chunk IDs out.

This is the entire surface the retrieval test suite talks to, and it is
deliberately the smallest thing that can be tested. There is no prompt
template here, no generator call, no output parsing, no LLM client anywhere in
the import graph. Retrieval is an information-retrieval problem, so it gets
tested as one: deterministic input, deterministic ranked output, no model in
the loop and no API key required.

The default index is a self-contained in-memory cosine-similarity index over
the help-center corpus in `src/corpus.py`, so this repo runs on a clean
checkout with nothing but numpy and pytest installed. Everything about your
real store lives behind one method (`_search`) at the adapter seam below.
"""

from __future__ import annotations

from dataclasses import dataclass
from typing import Iterable, Sequence

import numpy as np

from src.corpus import HELP_CENTER_CHUNKS, Chunk, chunk_text
from src.embedding import EMBEDDING_MODEL_VERSION, HashingEmbedder

# The retrieval window the labeled test set is scored against.
DEFAULT_TOP_K = 5


@dataclass(frozen=True)
class RetrieverConfig:
    """Test-visible retriever configuration.

    `embedding_model_version` is the important field. The article's point is
    that an embedding model bump between index time and query time silently
    reshuffles every neighborhood in the index, and nothing about the failure
    looks like a failure: no exception, no error log, just worse answers. So
    the version is an explicit, pinnable value that the regression suite
    asserts against the version recorded in the labeled fixture. Bump the model
    without re-measuring your baseline and CI tells you, instead of a customer.
    """

    embedding_model_version: str = EMBEDDING_MODEL_VERSION
    top_k: int = DEFAULT_TOP_K


class Retriever:
    """Ranked chunk IDs for a query. That is the whole contract."""

    def __init__(
        self,
        chunks: Iterable[Chunk] | None = None,
        config: RetrieverConfig | None = None,
    ) -> None:
        self.config = config or RetrieverConfig()
        self._chunks: list[Chunk] = list(chunks if chunks is not None else HELP_CENTER_CHUNKS)
        if not self._chunks:
            raise ValueError("Retriever needs a non-empty corpus")

        documents = [chunk_text(chunk) for chunk in self._chunks]
        self._embedder = HashingEmbedder(documents, version=self.config.embedding_model_version)
        self._chunk_ids: list[str] = [chunk["chunk_id"] for chunk in self._chunks]
        # Rows are already L2-normalized, so a dot product is cosine similarity.
        self._matrix: np.ndarray = self._embedder.embed_all(documents)

    @property
    def embedding_model_version(self) -> str:
        """The pinned embedding model version backing this index."""
        return self._embedder.version

    def retrieve(self, query: str, k: int | None = None) -> list[str]:
        """Return up to `k` chunk IDs, most similar first.

        Chunk IDs only. Not documents, not scores, not a prompt, not an answer.
        Everything the metrics in `src/metrics/` need is the ranked ID list,
        and keeping the return type this narrow is what makes retrieval
        testable in isolation from generation.
        """
        window = self.config.top_k if k is None else k
        if window <= 0 or not query or not query.strip():
            return []
        return self._search(query, window)

    # ------------------------------------------------------------------
    # ADAPTER SEAM
    # ------------------------------------------------------------------
    # Everything above this line is store-agnostic and is what your test suite
    # should depend on. Everything below is the one method you replace to point
    # at a real vector store. Keep the signature and the tests do not change:
    #
    #   Pinecone:
    #       matches = index.query(vector=embed(query), top_k=k)["matches"]
    #       return [match["id"] for match in matches]
    #
    #   Weaviate:
    #       response = collection.query.near_vector(embed(query), limit=k)
    #       return [obj.properties["chunk_id"] for obj in response.objects]
    #
    #   Qdrant:
    #       hits = client.search(collection_name="chunks",
    #                            query_vector=embed(query), limit=k)
    #       return [hit.payload["chunk_id"] for hit in hits]
    #
    #   pgvector:
    #       cursor.execute("SELECT chunk_id FROM chunks "
    #                      "ORDER BY embedding <=> %s LIMIT %s", (embed(query), k))
    #       return [row[0] for row in cursor.fetchall()]
    #
    #   FAISS:
    #       _, indices = index.search(embed(query).reshape(1, -1), k)
    #       return [self._chunk_ids[i] for i in indices[0] if i != -1]
    #
    # When you swap this out, pin the real embedding model version in
    # RetrieverConfig and in tests/fixtures/labeled_queries.json so the suite
    # keeps failing loudly on an upstream model upgrade.
    # ------------------------------------------------------------------
    def _search(self, query: str, k: int) -> list[str]:
        """In-memory cosine similarity over the corpus. Replace with your store."""
        query_vector = self._embedder.embed(query)
        if not query_vector.any():
            return []
        scores = self._matrix @ query_vector
        # Sort by descending score, breaking ties on chunk ID so the ranking is
        # byte-stable across runs and platforms. An unstable tiebreak is a
        # flaky retrieval test waiting to happen.
        order = sorted(
            range(len(self._chunk_ids)),
            key=lambda index: (-float(scores[index]), self._chunk_ids[index]),
        )
        return [self._chunk_ids[index] for index in order[:k] if scores[index] > 0.0]

    def titles_for(self, chunk_ids: Sequence[str]) -> list[str]:
        """Titles for a list of chunk IDs. Used only to print readable reports."""
        lookup = {chunk["chunk_id"]: chunk["title"] for chunk in self._chunks}
        return [lookup.get(chunk_id, "<unknown chunk>") for chunk_id in chunk_ids]

"""Hit rate, also known as recall@k, plus its own tests.

The bug it catches: the retriever never surfaces the right chunk at all. If the
needed chunk is not in the top-k, no amount of prompt engineering downstream can
save the answer, because the information was never in the context window.

Hit rate is binary per query: the expected chunk is either somewhere in the
top-k or it is not. Averaged across a labeled set, that is your headline
retrieval number, and it is also where most teams stop, which is the mistake
this repo is built around. A chunk sitting at rank 9 counts as a full hit at
k=10 while being functionally invisible to a generator whose context window
only fits the top 3. See `src/metrics/mrr.py` for the metric that catches that.

Run this file on its own:

    pytest src/metrics/hit_rate.py -v
"""

from __future__ import annotations

from typing import Collection, Sequence

import pytest

from src.metrics.common import dedupe, require_expected, require_parallel


def hit_rate(
    retrieved_ids: Sequence[str],
    expected_ids: Collection[str],
    k: int | None = None,
) -> float:
    """1.0 if any expected chunk appears in the top-k retrieved IDs, else 0.0.

    `k=None` scores the full retrieved list as given.
    """
    expected = require_expected(expected_ids)
    window = dedupe(retrieved_ids)[: k if k is not None else None]
    return 1.0 if expected.intersection(window) else 0.0


def recall_at_k(
    retrieved_ids: Sequence[str],
    expected_ids: Collection[str],
    k: int,
) -> float:
    """Recall@k: the IR-literature name for the same metric as `hit_rate`.

    Both are kept because both names show up in RAG evaluation write-ups and
    people reasonably assume they measure different things. They do not: this
    delegates. The graded question ("how many of the chunks I needed did I
    actually get?") is a different metric, and it lives in
    `src/metrics/context_precision_recall.py` as `context_recall`.
    """
    return hit_rate(retrieved_ids, expected_ids, k=k)


def mean_hit_rate(
    retrieved_ids: Sequence[Sequence[str]],
    expected_ids: Sequence[Collection[str]],
    k: int | None = None,
) -> float:
    """Hit rate averaged over a labeled set.

    `retrieved_ids[i]` is the ranked list for query i and `expected_ids[i]` is
    that query's set of known-correct chunk IDs.
    """
    require_parallel(retrieved_ids, expected_ids)
    scores = [
        hit_rate(retrieved, expected, k=k)
        for retrieved, expected in zip(retrieved_ids, expected_ids)
    ]
    return sum(scores) / len(scores)


# ----------------------------------------------------------------------
# Tests
# ----------------------------------------------------------------------

RANKED_TEN = [f"chunk-{position:02d}" for position in range(1, 11)]


def test_expected_chunk_at_rank_one_is_a_hit():
    assert hit_rate(RANKED_TEN, {"chunk-01"}, k=5) == 1.0


def test_chunk_ranked_low_still_counts_as_a_hit():
    """The edge case the whole article hinges on.

    The same retrieval is a pass at k=10 and a miss at k=5, and the chunk never
    moved. Hit rate cannot tell you the difference between rank 1 and rank 9,
    which is why hit rate alone is not enough to gate a build on.
    """
    assert hit_rate(RANKED_TEN, {"chunk-09"}, k=10) == 1.0
    assert hit_rate(RANKED_TEN, {"chunk-09"}, k=5) == 0.0


def test_missing_chunk_is_not_a_hit():
    assert hit_rate(RANKED_TEN, {"chunk-99"}, k=10) == 0.0


def test_any_one_expected_chunk_is_enough():
    """Hit rate is satisfied by a single relevant chunk, however many are needed.

    Two of the three needed chunks are absent and hit rate still reports a
    perfect score. That gap is what `context_recall` measures.
    """
    expected = {"chunk-02", "chunk-77", "chunk-88"}
    assert hit_rate(RANKED_TEN, expected, k=5) == 1.0


def test_recall_at_k_is_hit_rate_under_another_name():
    for k in (1, 3, 5, 10):
        assert recall_at_k(RANKED_TEN, {"chunk-04"}, k) == hit_rate(RANKED_TEN, {"chunk-04"}, k=k)


def test_empty_retrieval_is_a_miss_not_an_error():
    assert hit_rate([], {"chunk-01"}, k=5) == 0.0


def test_duplicate_ids_are_collapsed_before_the_window_is_applied():
    """A store echoing the same chunk four times has retrieved one chunk.

    Six rows come back but only three distinct chunks, so at k=3 the window is
    chunk-07, chunk-01, chunk-02 and the expected chunk is a hit. Counting rows
    instead of chunks would have called this a miss and sent someone off to
    debug a retriever that was working.
    """
    noisy = ["chunk-07"] * 4 + ["chunk-01", "chunk-02"]
    assert hit_rate(noisy, {"chunk-02"}, k=3) == 1.0
    assert hit_rate(noisy, {"chunk-01"}, k=2) == 1.0
    assert hit_rate(noisy, {"chunk-02"}, k=2) == 0.0


def test_empty_expected_set_raises():
    with pytest.raises(ValueError):
        hit_rate(RANKED_TEN, set())


def test_mean_hit_rate_matches_the_worked_example():
    """Five of six labeled queries surface their chunk inside the top 10."""
    ranks = [1, 2, 1, 9, 2, None]
    retrieved = [RANKED_TEN for _ in ranks]
    expected = [
        {f"chunk-{rank:02d}"} if rank is not None else {"chunk-99"} for rank in ranks
    ]
    assert mean_hit_rate(retrieved, expected, k=10) == pytest.approx(5 / 6)
    assert round(mean_hit_rate(retrieved, expected, k=10), 2) == 0.83
    # The same six queries at k=5: the rank-9 query drops out and the headline
    # number falls, without a single chunk changing position.
    assert mean_hit_rate(retrieved, expected, k=5) == pytest.approx(4 / 6)


def test_mean_hit_rate_rejects_misaligned_inputs():
    with pytest.raises(ValueError):
        mean_hit_rate([RANKED_TEN], [{"chunk-01"}, {"chunk-02"}])

"""Mean Reciprocal Rank, plus its own tests.

The bug it catches: the right chunk is in the results but buried. Hit rate calls
rank 1 and rank 9 the same pass. MRR does not. It averages one over the rank of
the first relevant result, so rank 1 scores a full point and rank 9 scores
barely a tenth of one, which is roughly how much use a generator whose context
window fits three chunks will get out of it.

MRR predates RAG by decades: it was standardized in the TREC question-answering
track, which is the strongest available argument that retrieval quality is an
old, solved-enough measurement problem rather than something new that arrived
with LLMs.

Run this file on its own:

    pytest src/metrics/mrr.py -v
"""

from __future__ import annotations

from typing import Collection, Sequence

import pytest

from src.metrics.common import dedupe, require_expected, require_parallel
from src.metrics.hit_rate import mean_hit_rate


def reciprocal_rank(
    retrieved_ids: Sequence[str],
    expected_ids: Collection[str],
    k: int | None = None,
) -> float:
    """1 / rank of the first relevant chunk, or 0.0 if none is in the top-k.

    Ranks are 1-based, so a relevant chunk at the top of the list scores 1.0. A
    miss scores 0.0 rather than being dropped from the average: a query whose
    answer is nowhere in the results is a real, countable failure, and skipping
    it would let a retriever improve its MRR by returning nothing at all.
    """
    expected = require_expected(expected_ids)
    window = dedupe(retrieved_ids)[: k if k is not None else None]
    for position, chunk_id in enumerate(window, start=1):
        if chunk_id in expected:
            return 1.0 / position
    return 0.0


def mean_reciprocal_rank(
    retrieved_ids: Sequence[Sequence[str]],
    expected_ids: Sequence[Collection[str]],
    k: int | None = None,
) -> float:
    """MRR over a labeled set.

    `retrieved_ids[i]` is the ranked chunk-ID list for query i, and
    `expected_ids[i]` is that query's set of known-correct chunk IDs.
    """
    require_parallel(retrieved_ids, expected_ids)
    scores = [
        reciprocal_rank(retrieved, expected, k=k)
        for retrieved, expected in zip(retrieved_ids, expected_ids)
    ]
    return sum(scores) / len(scores)


# ----------------------------------------------------------------------
# Tests
# ----------------------------------------------------------------------

RANKED_TEN = [f"chunk-{position:02d}" for position in range(1, 11)]


def _labeled_set_with_ranks(ranks: Sequence[int | None]):
    """Build parallel (retrieved, expected) sequences placing the relevant chunk
    at a given rank per query. `None` means the relevant chunk is not in the
    list at all."""
    retrieved = [RANKED_TEN for _ in ranks]
    expected = [{f"chunk-{rank:02d}"} if rank is not None else {"chunk-99"} for rank in ranks]
    return retrieved, expected


def test_rank_one_earns_full_credit():
    assert reciprocal_rank(RANKED_TEN, {"chunk-01"}) == 1.0


def test_rank_two_earns_half_credit():
    assert reciprocal_rank(RANKED_TEN, {"chunk-02"}) == pytest.approx(0.5)


def test_rank_nine_earns_barely_a_tenth():
    """The failure MRR exists to surface, and hit rate cannot see."""
    assert reciprocal_rank(RANKED_TEN, {"chunk-09"}) == pytest.approx(1 / 9)
    assert reciprocal_rank(RANKED_TEN, {"chunk-09"}) < 0.12


def test_a_miss_scores_zero():
    assert reciprocal_rank(RANKED_TEN, {"chunk-99"}) == 0.0


def test_the_highest_ranked_relevant_chunk_wins():
    """Reciprocal rank looks at the first relevant hit, not the best case."""
    assert reciprocal_rank(RANKED_TEN, {"chunk-03", "chunk-07"}) == pytest.approx(1 / 3)


def test_k_cutoff_turns_a_low_rank_into_a_miss():
    assert reciprocal_rank(RANKED_TEN, {"chunk-09"}, k=5) == 0.0


def test_empty_retrieval_scores_zero():
    assert reciprocal_rank([], {"chunk-01"}) == 0.0


def test_duplicates_do_not_shift_ranks():
    """Rank is a position among distinct chunks, not among rows returned."""
    noisy = ["chunk-05", "chunk-05", "chunk-05", "chunk-02"]
    assert reciprocal_rank(noisy, {"chunk-02"}) == pytest.approx(0.5)


def test_empty_expected_set_raises():
    with pytest.raises(ValueError):
        reciprocal_rank(RANKED_TEN, [])


def test_mean_reciprocal_rank_rejects_misaligned_inputs():
    with pytest.raises(ValueError):
        mean_reciprocal_rank([RANKED_TEN], [{"chunk-01"}, {"chunk-02"}])


def test_article_worked_example_six_queries():
    """The exact scenario in the article's labeled-set diagram.

    Six labeled queries, whose expected chunk landed at ranks 1, 2, 1, 9 and 2,
    with the sixth a miss outside the top ten. Hit rate @10 reads 5/6 = 0.83 and
    calls four of those five equally fine. MRR reads 0.52, because the rank-9
    query (the exact-identifier lookup) and the miss drag it down. Two numbers,
    same six queries, and only one of them tells you the retriever has a
    ranking problem.
    """
    ranks = [1, 2, 1, 9, 2, None]
    retrieved, expected = _labeled_set_with_ranks(ranks)

    mrr = mean_reciprocal_rank(retrieved, expected, k=10)
    expected_mrr = (1 + 0.5 + 1 + 1 / 9 + 0.5 + 0) / 6

    assert mrr == pytest.approx(expected_mrr)
    assert round(mrr, 2) == 0.52
    assert round(mean_hit_rate(retrieved, expected, k=10), 2) == 0.83
    assert mrr < mean_hit_rate(retrieved, expected, k=10)

"""Context precision and context recall, plus their own tests.

Two metrics, two distinct bugs, and conflating them is how teams end up applying
the wrong fix:

`context_precision` is the share of the chunks you retrieved that were actually
relevant. The bug it catches is noisy over-retrieval. Pull ten chunks where two
are on topic and you have handed the generator eight chunks of noise, which is
exactly the condition under which a model starts blending unrelated details into
an answer that reads confident and is quietly wrong. Low precision with healthy
recall means tighten k or add a reranker.

`context_recall` is the share of the chunks you needed that you actually got.
The bug it catches is the partial answer. A retriever that finds one of the three
paragraphs required for a complete answer produces a response that is
incomplete and does not sound incomplete, and neither hit rate nor MRR flags it,
because technically a relevant chunk did show up. Low recall with healthy
precision usually means chunk boundaries are splitting one answer into pieces
you are only grabbing one of.

Both are set metrics over whatever list you hand them, which matters: precision
should be scored over the chunks you actually paste into the prompt, while
recall should be scored over the full retrieval window. See
`tests/test_retrieval_regression.py` for how those two windows differ.

Run this file on its own:

    pytest src/metrics/context_precision_recall.py -v
"""

from __future__ import annotations

from typing import Collection, Sequence

import pytest

from src.metrics.common import dedupe, require_expected, require_parallel


def context_precision(
    retrieved_ids: Sequence[str],
    expected_ids: Collection[str],
) -> float:
    """Share of retrieved chunks that are relevant. Catches noisy context.

    Truncate `retrieved_ids` to the context window you actually send to the
    generator before calling this. Precision over a list you never used tells
    you nothing about what diluted the prompt.
    """
    expected = require_expected(expected_ids)
    retrieved = dedupe(retrieved_ids)
    if not retrieved:
        return 0.0
    relevant = sum(1 for chunk_id in retrieved if chunk_id in expected)
    return relevant / len(retrieved)


def context_recall(
    retrieved_ids: Sequence[str],
    expected_ids: Collection[str],
) -> float:
    """Share of needed chunks that were retrieved. Catches partial answers."""
    expected = require_expected(expected_ids)
    retrieved = set(dedupe(retrieved_ids))
    return len(expected.intersection(retrieved)) / len(expected)


def mean_context_precision(
    retrieved_ids: Sequence[Sequence[str]],
    expected_ids: Sequence[Collection[str]],
) -> float:
    """Context precision averaged over a labeled set."""
    require_parallel(retrieved_ids, expected_ids)
    scores = [
        context_precision(retrieved, expected)
        for retrieved, expected in zip(retrieved_ids, expected_ids)
    ]
    return sum(scores) / len(scores)


def mean_context_recall(
    retrieved_ids: Sequence[Sequence[str]],
    expected_ids: Sequence[Collection[str]],
) -> float:
    """Context recall averaged over a labeled set."""
    require_parallel(retrieved_ids, expected_ids)
    scores = [
        context_recall(retrieved, expected)
        for retrieved, expected in zip(retrieved_ids, expected_ids)
    ]
    return sum(scores) / len(scores)


# ----------------------------------------------------------------------
# Tests
# ----------------------------------------------------------------------


def test_all_retrieved_chunks_relevant_is_perfect_precision():
    retrieved = ["refund-policy-window", "refund-processing-time"]
    expected = {"refund-policy-window", "refund-processing-time"}
    assert context_precision(retrieved, expected) == 1.0


def test_noise_dilutes_precision():
    """Retrieve ten, only two on topic: the article's dilution example."""
    retrieved = [f"chunk-{position:02d}" for position in range(1, 11)]
    expected = {"chunk-03", "chunk-07"}
    assert context_precision(retrieved, expected) == pytest.approx(0.2)
    # The same retrieval trimmed to the two relevant chunks scores perfectly.
    assert context_precision(["chunk-03", "chunk-07"], expected) == 1.0


def test_partial_coverage_shows_up_in_recall():
    """One of the three paragraphs an answer needs: a confident half-answer."""
    retrieved = ["cancel-subscription-steps", "shipping-rates", "api-rate-limits"]
    expected = {
        "cancel-subscription-steps",
        "cancel-subscription-proration",
        "cancel-subscription-data-retention",
    }
    assert context_recall(retrieved, expected) == pytest.approx(1 / 3)


def test_full_coverage_is_perfect_recall():
    expected = {"csv-export-orders", "csv-export-limits", "csv-export-scheduling"}
    retrieved = ["csv-export-limits", "csv-export-orders", "csv-export-scheduling"]
    assert context_recall(retrieved, expected) == 1.0


def test_precision_and_recall_diverge_and_that_is_the_point():
    """Two failures that need opposite fixes, and only these metrics tell them apart."""
    expected = {"invoice-lookup-by-number", "invoice-download-pdf"}

    # Too much: both needed chunks plus six chunks of noise. Recall is perfect,
    # precision is poor. Fix by tightening k or reranking, not by re-indexing.
    over_retrieved = list(expected) + [f"noise-{position}" for position in range(6)]
    assert context_recall(over_retrieved, expected) == 1.0
    assert context_precision(over_retrieved, expected) == pytest.approx(0.25)

    # Too little: one needed chunk and nothing else. Precision is perfect,
    # recall is halved. Fix the chunking, not the ranking.
    under_retrieved = ["invoice-lookup-by-number"]
    assert context_precision(under_retrieved, expected) == 1.0
    assert context_recall(under_retrieved, expected) == pytest.approx(0.5)


def test_recall_ignores_rank_entirely():
    """Both metrics are set metrics: reordering cannot change either one."""
    expected = {"api-error-codes", "api-rate-limits"}
    forward = ["api-error-codes", "api-rate-limits", "api-key-rotation"]
    backward = list(reversed(forward))
    assert context_recall(forward, expected) == context_recall(backward, expected)
    assert context_precision(forward, expected) == context_precision(backward, expected)


def test_empty_retrieval_scores_zero_on_both():
    expected = {"team-invite-members"}
    assert context_precision([], expected) == 0.0
    assert context_recall([], expected) == 0.0


def test_duplicate_chunks_do_not_inflate_the_denominator():
    expected = {"shipping-tracking"}
    noisy = ["shipping-tracking", "shipping-tracking", "shipping-rates"]
    assert context_precision(noisy, expected) == pytest.approx(0.5)


def test_empty_expected_set_raises():
    with pytest.raises(ValueError):
        context_precision(["chunk-01"], set())
    with pytest.raises(ValueError):
        context_recall(["chunk-01"], set())


def test_means_over_a_labeled_set():
    retrieved = [["a", "b"], ["a", "x"]]
    expected = [{"a", "b"}, {"a", "b"}]
    assert mean_context_precision(retrieved, expected) == pytest.approx(0.75)
    assert mean_context_recall(retrieved, expected) == pytest.approx(0.75)


def test_means_reject_misaligned_inputs():
    with pytest.raises(ValueError):
        mean_context_precision([["a"]], [{"a"}, {"b"}])
    with pytest.raises(ValueError):
        mean_context_recall([["a"]], [{"a"}, {"b"}])

  "embedding_model_version": "local-hashed-tfidf-v1",
  "queries": [
    {
      "id": "q01",
      "query": "how do refunds work",
      "expected_chunk_ids": ["refund-policy-window", "refund-processing-time", "refund-eligibility-exclusions"],
      "tags": ["refunds"],
      "source": "production-log"
    },
    {
      "id": "q02",
      "query": "what is your refund policy",
      "expected_chunk_ids": ["refund-policy-window", "refund-processing-time", "refund-eligibility-exclusions"],
      "tags": ["refunds"],
      "source": "production-log"
    },

# ----------------------------------------------------------------------
# The gates
# ----------------------------------------------------------------------


def test_hit_rate_meets_threshold(run):
    """Is the needed chunk in the top-k at all? Catches the retriever missing
    the document outright."""
    assert run.hit_rate >= MIN_HIT_RATE, (
        f"hit rate@{RETRIEVAL_K} regressed to {run.hit_rate:.3f}, "
        f"floor is {MIN_HIT_RATE}\n\n{run.report()}"
    )


def test_mean_reciprocal_rank_meets_threshold(run):
    """How high did it rank? Catches a chunk technically retrieved and
    practically unused, buried below the generator's context window."""
    assert run.mrr >= MIN_MRR, (
        f"MRR regressed to {run.mrr:.3f}, floor is {MIN_MRR}\n\n{run.report()}"
    )


def test_context_precision_meets_threshold(run):
    """How much of what we sent was relevant? Catches noisy over-retrieval
    diluting the prompt."""
    assert run.context_precision >= MIN_CONTEXT_PRECISION, (
        f"context precision@{PROMPT_CONTEXT_K} regressed to "
        f"{run.context_precision:.3f}, floor is {MIN_CONTEXT_PRECISION}\n\n{run.report()}"
    )


def test_context_recall_meets_threshold(run):
    """Did we retrieve everything the answer needed? Catches partial,
    confidently incomplete answers."""
    assert run.context_recall >= MIN_CONTEXT_RECALL, (
        f"context recall@{RETRIEVAL_K} regressed to {run.context_recall:.3f}, "
        f"floor is {MIN_CONTEXT_RECALL}\n\n{run.report()}"
    )

name: retrieval regression

# Gate every pull request on retrieval quality. The suite is deterministic and
# needs no API key, no vector database, and no network, so it runs in seconds on
# a stock runner and belongs on every PR rather than on a nightly cron.
on:
  pull_request:
  push:
    branches: [main]

jobs:
  retrieval-metrics:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          # Pinned, not "3.x". A silent interpreter bump is one more way for a
          # baseline to stop meaning what it meant when it was measured.
          python-version: "3.12"
          cache: pip

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Run the retrieval regression suite
        # Fails the build when hit rate, MRR, context precision, or context
        # recall drops below the thresholds in tests/test_retrieval_regression.py.
        # The failure output carries a per-query rank report, so a red build tells
        # you which queries stopped ranking instead of just which number moved.
        run: pytest -v