cookbook.eval

cookbook.eval

Evaluation glue.

Wraps RAGAS and DeepEval into a single evaluate() call so every notebook ends the same way: pipeline in, metrics dict out. Recipes that need finer control (recipe 37 for RAGAS, recipe 38 for DeepEval-in-CI) call the underlying libraries directly.

EvalSample dataclass

One row in an evaluation set.

Source code in cookbook/eval.py

@dataclass
class EvalSample:
    """One row in an evaluation set."""

    question: str
    expected_answer: str
    contexts: list[str]
    actual_answer: str

evaluate(samples, *, use_ragas=True, use_deepeval=False)

Run the requested metric suites and merge their results.

Source code in cookbook/eval.py

def evaluate(
    samples: Sequence[EvalSample],
    *,
    use_ragas: bool = True,
    use_deepeval: bool = False,
) -> dict[str, float]:
    """Run the requested metric suites and merge their results."""
    out: dict[str, float] = {}
    if use_ragas:
        out.update(_run_ragas(samples))
    if use_deepeval:
        out.update(_run_deepeval(samples))
    return out

run_qa_against(pipeline, questions)

Run a pipeline(question) -> (answer, contexts) over an eval set.

Source code in cookbook/eval.py

def run_qa_against(
    pipeline: Callable[[str], tuple[str, list[str]]],
    questions: Iterable[dict],
) -> list[EvalSample]:
    """Run a `pipeline(question) -> (answer, contexts)` over an eval set."""
    out: list[EvalSample] = []
    for row in questions:
        answer, contexts = pipeline(row["question"])
        out.append(
            EvalSample(
                question=row["question"],
                expected_answer=row.get("answer", ""),
                contexts=contexts,
                actual_answer=answer,
            )
        )
    return out