replay production sessions instead of scoring a dataset

# Braintrust keeps logging. Nothing changes in production.
# Export the project logs, then import them once.
kitaru session import braintrust-logs.jsonl \
--importer kitaru/braintrust@latest \
--agent support-agent@latest \
--media-type application/x-ndjson \
--tag imported-baseline --wait

# Freeze the sessions that matter into the test population.
kitaru cohort create refund-regression --agent support-agent \
--tag imported-baseline --display-version week-32

# Move one variable. Tool calls come from the recording, so
# nothing external fires and no card is refunded twice.
kitaru experiment create cheaper-model \
--agent support-agent \
--evaluator refund-check@latest \
--override '{"model": {"openai:gpt-5.4": "openai:gpt-5-nano"}}' \
--tool-policy '{"default": {"type": "history",
                "scope": "cohort_version", "on_miss": "fail"}}'

# Re-execute the whole cohort against the change.
# Exits nonzero on failure, so CI can gate on it.
kitaru experiment run start cheaper-model \
--cohort-version <cohort-version-id> \
--agent support-agent@1 --evaluate-baselines --wait

from braintrust import Eval
from autoevals import Factuality

# The dataset is the population. You write it, or build it from logs.
def load_cases():
  return [
      {"input": "Refund order #4821", "expected": "Refund issued for order #4821"},
      # ... and the rest, by hand
  ]

# task() is your code. Braintrust runs it on every row.
# Tools inside it are live unless you stub them yourself.
def task(input):
  return support_agent(input)

Eval(
  "support-agent",
  data=load_cases,
  task=task,
  scores=[Factuality],
)

# Scores this run and compares it with the last experiment.
# It cannot replay a production conversation with the tool
# results that conversation actually saw.