Trace evaluations

client.trace_evaluations — run a judge against a trace, poll, retrieve results, estimate cost.

A trace evaluation runs a single judge against a single trace.

Estimate cost

Always estimate before running large batches:

estimate = client.trace_evaluations.estimate_cost(
 trace_ids=[t.id for t in traces],
 judge_id=judge.id,
)
# CostEstimateResponse fields: trace_count, input_tokens, output_tokens, model

Create

trace_eval = client.trace_evaluations.create(
 trace_id=trace.id,
 judge_id=judge.id,
)
# trace_eval.id and trace_eval.status (TraceEvaluationStatus enum: pending, in_progress, success, failure)

Wait for completion

trace_eval = client.trace_evaluations.wait_for_completion(
 trace_eval.id,
 interval_seconds=3,
 timeout_seconds=300, # None for indefinite
)

Get results

results = client.trace_evaluations.get_results(trace_eval.id)
# TraceEvaluationResult fields:
# - score (typically 0.0–1.0)
# - passed (bool)
# - reasoning (text)
# - steps (list with tool + result entries showing judge reasoning steps)
# - latency_ms
# - prompt_tokens, completion_tokens
# - total_cost

Get by ID

trace_eval = client.trace_evaluations.get(trace_eval_id)

List with filters

response = client.trace_evaluations.get_many(
 page=1, page_size=50,
 judge_id="...",
 trace_id="...",
 outcome="...",
 time_range={"from": "...", "to": "..."},
 search="...",
 sort_by="created_at",
 sort_order="desc",
)

Async / concurrent

import asyncio
from layerlens import AsyncStratix

async def evaluate_many(trace_ids, judge_id):
 client = AsyncStratix()
 creates = [client.trace_evaluations.create(trace_id=tid, judge_id=judge_id) for tid in trace_ids]
 evals = await asyncio.gather(*creates)
 waits = [client.trace_evaluations.wait_for_completion(e.id) for e in evals]
 return await asyncio.gather(*waits)

Good night