GEPA holdout

Recipe — held-out validation when running judge optimization, to detect over-fit.

Split your labeled traces 80/20 before optimization. Run optimization against the 80; manually validate the resulting judge against the held-out 20. If validation agreement is meaningfully lower than the optimization's reported optimized_accuracy, you're over-fit — expand the training set or check for label inconsistency.

from layerlens import Stratix
client = Stratix()

# Estimate cost first
estimate = client.judge_optimizations.estimate(judge_id=judge.id, budget="medium")

# Run optimization (uses platform-side training data; budget controls exploration depth)
optimization = client.judge_optimizations.create(judge_id=judge.id, budget="medium")
while True:
 optimization = client.judge_optimizations.get(optimization.id)
 if optimization.status.value in ("success", "failure"):
 break

if optimization.status.value == "success":
 print(f"Baseline: {optimization.baseline_accuracy:.3f}")
 print(f"Optimized: {optimization.optimized_accuracy:.3f}")

 # Apply to create the new judge version
 client.judge_optimizations.apply(optimization.id)

 # Manually validate the new judge version against your held-out trace set
 # (run trace_evaluations on each holdout trace, compare against your labels)
 correct = 0
 for holdout_trace_id, expected_label in holdout_set.items():
 eval_run = client.trace_evaluations.create(trace_id=holdout_trace_id, judge_id=judge.id)
 result = client.trace_evaluations.wait_for_completion(eval_run.id)
 if result.passed == expected_label:
 correct += 1
 holdout_agreement = correct / len(holdout_set)
 print(f"Holdout agreement: {holdout_agreement:.3f}")

 if holdout_agreement < optimization.optimized_accuracy - 0.10:
 print("Possible over-fit; expand training set or revert.")

Good evening

GEPA holdout

See also