Creating Evaluations

Create evaluations — sync, async, parallel, comparison, error handling.

Examples for creating evaluations on the Stratix platform using the LayerLens Python SDK.

Before running the below examples ensure the model and benchmark being run are present on your organization.

Basic Evaluation

Using Synchronous Client

Source: samples/core/benchmark_evaluation.py in the stratix-python repo.

from layerlens import Stratix

# Construct sync client (API key from env or inline)
client = Stratix()

# --- Models
models = client.models.get()
print(f"Found {len(models)} models")

# --- Benchmarks
benchmarks = client.benchmarks.get()
print(f"Found {len(benchmarks)} benchmarks")

# --- Create evaluation
evaluation = client.evaluations.create(
 model=models[0],
 benchmark=benchmarks[0],
)
print(f"Created evaluation {evaluation.id}, status={evaluation.status}")

# --- Wait for completion
evaluation = client.evaluations.wait_for_completion(
 evaluation,
 interval_seconds=10,
 timeout_seconds=600, # 10 minutes
)
print(f"Evaluation {evaluation.id} finished with status={evaluation.status}")

# --- Results
if evaluation.is_success:
 results = client.results.get(evaluation=evaluation)
 print("Results:", results)
else:
 print("Evaluation did not succeed, no results to show.")

Minimal Sync Example

from layerlens import Stratix

client = Stratix()

models = client.models.get(type="public", name="gpt-4o")
model = models[0]

benchmarks = client.benchmarks.get(type="public", name="simpleQA")
benchmark = benchmarks[0]

evaluation = client.evaluations.create(
 model=model,
 benchmark=benchmark,
)

Using Async Client

import asyncio

from layerlens import AsyncStratix

async def main():
 client = AsyncStratix()

 models = await client.models.get()
 print(f"Found {len(models)} models")

 benchmarks = await client.benchmarks.get()
 print(f"Found {len(benchmarks)} benchmarks")

 evaluation = await client.evaluations.create(model=models[0], benchmark=benchmarks[0])
 print(f"Created evaluation {evaluation.id}, status={evaluation.status}")

 await evaluation.wait_for_completion_async(interval_seconds=10, timeout_seconds=600)
 print(f"Evaluation {evaluation.id} finished with status={evaluation.status}")

 if evaluation.is_success:
 results = await evaluation.get_results_async()
 print("Results:", results)
 else:
 print("Evaluation did not succeed, no results to show.")

if __name__ == "__main__":
 asyncio.run(main())

Sorting and Filtering Evaluations

import asyncio

from layerlens import AsyncStratix
from layerlens.models import EvaluationStatus

async def main():
 client = AsyncStratix()

 # --- Sort by accuracy (highest first)
 response = await client.evaluations.get_many(
 sort_by="accuracy",
 order="desc",
 page_size=10,
 )
 if response:
 print(f"Top {len(response.evaluations)} evaluations by accuracy:")
 for evaluation in response.evaluations:
 print(f" - {evaluation.id}: accuracy={evaluation.accuracy:.2f}%")

 # --- Filter by status (only successful)
 response = await client.evaluations.get_many(
 status=EvaluationStatus.SUCCESS,
 sort_by="accuracy",
 order="desc",
 )
 if response:
 print(f"Successful evaluations: {response.pagination.total_count}")

 # --- Filter by model or benchmark IDs
 response = await client.evaluations.get_many(
 model_ids=["your-model-id"],
 sort_by="accuracy",
 order="desc",
 )

 # --- Combine sorting, filtering, and pagination
 response = await client.evaluations.get_many(
 status=EvaluationStatus.SUCCESS,
 sort_by="accuracy",
 order="desc",
 page=1,
 page_size=20,
 )
 if response:
 print(f"Page 1: {response.pagination.total_count} total, {response.pagination.total_pages} pages")

if __name__ == "__main__":
 asyncio.run(main())

Comparing Evaluations

from layerlens import PublicClient

client = PublicClient()

# Compare two models on a benchmark
comparison = client.comparisons.compare_models(
 benchmark_id="682bddc1e014f9fa440f8a91",
 model_id_1="699f9761e014f9c3072b0513",
 model_id_2="699f9761e014f9c3072b0512",
 page=1,
 page_size=10,
)

if comparison:
 print(f"Model 1: {comparison.correct_count_1}/{comparison.total_results_1} correct")
 print(f"Model 2: {comparison.correct_count_2}/{comparison.total_results_2} correct")

# Filter: where model 1 fails but model 2 succeeds
comparison = client.comparisons.compare_models(
 benchmark_id="682bddc1e014f9fa440f8a91",
 model_id_1="699f9761e014f9c3072b0513",
 model_id_2="699f9761e014f9c3072b0512",
 outcome_filter="reference_fails",
)

# Or compare using evaluation IDs directly
comparison = client.comparisons.compare(
 evaluation_id_1="699f9938a03d70bf6607081f",
 evaluation_id_2="699f991ca782d00ebd666ba1",
)

Running Multiple Evaluations in Parallel

import asyncio

from layerlens import AsyncStratix

async def create_and_run_evaluation(client, model, benchmark, eval_number):
 try:
 evaluation = await client.evaluations.create(model=model, benchmark=benchmark)

 evaluation = await client.evaluations.wait_for_completion(
 evaluation,
 interval_seconds=10,
 timeout_seconds=600,
 )

 if evaluation.is_success:
 results = await client.results.get_all(evaluation=evaluation)
 print(f"Evaluation #{eval_number} completed with {len(results)} results")
 return eval_number, evaluation.id, len(results), True
 else:
 return eval_number, evaluation.id, 0, False

 except Exception as e:
 print(f"Error in evaluation #{eval_number}: {e}")
 return eval_number, None, 0, False

async def main():
 client = AsyncStratix()

 models = await client.models.get()
 benchmarks = await client.benchmarks.get()

 num_evaluations = 3
 tasks = [
 create_and_run_evaluation(client, models[0], benchmarks[0], i + 1)
 for i in range(num_evaluations)
 ]

 results = await asyncio.gather(*tasks, return_exceptions=True)

if __name__ == "__main__":
 asyncio.run(main())

Fetching Results

See Retrieving Results for paginated and concurrent result fetching patterns.

Error Handling

from layerlens import Stratix
import layerlens

client = Stratix()

try:
 models = client.models.get()
 benchmarks = client.benchmarks.get()

 evaluation = client.evaluations.create(
 model=models[0],
 benchmark=benchmarks[0],
 )

except layerlens.AuthenticationError:
 print("Check your API key")
except layerlens.NotFoundError:
 print("Model or benchmark not found")
except layerlens.APIError as e:
 print(f"API error: {e}")

Related Resources

• Evaluations resource reference
• Results resource reference
• Public client reference