> For the complete documentation index, see [llms.txt](https://docs.layerlens.ai/llms.txt). Markdown versions of documentation pages are available by appending `.md` to page URLs; this page is available as [Markdown](https://docs.layerlens.ai/more-in-this-section-9/creating-evaluations.md).

# Creating Evaluations

Create evaluations — sync, async, parallel, comparison, error handling.

Examples for creating evaluations on the Stratix platform using the LayerLens Python SDK.

> Before running the below examples ensure the model and benchmark being run are present on your organization.

## Basic Evaluation

### Using Synchronous Client

> Source: `samples/core/benchmark_evaluation.py` in the [`stratix-python`](https://github.com/layerlens/stratix-python) repo.

```python
from layerlens import Stratix

# Construct sync client (API key from env or inline)
client = Stratix()

# --- Models
models = client.models.get()
print(f"Found {len(models)} models")

# --- Benchmarks
benchmarks = client.benchmarks.get()
print(f"Found {len(benchmarks)} benchmarks")

# --- Create evaluation
evaluation = client.evaluations.create(
 model=models[0],
 benchmark=benchmarks[0],
)
print(f"Created evaluation {evaluation.id}, status={evaluation.status}")

# --- Wait for completion
evaluation = client.evaluations.wait_for_completion(
 evaluation,
 interval_seconds=10,
 timeout_seconds=600, # 10 minutes
)
print(f"Evaluation {evaluation.id} finished with status={evaluation.status}")

# --- Results
if evaluation.is_success:
 results = client.results.get(evaluation=evaluation)
 print("Results:", results)
else:
 print("Evaluation did not succeed, no results to show.")
```

### Minimal Sync Example

```python
from layerlens import Stratix

client = Stratix()

models = client.models.get(type="public", name="gpt-4o")
model = models[0]

benchmarks = client.benchmarks.get(type="public", name="simpleQA")
benchmark = benchmarks[0]

evaluation = client.evaluations.create(
 model=model,
 benchmark=benchmark,
)
```

### Using Async Client

```python
import asyncio

from layerlens import AsyncStratix

async def main():
 client = AsyncStratix()

 models = await client.models.get()
 print(f"Found {len(models)} models")

 benchmarks = await client.benchmarks.get()
 print(f"Found {len(benchmarks)} benchmarks")

 evaluation = await client.evaluations.create(model=models[0], benchmark=benchmarks[0])
 print(f"Created evaluation {evaluation.id}, status={evaluation.status}")

 await evaluation.wait_for_completion_async(interval_seconds=10, timeout_seconds=600)
 print(f"Evaluation {evaluation.id} finished with status={evaluation.status}")

 if evaluation.is_success:
 results = await evaluation.get_results_async()
 print("Results:", results)
 else:
 print("Evaluation did not succeed, no results to show.")

if __name__ == "__main__":
 asyncio.run(main())
```

## Sorting and Filtering Evaluations

```python
import asyncio

from layerlens import AsyncStratix
from layerlens.models import EvaluationStatus

async def main():
 client = AsyncStratix()

 # --- Sort by accuracy (highest first)
 response = await client.evaluations.get_many(
 sort_by="accuracy",
 order="desc",
 page_size=10,
 )
 if response:
 print(f"Top {len(response.evaluations)} evaluations by accuracy:")
 for evaluation in response.evaluations:
 print(f" - {evaluation.id}: accuracy={evaluation.accuracy:.2f}%")

 # --- Filter by status (only successful)
 response = await client.evaluations.get_many(
 status=EvaluationStatus.SUCCESS,
 sort_by="accuracy",
 order="desc",
 )
 if response:
 print(f"Successful evaluations: {response.pagination.total_count}")

 # --- Filter by model or benchmark IDs
 response = await client.evaluations.get_many(
 model_ids=["your-model-id"],
 sort_by="accuracy",
 order="desc",
 )

 # --- Combine sorting, filtering, and pagination
 response = await client.evaluations.get_many(
 status=EvaluationStatus.SUCCESS,
 sort_by="accuracy",
 order="desc",
 page=1,
 page_size=20,
 )
 if response:
 print(f"Page 1: {response.pagination.total_count} total, {response.pagination.total_pages} pages")

if __name__ == "__main__":
 asyncio.run(main())
```

## Comparing Evaluations

```python
from layerlens import PublicClient

client = PublicClient()

# Compare two models on a benchmark
comparison = client.comparisons.compare_models(
 benchmark_id="682bddc1e014f9fa440f8a91",
 model_id_1="699f9761e014f9c3072b0513",
 model_id_2="699f9761e014f9c3072b0512",
 page=1,
 page_size=10,
)

if comparison:
 print(f"Model 1: {comparison.correct_count_1}/{comparison.total_results_1} correct")
 print(f"Model 2: {comparison.correct_count_2}/{comparison.total_results_2} correct")

# Filter: where model 1 fails but model 2 succeeds
comparison = client.comparisons.compare_models(
 benchmark_id="682bddc1e014f9fa440f8a91",
 model_id_1="699f9761e014f9c3072b0513",
 model_id_2="699f9761e014f9c3072b0512",
 outcome_filter="reference_fails",
)

# Or compare using evaluation IDs directly
comparison = client.comparisons.compare(
 evaluation_id_1="699f9938a03d70bf6607081f",
 evaluation_id_2="699f991ca782d00ebd666ba1",
)
```

## Running Multiple Evaluations in Parallel

```python
import asyncio

from layerlens import AsyncStratix

async def create_and_run_evaluation(client, model, benchmark, eval_number):
 try:
 evaluation = await client.evaluations.create(model=model, benchmark=benchmark)

 evaluation = await client.evaluations.wait_for_completion(
 evaluation,
 interval_seconds=10,
 timeout_seconds=600,
 )

 if evaluation.is_success:
 results = await client.results.get_all(evaluation=evaluation)
 print(f"Evaluation #{eval_number} completed with {len(results)} results")
 return eval_number, evaluation.id, len(results), True
 else:
 return eval_number, evaluation.id, 0, False

 except Exception as e:
 print(f"Error in evaluation #{eval_number}: {e}")
 return eval_number, None, 0, False

async def main():
 client = AsyncStratix()

 models = await client.models.get()
 benchmarks = await client.benchmarks.get()

 num_evaluations = 3
 tasks = [
 create_and_run_evaluation(client, models[0], benchmarks[0], i + 1)
 for i in range(num_evaluations)
 ]

 results = await asyncio.gather(*tasks, return_exceptions=True)

if __name__ == "__main__":
 asyncio.run(main())
```

## Fetching Results

See [Retrieving Results](/more-in-this-section-9/retrieving-results.md) for paginated and concurrent result fetching patterns.

## Error Handling

```python
from layerlens import Stratix
import layerlens

client = Stratix()

try:
 models = client.models.get()
 benchmarks = client.benchmarks.get()

 evaluation = client.evaluations.create(
 model=models[0],
 benchmark=benchmarks[0],
 )

except layerlens.AuthenticationError:
 print("Check your API key")
except layerlens.NotFoundError:
 print("Model or benchmark not found")
except layerlens.APIError as e:
 print(f"API error: {e}")
```

## Related Resources

* [Evaluations resource reference](/more-in-this-section-9/evaluations-1.md)
* [Results resource reference](/more-in-this-section-9/results.md)
* [Public client reference](/more-in-this-section-9/public-client.md)