Models and benchmarks

client.models and client.benchmarks — manage models, benchmarks, custom registrations, smart benchmarks.

client.models

# List/filter
models = client.models.get(
 type="public", # public / custom
 name="gpt-4o",
 key="openai/gpt-4o",
 categories=["transformer"],
 companies=["OpenAI"],
 regions=["usa"],
 licenses=["mit"],
)
# models -> Optional[List[Model]]; each Model has.id,.name,.key,.description

# Direct lookups
model = client.models.get_by_id("model_abc")
model = client.models.get_by_key("openai/gpt-4o") # case-sensitive

# Project membership
client.models.add(model_id_1, model_id_2) # variadic
client.models.remove(model_id_1)

# Custom model — OpenAI-compatible endpoint
response = client.models.create_custom(
 name="My Fine-tuned Model",
 key="my-org/custom-model-v1", # lowercase alphanumeric with dots/hyphens/slashes
 description="Fine-tuned GPT for medical Q&A",
 api_url="https://my-api.example.com/v1",
 max_tokens=4096,
 api_key=os.environ.get("MY_PROVIDER_API_KEY"), # optional
)
print(f"Created model: {response.model_id}")

client.benchmarks

# List/filter
benchmarks = client.benchmarks.get(
 type="public",
 name="mmlu",
 key="mmlu",
 categories=["reasoning", "knowledge", "coding"],
 languages=["english"],
)
# Each Benchmark has.id,.key,.name

# Direct lookups
benchmark = client.benchmarks.get_by_id("bench_abc")
benchmark = client.benchmarks.get_by_key("mmlu")

# Project membership
client.benchmarks.add(*benchmark_ids)
client.benchmarks.remove(*benchmark_ids)

# Custom benchmark from JSONL
response = client.benchmarks.create_custom(
 name="My Benchmark",
 description="Domain-specific eval",
 file_path="./benchmark.jsonl", # JSONL: {"input": "...", "truth": "...", "subset": "optional"}
 additional_metrics=["readability", "toxicity", "hallucination"],
 custom_scorer_ids=[scorer_id_1],
 input_type="text",
)

# Smart benchmark — auto-generated from documents (async)
response = client.benchmarks.create_smart(
 name="Manual Q&A",
 description="Auto-generated from policy docs",
 system_prompt="Answer based on the policy document",
 file_paths=["./policy_v1.pdf", "./policy_v2.pdf"],
 metrics=["readability"],
)

JSONL format for custom benchmarks

{"input": "What is 2+2?", "truth": "4"}
{"input": "What is 2+2?", "truth": "4", "subset": "arithmetic"}

subset is optional, used for grouping results.

Async

from layerlens import AsyncStratix
client = AsyncStratix()
models = await client.models.get(type="public")
custom = await client.models.create_custom(...)

See also

• Public client — read-only public catalog
• Concept: Models and benchmarks
• BYOK custom models
• SDK examples: models-and-benchmarks