Add Cartesia Managed Agents framework

.env.example

@@ -139,14 +139,13 @@ EVA_MODEL__TTS_PARAMS='{"api_key": "your_cartesia_api_key", "model": "sonic"}'
 #d json_object
 #x pipeline_mode=S2S
 #v EVA_MODEL__S2S_PARAMS='{"model": "gpt-realtime-mini", "api_key": ""}'
-
-# Smallest Hydra S2S (set EVA_FRAMEWORK=smallest_hydra). Hydra emits no transcript
-# on the wire, so a batch STT transcribes each turn for the audit log; it defaults
-# to Smallest Pulse keyed on the same api_key. Override via the "transcription" block
-# (provider: smallest|openai|deepgram). voice: wren|sloane|marlowe|reed|knox|tate.
-#x pipeline_mode=S2S
-#v EVA_MODEL__S2S=hydra
-#v EVA_MODEL__S2S_PARAMS='{"model": "hydra", "api_key": "", "voice": "wren", "generate_initial_response": true, "transcription": {"provider": "smallest", "model": "pulse-pro", "language": "en"}}'
+# Alternatives (set EVA_FRAMEWORK to match):
+#   Hydra (smallest_hydra): batch STT transcribes turns; override via "transcription" (provider: smallest|openai|deepgram).
+#     EVA_MODEL__S2S=hydra
+#     EVA_MODEL__S2S_PARAMS='{"model": "hydra", "api_key": "", "voice": "wren", "generate_initial_response": true, "transcription": {"provider": "smallest", "model": "pulse-pro", "language": "en"}}'
+#   Cartesia Managed (cartesia): provision agents with scripts/provision_cartesia_agents.py; see docs/experiment_setup.md.
+#     EVA_MODEL__S2S=cartesia
+#     EVA_MODEL__S2S_PARAMS='{"model": "gpt-5.5", "api_key": "", "assistant_agent_id": ""}'
 
 # --- AudioLLM mode ---
 #i Audio-input LLM model name.
@@ -162,7 +161,7 @@ EVA_MODEL__TTS_PARAMS='{"api_key": "your_cartesia_api_key", "model": "sonic"}'
 # --- Framework (S2S / AudioLLM) ---
 #i Base framework for S2S or AudioLLM pipelines.
 #d enum
-#e pipecat,openai_realtime,gemini_live,elevenlabs,grok_voice,smallest_hydra
+#e pipecat,openai_realtime,gemini_live,elevenlabs,grok_voice,smallest_hydra,cartesia
 #v EVA_FRAMEWORK=openai_realtime
 
 # ==============================================

docs/experiment_setup.md

@@ -92,6 +92,32 @@ We use the default turn detection configurations for most framework in our exper
 
 EVA-Bench makes turn detection parameters and options configurable via the CLI, so practitioners can run experiments using the turn detection settings available to their chosen framework. The only exception is ElevenAgents, where users must register and configure their agents separately prior to evaluation.
 
+### Cartesia Managed Agents
+
+`framework: cartesia` runs a Cartesia managed agent over a WebSocket connection. Managed Agents runs the whole agent loop: audio orchestration (listening, turn-taking, interruptions and speech) along with tool calling, reasoning orchestration and knowledge retrieval, around the reasoning model you choose. EVA's tools run as client tools, and EVA's system prompt and greeting are sent with each call as dynamic variables. One agent per domain is registered before evaluation, with its config in `scripts/provision_cartesia_agents.py`. Model fallback is disabled on these agents, so only the chosen model answers.
+
+Some agent settings can't be changed per call, so they are set when the agent is provisioned from EVA's options. The assistant server checks them against the run config and refuses to start if they don't match:
+
+| EVA option | Agent setting |
+|---|---|
+| `EVA_LANGUAGE` | `language.primary` (regional variants such as `fr-CA` use the base code) |
+| `EVA_MODEL__PRE_TOOL_SPEECH` (`auto` or `off`) | each tool's `pre_tool_speech` |
+| `EVA_TURN_END_FALLBACK_TIME` (5 to 60 seconds, or unset) | `turn.inactivity_check_in_secs` (unset turns check-ins off) |
+
+```bash
+# Create or update the agents; --model picks the LLM (default gpt-5.5). Prints each domain's s2s_params.
+python scripts/provision_cartesia_agents.py --pre-tool-speech auto
+
+EVA_FRAMEWORK=cartesia \
+EVA_MODEL__S2S=cartesia \
+EVA_MODEL__S2S_PARAMS='{"model": "gpt-5.5", "api_key": "<cartesia key>", "assistant_agent_id": "<airline agent id>"}' \
+python main.py --domain airline
+```
+
+`--voice` sets the agent's voice; it isn't checked against the run config.
+
+`s2s_params`: `api_key`, `model` and `assistant_agent_id` (the agent for the run's domain) are required, as for ElevenLabs. `model` must match the model configured on the agent; `alias` optionally labels the run. `base_url` (default `wss://api.cartesia.ai`) and `cartesia_version` are optional. `EVA_MODEL__LLM` must be unset, as for any S2S run.
+
 ## Judge Models
 
 The table below lists the API-hosted models used as a judge.

scripts/provision_cartesia_agents.py

@@ -0,0 +1,153 @@
+"""Create or update one Cartesia managed agent per EVA domain, with EVA's tools as client tools.
+
+The agent passes EVA's system prompt and greeting through untouched: they are sent with each call as
+the dynamic variables system_prompt and initial_message. Settings that cannot be changed per call are
+set here from EVA's own options, and the assistant server refuses to run against an agent whose
+settings do not match the run config:
+
+    --language                -> language.primary              (EVA_LANGUAGE, default en; fr-CA uses fr)
+    --pre-tool-speech         -> each tool's pre_tool_speech  (EVA_MODEL__PRE_TOOL_SPEECH, default auto)
+    --turn-end-fallback-time  -> turn.inactivity_check_in_secs (EVA_TURN_END_FALLBACK_TIME, default unset)
+
+--voice sets the agent's voice. It is not checked: like any agent setting that doesn't affect the
+benchmark, it is identified by the run's assistant_agent_id.
+
+Model fallback is disabled, so only the chosen model answers.
+
+Usage:
+    python scripts/provision_cartesia_agents.py [--model gpt-5.5] [--language en] [--voice VOICE_ID]
+        [--pre-tool-speech auto|off] [--turn-end-fallback-time SECONDS] [--domains airline itsm medical_hr]
+
+Reads CARTESIA_API_KEY from the environment or .env. Tool and agent IDs are kept in
+output/cartesia_agents.json so a rerun updates the same agents. Prints each domain's
+EVA_MODEL__S2S_PARAMS (without the api_key) to use for the run.
+"""
+
+import argparse
+import json
+import os
+import sys
+from functools import partial
+from pathlib import Path
+
+from dotenv import dotenv_values
+
+from eva.models.agents import AgentConfig
+from eva.models.config import LANGUAGE_DISPLAY_NAMES, ModelConfig, base_language
+from eva.utils import cartesia_agents
+
+DOMAINS = ("airline", "itsm", "medical_hr")
+DEFAULT_VOICE_ID = "f786b574-daa5-4673-aa0c-cbe3e8534c02"  # Katie
+STATE = Path("output/cartesia_agents.json")
+
+
+def tool_body(fn: dict, pre_tool_speech: str) -> dict:
+    return {
+        "type": "client",
+        "name": fn["name"],
+        "description": fn["description"],
+        "parameters": fn["parameters"],
+        "expects_response": True,
+        "response_timeout_secs": 20,
+        "pre_tool_speech": pre_tool_speech,
+        "execution_mode": "immediate",
+    }
+
+
+def agent_config(
+    model: str, tool_ids: list[str], turn_end_fallback_time: int | None, language: str, voice_id: str
+) -> dict:
+    return {
+        "instructions": cartesia_agents.SYSTEM_PROMPT_VARIABLE,
+        "prompt_optimization": False,  # use EVA's system prompt as is
+        "initial_message": cartesia_agents.INITIAL_MESSAGE_VARIABLE,
+        "speculative_generation": True,
+        "model": {"id": model, "fallback_strategy": None},
+        "language": {"primary": base_language(language)},
+        "turn": {"inactivity_check_in_secs": turn_end_fallback_time},
+        "audio": {
+            "input": {"noise_suppression": "off", "asr_model": "auto"},
+            "output": {"voice_id": voice_id},
+        },
+        "tools": [{"id": t} for t in tool_ids],
+    }
+
+
+def check_model(request, model: str) -> None:
+    path = "/v1/agents/models"
+    while True:
+        page = request("GET", path)
+        if any(m.get("id") == model for m in page["data"]):
+            return
+        if not page.get("has_more"):
+            raise SystemExit(f"{model} is not in GET /v1/agents/models")
+        path = f"/v1/agents/models?starting_after={page['next_page']}"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--model", default="gpt-5.5", help="Any model id from GET /v1/agents/models")
+    parser.add_argument(
+        "--language",
+        choices=[lang.value for lang in LANGUAGE_DISPLAY_NAMES],
+        default="en",
+        help="Must match EVA_LANGUAGE for the run",
+    )
+    parser.add_argument("--voice", default=DEFAULT_VOICE_ID, help="Cartesia voice ID for the agent (default: Katie)")
+    parser.add_argument(
+        "--pre-tool-speech",
+        choices=["auto", "off"],
+        default=ModelConfig.model_fields["pre_tool_speech"].default,
+        help="Must match EVA_MODEL__PRE_TOOL_SPEECH for the run",
+    )
+    parser.add_argument(
+        "--turn-end-fallback-time",
+        type=int,
+        default=None,
+        help="Must match EVA_TURN_END_FALLBACK_TIME for the run (5-60 seconds; unset disables check-ins)",
+    )
+    parser.add_argument("--domains", nargs="+", default=list(DOMAINS), choices=DOMAINS)
+    args = parser.parse_args()
+    if args.turn_end_fallback_time is not None and not 5 <= args.turn_end_fallback_time <= 60:
+        raise SystemExit("Cartesia supports --turn-end-fallback-time between 5 and 60 seconds")
+
+    key = os.environ.get("CARTESIA_API_KEY") or dotenv_values(".env").get("CARTESIA_API_KEY")
+    if not key:
+        raise SystemExit("Set CARTESIA_API_KEY in the environment or .env")
+    request = partial(cartesia_agents.request, api_key=key)
+    check_model(request, args.model)
+
+    state = json.loads(STATE.read_text()) if STATE.exists() else {"tools": {}, "agents": {}}
+    STATE.parent.mkdir(parents=True, exist_ok=True)
+    for domain in args.domains:
+        # Tools keep EVA's order, the order the model sees them in.
+        tool_ids = []
+        for tool in AgentConfig.from_yaml(f"configs/agents/{domain}_agent.yaml").build_tools_for_agent():
+            fn = tool["function"]
+            key = f"{domain}/{fn['name']}"
+            body = tool_body(fn, args.pre_tool_speech)
+            if key in state["tools"]:
+                patch = {k: v for k, v in body.items() if k != "type"}  # a tool's type cannot change
+                request("PATCH", f"/v1/agents/tools/{state['tools'][key]}", body=patch)
+            else:
+                state["tools"][key] = request("POST", "/v1/agents/tools", body=body)["id"]
+                STATE.write_text(json.dumps(state, indent=2))
+            tool_ids.append(state["tools"][key])
+
+        name = f"eva-{domain.replace('_', '-')}-{args.model.replace('.', '').replace('-', '')}"
+        payload = {
+            "name": name,
+            "description": "EVA-Bench agent",
+            "config": agent_config(args.model, tool_ids, args.turn_end_fallback_time, args.language, args.voice),
+        }
+        if domain in state["agents"]:
+            request("PATCH", f"/v1/agents/{state['agents'][domain]}", body=payload)
+        else:
+            state["agents"][domain] = request("POST", "/v1/agents", body=payload)["id"]
+        STATE.write_text(json.dumps(state, indent=2))
+        print(f"{domain}: {state['agents'][domain]} with {len(tool_ids)} tools", file=sys.stderr)
+        print(f"{domain}: " + json.dumps({"model": args.model, "assistant_agent_id": state["agents"][domain]}))
+
+
+if __name__ == "__main__":
+    main()