from abc import ABC, abstractmethod import base64 import json import threading from typing import Callable, Optional, Awaitable, Union, Any, Literal, Dict, Tuple import asyncio from concurrent.futures import ThreadPoolExecutor from enum import Enum from websockets.sync.client import connect, Connection from websockets.exceptions import ConnectionClosedOK from ..base_client import BaseElevenLabs class ClientToOrchestratorEvent(str, Enum): """Event types that can be sent from client to orchestrator.""" # Response to a ping request. PONG = "pong" CLIENT_TOOL_RESULT = "client_tool_result" CONVERSATION_INITIATION_CLIENT_DATA = "conversation_initiation_client_data" FEEDBACK = "feedback" # Non-interrupting content that is sent to the server to update the conversation state. CONTEXTUAL_UPDATE = "contextual_update" # User text message. USER_MESSAGE = "user_message" USER_ACTIVITY = "user_activity" class UserMessageClientToOrchestratorEvent: """Event for sending user text messages.""" def __init__(self, text: Optional[str] = None): self.type: Literal[ClientToOrchestratorEvent.USER_MESSAGE] = ClientToOrchestratorEvent.USER_MESSAGE self.text = text def to_dict(self) -> dict: return { "type": self.type, "text": self.text } class UserActivityClientToOrchestratorEvent: """Event for registering user activity (ping to prevent timeout).""" def __init__(self) -> None: self.type: Literal[ClientToOrchestratorEvent.USER_ACTIVITY] = ClientToOrchestratorEvent.USER_ACTIVITY def to_dict(self) -> dict: return { "type": self.type } class ContextualUpdateClientToOrchestratorEvent: """Event for sending non-interrupting contextual updates to the conversation state.""" def __init__(self, content: str): self.type: Literal[ClientToOrchestratorEvent.CONTEXTUAL_UPDATE] = ClientToOrchestratorEvent.CONTEXTUAL_UPDATE self.content = content def to_dict(self) -> dict: return { "type": self.type, "content": self.content } class AudioInterface(ABC): """AudioInterface provides an abstraction for handling audio input and output.""" @abstractmethod def start(self, input_callback: Callable[[bytes], None]): """Starts the audio interface. Called one time before the conversation starts. The `input_callback` should be called regularly with input audio chunks from the user. The audio should be in 16-bit PCM mono format at 16kHz. Recommended chunk size is 4000 samples (250 milliseconds). """ pass @abstractmethod def stop(self): """Stops the audio interface. Called one time after the conversation ends. Should clean up any resources used by the audio interface and stop any audio streams. Do not call the `input_callback` from `start` after this method is called. """ pass @abstractmethod def output(self, audio: bytes): """Output audio to the user. The `audio` input is in 16-bit PCM mono format at 16kHz. Implementations can choose to do additional buffering. This method should return quickly and not block the calling thread. """ pass @abstractmethod def interrupt(self): """Interruption signal to stop any audio output. User has interrupted the agent and all previosly buffered audio output should be stopped. """ pass class ClientTools: """Handles registration and execution of client-side tools that can be called by the agent. Supports both synchronous and asynchronous tools running in a dedicated event loop, ensuring non-blocking operation of the main conversation thread. """ def __init__(self) -> None: self.tools: Dict[str, Tuple[Union[Callable[[dict], Any], Callable[[dict], Awaitable[Any]]], bool]] = {} self.lock = threading.Lock() self._loop = None self._thread = None self._running = threading.Event() self.thread_pool = ThreadPoolExecutor() def start(self): """Start the event loop in a separate thread for handling async operations.""" if self._running.is_set(): return def run_event_loop(): self._loop = asyncio.new_event_loop() asyncio.set_event_loop(self._loop) self._running.set() try: self._loop.run_forever() finally: self._running.clear() self._loop.close() self._loop = None self._thread = threading.Thread(target=run_event_loop, daemon=True, name="ClientTools-EventLoop") self._thread.start() # Wait for loop to be ready self._running.wait() def stop(self): """Gracefully stop the event loop and clean up resources.""" if self._loop and self._running.is_set(): self._loop.call_soon_threadsafe(self._loop.stop) self._thread.join() self.thread_pool.shutdown(wait=False) def register( self, tool_name: str, handler: Union[Callable[[dict], Any], Callable[[dict], Awaitable[Any]]], is_async: bool = False, ) -> None: """Register a new tool that can be called by the AI agent. Args: tool_name: Unique identifier for the tool handler: Function that implements the tool's logic is_async: Whether the handler is an async function """ with self.lock: if not callable(handler): raise ValueError("Handler must be callable") if tool_name in self.tools: raise ValueError(f"Tool '{tool_name}' is already registered") self.tools[tool_name] = (handler, is_async) async def handle(self, tool_name: str, parameters: dict) -> Any: """Execute a registered tool with the given parameters. Returns the result of the tool execution. """ with self.lock: if tool_name not in self.tools: raise ValueError(f"Tool '{tool_name}' is not registered") handler, is_async = self.tools[tool_name] if is_async: return await handler(parameters) else: return await asyncio.get_event_loop().run_in_executor(self.thread_pool, handler, parameters) def execute_tool(self, tool_name: str, parameters: dict, callback: Callable[[dict], None]): """Execute a tool and send its result via the provided callback. This method is non-blocking and handles both sync and async tools. """ if not self._running.is_set(): raise RuntimeError("ClientTools event loop is not running") if self._loop is None: raise RuntimeError("Event loop is not available") async def _execute_and_callback(): try: result = await self.handle(tool_name, parameters) response = { "type": "client_tool_result", "tool_call_id": parameters.get("tool_call_id"), "result": result or f"Client tool: {tool_name} called successfully.", "is_error": False, } except Exception as e: response = { "type": "client_tool_result", "tool_call_id": parameters.get("tool_call_id"), "result": str(e), "is_error": True, } callback(response) asyncio.run_coroutine_threadsafe(_execute_and_callback(), self._loop) class ConversationInitiationData: """Configuration options for the Conversation.""" def __init__( self, extra_body: Optional[dict] = None, conversation_config_override: Optional[dict] = None, dynamic_variables: Optional[dict] = None, ): self.extra_body = extra_body or {} self.conversation_config_override = conversation_config_override or {} self.dynamic_variables = dynamic_variables or {} class Conversation: client: BaseElevenLabs agent_id: str requires_auth: bool config: ConversationInitiationData audio_interface: AudioInterface client_tools: Optional[ClientTools] callback_agent_response: Optional[Callable[[str], None]] callback_agent_response_correction: Optional[Callable[[str, str], None]] callback_user_transcript: Optional[Callable[[str], None]] callback_latency_measurement: Optional[Callable[[int], None]] _thread: Optional[threading.Thread] _should_stop: threading.Event _conversation_id: Optional[str] _last_interrupt_id: int _ws: Optional[Connection] def __init__( self, client: BaseElevenLabs, agent_id: str, *, requires_auth: bool, audio_interface: AudioInterface, config: Optional[ConversationInitiationData] = None, client_tools: Optional[ClientTools] = None, callback_agent_response: Optional[Callable[[str], None]] = None, callback_agent_response_correction: Optional[Callable[[str, str], None]] = None, callback_user_transcript: Optional[Callable[[str], None]] = None, callback_latency_measurement: Optional[Callable[[int], None]] = None, ): """Conversational AI session. BETA: This API is subject to change without regard to backwards compatibility. Args: client: The ElevenLabs client to use for the conversation. agent_id: The ID of the agent to converse with. requires_auth: Whether the agent requires authentication. audio_interface: The audio interface to use for input and output. client_tools: The client tools to use for the conversation. callback_agent_response: Callback for agent responses. callback_agent_response_correction: Callback for agent response corrections. First argument is the original response (previously given to callback_agent_response), second argument is the corrected response. callback_user_transcript: Callback for user transcripts. callback_latency_measurement: Callback for latency measurements (in milliseconds). """ self.client = client self.agent_id = agent_id self.requires_auth = requires_auth self.audio_interface = audio_interface self.callback_agent_response = callback_agent_response self.config = config or ConversationInitiationData() self.client_tools = client_tools or ClientTools() self.callback_agent_response_correction = callback_agent_response_correction self.callback_user_transcript = callback_user_transcript self.callback_latency_measurement = callback_latency_measurement self.client_tools.start() self._thread = None self._ws: Optional[Connection] = None self._should_stop = threading.Event() self._conversation_id = None self._last_interrupt_id = 0 def start_session(self): """Starts the conversation session. Will run in background thread until `end_session` is called. """ ws_url = self._get_signed_url() if self.requires_auth else self._get_wss_url() self._thread = threading.Thread(target=self._run, args=(ws_url,)) self._thread.start() def end_session(self): """Ends the conversation session and cleans up resources.""" self.audio_interface.stop() self.client_tools.stop() self._ws = None self._should_stop.set() def wait_for_session_end(self) -> Optional[str]: """Waits for the conversation session to end. You must call `end_session` before calling this method, otherwise it will block. Returns the conversation ID, if available. """ if not self._thread: raise RuntimeError("Session not started.") self._thread.join() return self._conversation_id def send_user_message(self, text: str): """Send a text message from the user to the agent. Args: text: The text message to send to the agent. Raises: RuntimeError: If the session is not active or websocket is not connected. """ if not self._ws: raise RuntimeError("Session not started or websocket not connected.") event = UserMessageClientToOrchestratorEvent(text=text) try: self._ws.send(json.dumps(event.to_dict())) except Exception as e: print(f"Error sending user message: {e}") raise def register_user_activity(self): """Register user activity to prevent session timeout. This sends a ping to the orchestrator to reset the timeout timer. Raises: RuntimeError: If the session is not active or websocket is not connected. """ if not self._ws: raise RuntimeError("Session not started or websocket not connected.") event = UserActivityClientToOrchestratorEvent() try: self._ws.send(json.dumps(event.to_dict())) except Exception as e: print(f"Error registering user activity: {e}") raise def send_contextual_update(self, content: str): """Send a contextual update to the conversation. Contextual updates are non-interrupting content that is sent to the server to update the conversation state without directly prompting the agent. Args: content: The contextual information to send to the conversation. Raises: RuntimeError: If the session is not active or websocket is not connected. """ if not self._ws: raise RuntimeError("Session not started or websocket not connected.") event = ContextualUpdateClientToOrchestratorEvent(content=content) try: self._ws.send(json.dumps(event.to_dict())) except Exception as e: print(f"Error sending contextual update: {e}") raise def _run(self, ws_url: str): with connect(ws_url, max_size=16 * 1024 * 1024) as ws: self._ws = ws ws.send( json.dumps( { "type": "conversation_initiation_client_data", "custom_llm_extra_body": self.config.extra_body, "conversation_config_override": self.config.conversation_config_override, "dynamic_variables": self.config.dynamic_variables, } ) ) self._ws = ws def input_callback(audio): try: ws.send( json.dumps( { "user_audio_chunk": base64.b64encode(audio).decode(), } ) ) except ConnectionClosedOK: self.end_session() except Exception as e: print(f"Error sending user audio chunk: {e}") self.end_session() self.audio_interface.start(input_callback) while not self._should_stop.is_set(): try: message = json.loads(ws.recv(timeout=0.5)) if self._should_stop.is_set(): return self._handle_message(message, ws) except ConnectionClosedOK as e: self.end_session() except TimeoutError: pass except Exception as e: print(f"Error receiving message: {e}") self.end_session() self._ws = None def _handle_message(self, message, ws): if message["type"] == "conversation_initiation_metadata": event = message["conversation_initiation_metadata_event"] assert self._conversation_id is None self._conversation_id = event["conversation_id"] elif message["type"] == "audio": event = message["audio_event"] if int(event["event_id"]) <= self._last_interrupt_id: return audio = base64.b64decode(event["audio_base_64"]) self.audio_interface.output(audio) elif message["type"] == "agent_response": if self.callback_agent_response: event = message["agent_response_event"] self.callback_agent_response(event["agent_response"].strip()) elif message["type"] == "agent_response_correction": if self.callback_agent_response_correction: event = message["agent_response_correction_event"] self.callback_agent_response_correction( event["original_agent_response"].strip(), event["corrected_agent_response"].strip() ) elif message["type"] == "user_transcript": if self.callback_user_transcript: event = message["user_transcription_event"] self.callback_user_transcript(event["user_transcript"].strip()) elif message["type"] == "interruption": event = message["interruption_event"] self._last_interrupt_id = int(event["event_id"]) self.audio_interface.interrupt() elif message["type"] == "ping": event = message["ping_event"] ws.send( json.dumps( { "type": "pong", "event_id": event["event_id"], } ) ) if self.callback_latency_measurement and event["ping_ms"]: self.callback_latency_measurement(int(event["ping_ms"])) elif message["type"] == "client_tool_call": tool_call = message.get("client_tool_call", {}) tool_name = tool_call.get("tool_name") parameters = {"tool_call_id": tool_call["tool_call_id"], **tool_call.get("parameters", {})} def send_response(response): if not self._should_stop.is_set(): ws.send(json.dumps(response)) self.client_tools.execute_tool(tool_name, parameters, send_response) else: pass # Ignore all other message types. def _get_wss_url(self): base_ws_url = self.client._client_wrapper.get_environment().wss return f"{base_ws_url}/v1/convai/conversation?agent_id={self.agent_id}" def _get_signed_url(self): response = self.client.conversational_ai.conversations.get_signed_url(agent_id=self.agent_id) return response.signed_url