# # Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Word Wrangler: A voice-based word guessing game. This demo version is intended to be deployed to Pipecat Cloud. For more information, visit: - Deployment Quickstart: https://docs.pipecat.daily.co/quickstart - Build for Twilio: https://docs.pipecat.daily.co/pipecat-in-production/telephony/twilio-mediastreams """ import asyncio import os import re from typing import Any, Mapping, Optional from dotenv import load_dotenv from loguru import logger from pipecat.audio.resamplers.soxr_resampler import SOXRAudioResampler from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, EndFrame, Frame, InputAudioRawFrame, LLMContextFrame, LLMFullResponseEndFrame, LLMRunFrame, LLMTextFrame, StartFrame, TTSAudioRawFrame, TTSSpeakFrame, ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, LLMUserAggregatorParams, ) from pipecat.processors.consumer_processor import ConsumerProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup from pipecat.processors.producer_processor import ProducerProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.google.gemini_live.llm import ( GeminiLiveLLMService, GeminiModalities, InputParams, ) from pipecat.services.google.tts import GoogleTTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.websocket.fastapi import ( FastAPIWebsocketParams, ) from pipecat.turns.mute.mute_until_first_bot_complete_user_mute_strategy import ( MuteUntilFirstBotCompleteUserMuteStrategy, ) from pipecat.turns.user_stop.turn_analyzer_user_turn_stop_strategy import ( TurnAnalyzerUserTurnStopStrategy, ) from pipecat.turns.user_turn_strategies import UserTurnStrategies from pipecat.utils.sync.base_notifier import BaseNotifier from pipecat.utils.sync.event_notifier import EventNotifier from pipecat.utils.text.base_text_filter import BaseTextFilter from word_list import generate_game_words load_dotenv(override=True) GAME_DURATION_SECONDS = 120 NUM_WORDS_PER_GAME = 20 HOST_VOICE_ID = "en-US-Chirp3-HD-Charon" PLAYER_VOICE_ID = "Kore" # Define conversation modes with their respective prompt templates game_player_prompt = """You are a player for a game of Word Wrangler. GAME RULES: 1. The user will be given a word or phrase that they must describe to you 2. The user CANNOT say any part of the word/phrase directly 3. You must try to guess the word/phrase based on the user's description 4. Once you guess correctly, the user will move on to their next word 5. The user is trying to get through as many words as possible in 60 seconds 6. The external application will handle timing and keeping score YOUR ROLE: 1. Listen carefully to the user's descriptions 2. Make intelligent guesses based on what they say 3. When you think you know the answer, state it clearly: "Is it [your guess]?" 4. If you're struggling, ask for more specific clues 5. Keep the game moving quickly - make guesses promptly 6. Be enthusiastic and encouraging IMPORTANT: - Keep all responses brief - the game is timed! - Make multiple guesses if needed - Use your common knowledge to make educated guesses - If the user indicates you got it right, just say "Got it!" and prepare for the next word - If you've made several wrong guesses, simply ask for "Another clue please?" Start by guessing once you hear the user describe the word or phrase.""" game_host_prompt = """You are the AI host for a game of Word Wrangler. There are two players in the game: the human describer and the AI guesser. GAME RULES: 1. You, the host, will give the human describer a word or phrase that they must describe 2. The describer CANNOT say any part of the word/phrase directly 3. The AI guesser will try to guess the word/phrase based on the describer's description 4. Once the guesser guesses correctly, move on to the next word 5. The describer is trying to get through as many words as possible in 60 seconds 6. The describer can say "skip" or "pass" to get a new word if they find a word too difficult 7. The describer can ask you to repeat the current word if they didn't hear it clearly 8. You'll keep track of the score (1 point for each correct guess) 9. The external application will handle timing YOUR ROLE: 1. Start with this exact brief introduction: "Welcome to Word Wrangler! I'll give you words to describe, and the A.I. player will try to guess them. Remember, don't say any part of the word itself. Here's your first word: [word]." 2. Provide words to the describer. Choose 1 or 2 word phrases that cover a variety of topics, including animals, objects, places, and actions. 3. IMPORTANT: You will hear DIFFERENT types of input: a. DESCRIPTIONS from the human (which you should IGNORE) b. AFFIRMATIONS from the human (like "correct", "that's right", "you got it") which you should IGNORE c. GUESSES from the AI player (which will be in the form of "Is it [word]?" or similar question format) d. SKIP REQUESTS from the human (if they say "skip", "pass", or "next word please") e. REPEAT REQUESTS from the human (if they say "repeat", "what was that?", "say again", etc.) 4. HOW TO RESPOND: - If you hear a DESCRIPTION or AFFIRMATION from the human, respond with exactly "IGNORE" (no other text) - If you hear a GUESS (in question form) and it's INCORRECT, respond with exactly "NO" (no other text) - If you hear a GUESS (in question form) and it's CORRECT, respond with "Correct! That's [N] points. Your next word is [new word]" where N is the current score - If you hear a SKIP REQUEST, respond with "The new word is [new word]" (don't change the score) - If you hear a REPEAT REQUEST, respond with "Your word is [current word]" (don't change the score) 5. SCORING: - Start with a score of 0 - Add 1 point for each correct guess by the AI player - Do NOT add points for skipped words - Announce the current score after every correct guess RESPONSE EXAMPLES: - Human says: "This is something you use to write" → You respond: "IGNORE" - Human says: "That's right!" or "You got it!" → You respond: "IGNORE" - Human says: "Wait, what was my word again?" → You respond: "Your word is [current word]" - Human says: "Can you repeat that?" → You respond: "Your word is [current word]" - AI says: "Is it a pen?" → If correct and it's the first point, you respond: "Correct! That's 1 point. Your next word is [new word]" - AI says: "Is it a pencil?" → If correct and it's the third point, you respond: "Correct! That's 3 points. Your next word is [new word]" - AI says: "Is it a marker?" → If incorrect, you respond: "NO" - Human says: "Skip this one" or "Pass" → You respond: "The new word is [new word]" IMPORTANT GUIDELINES: - Choose words that range from easy to moderately difficult - Keep all responses brief - the game is timed! - Your "NO" and "IGNORE" responses won't be verbalized, but will be visible in the chat - Always keep track of the CURRENT word so you can repeat it when asked - Always keep track of the CURRENT SCORE and announce it after every correct guess - Make sure your word choices are appropriate for all audiences - If the human asks to skip, always provide a new word immediately without changing the score - If the human asks you to repeat the word, say ONLY "Your word is [current word]" - don't add additional text - CRUCIAL: Never interpret the human saying "correct", "that's right", "good job", or similar affirmations as a correct guess. These are just the human giving feedback to the AI player. Start with the exact introduction specified above and give the first word.""" class HostResponseTextFilter(BaseTextFilter): """Custom text filter for Word Wrangler game. This filter removes "NO" and "IGNORE" responses from the host so they don't get verbalized, allowing for silent incorrect guess handling and ignoring descriptions. """ def __init__(self): self._interrupted = False def update_settings(self, settings: Mapping[str, Any]): # No settings to update for this filter pass async def filter(self, text: str) -> str: # Remove case and whitespace for comparison clean_text = text.strip().upper() # If the text is exactly "NO" or "IGNORE", return empty string if clean_text == "NO" or clean_text == "IGNORE": return "" return text async def handle_interruption(self): self._interrupted = True async def reset_interruption(self): self._interrupted = False class BotStoppedSpeakingNotifier(FrameProcessor): """A processor that notifies whenever a BotStoppedSpeakingFrame is detected.""" def __init__(self, notifier: BaseNotifier): super().__init__() self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) # Check if this is a BotStoppedSpeakingFrame if isinstance(frame, BotStoppedSpeakingFrame): logger.debug(f"{self}: Host bot stopped speaking, notifying listeners") await self._notifier.notify() # Always push the frame through await self.push_frame(frame, direction) class StartGate(FrameProcessor): """A gate that blocks the inital context message to prevent the player from responding first. Blocks LLMContextFrame until the gate opens. This frame is dropped (not stored) to ignore the initial message. Once opened, all frames pass through normally. Note that we don't need to block User input frames or speaking frames because the STTMuteFilter will handle that. """ def __init__(self, notifier: BaseNotifier): super().__init__() self._notifier = notifier self._gate_opened = False self._gate_task: Optional[asyncio.Task] = None async def setup(self, setup: FrameProcessorSetup): """Set up the processor with required components. Args: setup: Configuration object containing setup parameters. """ await super().setup(setup) self._gate_task = self.create_task(self._wait_for_notification()) async def cleanup(self): """Clean up the processor resources.""" await super().cleanup() if self._gate_task: await self.cancel_task(self._gate_task) self._gate_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if self._gate_opened: # Once the gate is open, let everything through await self.push_frame(frame, direction) elif isinstance(frame, LLMContextFrame): # Drop these frames until the gate opens - we want to ignore this audio logger.trace(f"{self}: Dropping {type(frame).__name__} until host bot stops speaking") # Start the gate task if not already running else: # Let all other frames through await self.push_frame(frame, direction) async def _wait_for_notification(self): try: # Wait for the notifier await self._notifier.wait() # Gate is now open - only run this code once if not self._gate_opened: self._gate_opened = True logger.debug(f"{self}: Gate opened, all frames will now pass through") except asyncio.CancelledError: logger.debug(f"{self}: Gate task was cancelled") raise except Exception as e: logger.exception(f"{self}: Error in gate task: {e}") raise class GameStateTracker(FrameProcessor): """Tracks game state including new words and score by monitoring host responses. This processor aggregates streamed text from the host LLM to detect: 1. New word announcements (triggering player LLM resets) 2. Score updates (to track the current score) """ def __init__(self, new_word_notifier: BaseNotifier): super().__init__() self._new_word_notifier = new_word_notifier self._text_buffer = "" self._current_score = 0 # Words/phrases that indicate a new word being provided self._key_phrases = ["your word is", "new word is", "next word is"] # Pattern to extract score from responses self._score_pattern = re.compile(r"that's (\d+) point", re.IGNORECASE) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) # Collect text from LLMTextFrames if isinstance(frame, LLMTextFrame): text = frame.text # Skip responses that are "NO" or "IGNORE" if text.strip() in ["NO", "IGNORE"]: logger.debug(f"Skipping NO/IGNORE response") await self.push_frame(frame, direction) return # Add the new text to our buffer self._text_buffer += text # Process complete responses when we get an end frame elif isinstance(frame, LLMFullResponseEndFrame): if self._text_buffer: buffer_lower = self._text_buffer.lower() # 1. Check for new word announcements new_word_detected = False for phrase in self._key_phrases: if phrase in buffer_lower: await self._new_word_notifier.notify() new_word_detected = True break if not new_word_detected: logger.debug(f"No new word phrases detected") # 2. Check for score updates score_match = self._score_pattern.search(buffer_lower) if score_match: try: score = int(score_match.group(1)) # Only update if the new score is higher if score > self._current_score: logger.debug(f"Score updated from {self._current_score} to {score}") self._current_score = score else: logger.debug( f"Ignoring score {score} <= current score {self._current_score}" ) except ValueError as e: logger.warning(f"Error parsing score: {e}") else: logger.debug(f"No score pattern match in: '{buffer_lower}'") # Reset the buffer after processing the complete response self._text_buffer = "" # Always push the frame through await self.push_frame(frame, direction) @property def current_score(self) -> int: """Get the current score.""" return self._current_score class GameTimer: """Manages the game timer and triggers end-game events.""" def __init__( self, task: PipelineTask, game_state_tracker: GameStateTracker, game_duration_seconds: int = 120, ): self._task = task self._game_state_tracker = game_state_tracker self._game_duration = game_duration_seconds self._timer_task = None self._start_time = None def start(self): """Start the game timer.""" if self._timer_task is None: self._start_time = asyncio.get_event_loop().time() self._timer_task = asyncio.create_task(self._run_timer()) logger.info(f"Game timer started: {self._game_duration} seconds") def stop(self): """Stop the game timer.""" if self._timer_task: self._timer_task.cancel() self._timer_task = None logger.info("Game timer stopped") def get_remaining_time(self) -> int: """Get the remaining time in seconds.""" if self._start_time is None: return self._game_duration elapsed = asyncio.get_event_loop().time() - self._start_time remaining = max(0, self._game_duration - int(elapsed)) return remaining async def _run_timer(self): """Run the timer and end the game when time is up.""" try: # Wait for the game duration await asyncio.sleep(self._game_duration) # Game time is up, get the final score final_score = self._game_state_tracker.current_score # Create end game message end_message = f"Time's up! Thank you for playing Word Wrangler. Your final score is {final_score} point" if final_score != 1: end_message += "s" end_message += ". Great job!" # Send end game message as TTSSpeakFrame logger.info(f"Game over! Final score: {final_score}") await self._task.queue_frames([TTSSpeakFrame(text=end_message)]) # End the game await self._task.queue_frames([EndFrame()]) except asyncio.CancelledError: logger.debug("Game timer task cancelled") except Exception as e: logger.exception(f"Error in game timer: {e}") class ResettablePlayerLLM(GeminiLiveLLMService): """A specialized LLM service that can reset its context when notified about a new word. This LLM intelligently waits for the host to finish speaking before reconnecting. """ def __init__( self, api_key: str, system_instruction: str, new_word_notifier: BaseNotifier, host_stopped_speaking_notifier: BaseNotifier, voice_id: str = PLAYER_VOICE_ID, **kwargs, ): super().__init__( api_key=api_key, voice_id=voice_id, system_instruction=system_instruction, **kwargs ) self._new_word_notifier = new_word_notifier self._host_stopped_speaking_notifier = host_stopped_speaking_notifier self._base_system_instruction = system_instruction self._reset_task: Optional[asyncio.Task] = None self._pending_reset: bool = False async def start(self, frame: StartFrame): await super().start(frame) # Start the notifier listener task if not self._reset_task or self._reset_task.done(): self._reset_task = self.create_task(self._listen_for_notifications()) async def stop(self, frame: EndFrame): # Cancel the reset task if it exists if self._reset_task and not self._reset_task.done(): await self.cancel_task(self._reset_task) self._reset_task = None await super().stop(frame) async def cancel(self, frame: CancelFrame): # Cancel the reset task if it exists if self._reset_task and not self._reset_task.done(): await self.cancel_task(self._reset_task) self._reset_task = None await super().cancel(frame) async def _listen_for_notifications(self): """Listen for new word and host stopped speaking notifications.""" try: # Create tasks for both notifiers new_word_task = self.create_task(self._listen_for_new_word()) host_stopped_task = self.create_task(self._listen_for_host_stopped()) # Wait for both tasks to complete (which should never happen) await asyncio.gather(new_word_task, host_stopped_task) except asyncio.CancelledError: logger.debug(f"{self}: Notification listener tasks cancelled") raise except Exception as e: logger.exception(f"{self}: Error in notification listeners: {e}") raise async def _listen_for_new_word(self): """Listen for new word notifications and flag a reset is needed.""" while True: # Wait for a new word notification await self._new_word_notifier.wait() logger.info( f"{self}: Received new word notification, disconnecting and waiting for host to finish" ) # Disconnect immediately to stop processing await self._disconnect() # Reset the system instruction self._system_instruction = self._base_system_instruction # Flag that we need to reconnect when the host stops speaking self._pending_reset = True async def _listen_for_host_stopped(self): """Listen for host stopped speaking and reconnect if a reset is pending.""" while True: # Wait for host stopped speaking notification await self._host_stopped_speaking_notifier.wait() # If we have a pending reset, reconnect now if self._pending_reset: logger.info(f"{self}: Host finished speaking, completing the LLM reset") # Reconnect await self._connect() # Reset the flag self._pending_reset = False logger.info(f"{self}: LLM reset complete") async def tts_audio_raw_frame_filter(frame: Frame): """Filter to check if the frame is a TTSAudioRawFrame.""" return isinstance(frame, TTSAudioRawFrame) # Create a resampler instance once resampler = SOXRAudioResampler() async def tts_to_input_audio_transformer(frame: Frame): """Transform TTS audio frames to InputAudioRawFrame with resampling. Converts 24kHz TTS output to 16kHz input audio required by the player LLM. Args: frame (Frame): The frame to transform (expected to be TTSAudioRawFrame) Returns: InputAudioRawFrame: The transformed and resampled input audio frame """ if isinstance(frame, TTSAudioRawFrame): # Resample the audio from 24kHz to 16kHz resampled_audio = await resampler.resample( frame.audio, frame.sample_rate, # Source rate (24kHz) 16000, # Target rate (16kHz) ) # Create a new InputAudioRawFrame with the resampled audio input_frame = InputAudioRawFrame( audio=resampled_audio, sample_rate=16000, # New sample rate num_channels=frame.num_channels, ) return input_frame async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.debug("Starting WebSocket bot") game_words = generate_game_words(NUM_WORDS_PER_GAME) words_string = ", ".join(f'"{word}"' for word in game_words) logger.debug(f"Game words: {words_string}") player_instruction = f"""{game_player_prompt} Important guidelines: 1. Your responses will be converted to speech, so keep them concise and conversational. 2. Don't use special characters or formatting that wouldn't be natural in speech. 3. Encourage the user to elaborate when appropriate.""" host_instruction = f"""{game_host_prompt} GAME WORDS: Use ONLY these words for the game (in any order): {words_string} Important guidelines: 1. Your responses will be converted to speech, so keep them concise and conversational. 2. Don't use special characters or formatting that wouldn't be natural in speech. 3. ONLY use words from the provided list above when giving words to the player.""" intro_message = """Start with this exact brief introduction: "Welcome to Word Wrangler! I'll give you words to describe, and the A.I. player will try to guess them. Remember, don't say any part of the word itself. Here's your first word: [word]." """ host_llm = GeminiLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=host_instruction, params=InputParams(modalities=GeminiModalities.TEXT), ) host_tts = GoogleTTSService( voice_id=HOST_VOICE_ID, credentials_path=os.getenv("GOOGLE_TEST_CREDENTIALS_FILE"), text_filters=[HostResponseTextFilter()], ) producer = ProducerProcessor( filter=tts_audio_raw_frame_filter, transformer=tts_to_input_audio_transformer, passthrough=True, ) consumer = ConsumerProcessor(producer=producer) # Create the notifiers bot_speaking_notifier = EventNotifier() new_word_notifier = EventNotifier() # Create BotStoppedSpeakingNotifier to detect when host bot stops speaking bot_stopped_speaking_detector = BotStoppedSpeakingNotifier(bot_speaking_notifier) # Create StartGate to block Player LLM until host has stopped speaking start_gate = StartGate(bot_speaking_notifier) # Create GameStateTracker to handle new words and score tracking game_state_tracker = GameStateTracker(new_word_notifier) # Create a resettable player LLM that coordinates between notifiers player_llm = ResettablePlayerLLM( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=player_instruction, new_word_notifier=new_word_notifier, host_stopped_speaking_notifier=bot_speaking_notifier, voice_id=PLAYER_VOICE_ID, ) # Set up the initial context for the conversation messages = [ { "role": "user", "content": intro_message, }, ] # While there are no context aggregators in the Pipeline, we need to create # one here to be able to push the context frame to the PipelineTask. This is # what initiates the conversation. context = LLMContext(messages) user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, user_params=LLMUserAggregatorParams( user_turn_strategies=UserTurnStrategies( stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())] ), user_mute_strategies=[ MuteUntilFirstBotCompleteUserMuteStrategy(), ], ), ) pipeline = Pipeline( [ transport.input(), # Receive audio/video from Daily call user_aggregator, ParallelPipeline( # Host branch: manages the game and provides words [ consumer, # Receives audio from the player branch host_llm, # AI host that provides words and tracks score game_state_tracker, # Tracks words and score from host responses host_tts, # Converts host text to speech bot_stopped_speaking_detector, # Notifies when host stops speaking ], # Player branch: guesses words based on human descriptions [ start_gate, # Gates the player until host finishes intro player_llm, # AI player that makes guesses producer, # Collects audio frames to be passed to the consumer ], ), transport.output(), # Send audio/video back to Daily call assistant_aggregator, ] ) task = PipelineTask( pipeline, params=PipelineParams( audio_out_sample_rate=8000, enable_metrics=True, enable_usage_metrics=True, ), ) # Create the game timer game_timer = GameTimer(task, game_state_tracker, game_duration_seconds=GAME_DURATION_SECONDS) @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") # Kick off the conversation by getting the context frame and pushing it. # There is no aggegrator in the Pipeline, so we need to rely on the # PipelineTask to push the frame. await task.queue_frames([LLMRunFrame()]) # Start the game timer game_timer.start() @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected: {client}") # Stop the timer game_timer.stop() # Cancel the pipeline task await task.cancel() runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) await runner.run(task) async def bot(runner_args: RunnerArguments): """Main bot entry point compatible with Pipecat Cloud.""" if os.environ.get("ENV") != "local": from pipecat.audio.filters.krisp_filter import KrispFilter krisp_filter = KrispFilter() else: krisp_filter = None # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. transport_params = { "webrtc": lambda: TransportParams( audio_in_enabled=True, audio_in_filter=krisp_filter, audio_out_enabled=True, vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), ), "twilio": lambda: FastAPIWebsocketParams( audio_in_enabled=True, audio_in_filter=krisp_filter, audio_out_enabled=True, vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), ), } transport = await create_transport(runner_args, transport_params) await run_bot(transport, runner_args) if __name__ == "__main__": from pipecat.runner.run import main main()