Here I want the llm to be used twice, first to generate the text query for web using ocr data and my voice transcription and then second time to present the web results as it isin the original script :: import mss from PIL import Image import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import ollama import keyboard import threading import time # For time.sleep()

Global variables for control
is_listening = False speech_text = ""

Initialize the TTS engine globally
tts_engine = pyttsx3.init() # Move this to the top

def configure_tts_voice(): global tts_engine voices = tts_engine.getProperty('voices')

# List available voices (optional)
for index, voice in enumerate(voices):
    print(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}")

# Change voice (select male/female, adjust the index as per your preference)
tts_engine.setProperty('voice', voices[1].id)  # Change index to select a different voice

# Adjust speaking rate (lower is slower and more human-like)
tts_engine.setProperty('rate', 160)  # Default is 200, lowering makes it more human-like

# Adjust volume (0.0 to 1.0)
tts_engine.setProperty('volume', 1.0)

OCR function (using Tesseract)
def perform_ocr(image): try: print("Performing OCR on the captured screen...") text = pytesseract.image_to_string(image) print(f"OCR complete: {text[:500]}...") # Limiting to first 100 characters for brevity return text except Exception as e: print(f"OCR Error: {e}") return ""

Screen capture function
def capture_screen(): print("Capturing screen...") with mss.mss() as sct: monitor = sct.monitors[0] screenshot = sct.grab(monitor) print("Screen captured successfully.") return Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")

Modified Speech-to-Text function with timeout handling
def transcribe_speech(): global is_listening, speech_text recognizer = sr.Recognizer() with sr.Microphone() as source: print("Press 'F2' to start speaking, 'F3' to stop.") recognizer.adjust_for_ambient_noise(source, duration=1) # Adjust for noise for 1 second while not is_listening: time.sleep(0.1) print("Listening... Speak now.") try: # Listen for speech, but stop after a timeout audio = recognizer.listen(source, timeout=10, phrase_time_limit=15) # Timeout to avoid getting stuck print("Processing the audio with Google Speech Recognition...") text = recognizer.recognize_google(audio) speech_text = text print(f"Speech Transcribed: {text}") except sr.WaitTimeoutError: print("Listening timed out, no speech detected.") except sr.UnknownValueError: print("Speech recognition could not understand audio.") except sr.RequestError as e: print(f"Could not request results from speech recognition service; {e}")

Text-to-Speech function
Text-to-Speech function with pause and stop control
def speak_text(text): global tts_engine print("Speaking out the response...")

# Speak the text
tts_engine.say(text)

# Add hotkeys for pausing (F4) and stopping (F5)
keyboard.add_hotkey('f4', pause_speaking)
keyboard.add_hotkey('f5', stop_speaking)

tts_engine.runAndWait()
print("Text-to-Speech completed.")

# Remove hotkeys after completion to avoid conflicts
keyboard.remove_hotkey('f4')
keyboard.remove_hotkey('f5')

Function to stop the speech (acts as pause since pyttsx3 doesn't have a true pause)
def pause_speaking(): global tts_engine print("Paused speaking...") tts_engine.stop()

Function to stop the speech completely
def stop_speaking(): global tts_engine print("Stopped speaking.") tts_engine.stop()

Asynchronous web search function
async def async_web_search(query): print(f"Searching the web for query: {query}") url = f"https://html.duckduckgo.com/html/?q={query}" async with aiohttp.ClientSession() as session: async with session.get(url) as response: html = await response.text()

soup = BeautifulSoup(html, 'html.parser')
results = []
for result in soup.find_all('div', class_='result__body')[:3]:
    title = result.find('a', class_='result__a').text
    snippet = result.find('a', class_='result__snippet').text
    results.append({'title': title, 'snippet': snippet})

print(f"Web search complete. Found {len(results)} results.")
return results

Function to process with LLM
def process_with_llm(ocr_text, speech_text, search_results): print("Processing data with the LLM (using ollama)...") context = f""" OCR Text: {ocr_text}

Transcribed Speech: {speech_text}

Web Search Results:
{search_results}

Based on the above information, please provide a relevant response.
"""

response = ollama.generate(model='llama3.2', prompt=context)
print("LLM processing complete. Response generated.")
return response['response']

Hotkey handler functions
def start_listening(): global is_listening is_listening = True print("Started listening...") screenshot = capture_screen() ocr_text = perform_ocr(screenshot) print("OCR Text:", ocr_text[:1000]) # Limiting to first 1000 characters for brevity threading.Thread(target=transcribe_speech).start()

def stop_listening(): global is_listening is_listening = False print("Stopped listening.")

Main asynchronous function
async def main(): global speech_text

configure_tts_voice()  # Voice configuration happens before anything else

# Set up hotkeys
keyboard.add_hotkey('f2', start_listening)
keyboard.add_hotkey('f3', stop_listening)

while True:
    # Capture screen and perform OCR
    ##screenshot = capture_screen()
    ##ocr_text = perform_ocr(screenshot)
    ##print("OCR Text:", ocr_text[:100])  # Limiting to first 100 characters for brevity

    # Start speech recognition in a separate thread
    ##print("Starting speech recognition thread...")
    ##speech_thread = threading.Thread(target=transcribe_speech)
    ##speech_thread.start()

    # Wait for speech input to complete
    print("Waiting for speech input to complete...")
    while is_listening or speech_text == "":
        await asyncio.sleep(0.1)

    # If no speech was captured, continue to next iteration
    if not speech_text:
        print("No speech detected. Press 'F2' to try again or 'Esc' to exit.")
        continue

    # Perform web search based on speech
    print("Performing web search based on the speech input...")
    search_results = await async_web_search(speech_text)
    print("Web search results:", search_results)

    # Process with LLM
    print("Processing the input with LLM...")
    llm_response = process_with_llm("", speech_text, search_results)
    print("LLM Response:", llm_response)

    # Speak the response
    print("Speaking the response...")
    speak_text(llm_response)

    # Reset speech_text for next iteration
    speech_text = ""
    print("Process complete. Press 'F2' to start again or 'Esc' to exit.")

    # Check for exit condition
    if keyboard.is_pressed('esc'):
        print("Exiting...")
        break

Run the main function
if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: print("Program terminated by user.") finally: # Clean up keyboard hooks keyboard.unhook_all()