Spaces:
Sleeping
Sleeping
from flask import Flask, render_template_string, request, jsonify | |
import speech_recognition as sr | |
from tempfile import NamedTemporaryFile | |
import os | |
import ffmpeg | |
from fuzzywuzzy import process, fuzz | |
from metaphone import doublemetaphone | |
import logging | |
app = Flask(__name__) | |
logging.basicConfig(level=logging.INFO) | |
# Global variables | |
cart = [] # Stores items as [item_name, price, quantity] in the cart | |
menu_preferences = None # Tracks the current menu preference | |
section_preferences = None # Tracks the current section preference | |
default_menu_preferences = "all" # To reset menu preferences | |
default_sections = { | |
"biryanis": ["veg biryani", "paneer biryani", "chicken biryani", "mutton biryani"], | |
"starters": ["samosa", "onion pakoda", "chilli gobi", "chicken manchurian", "veg manchurian"], | |
"curries": ["paneer butter", "chicken curry", "fish curry", "chilli chicken"], | |
"desserts": ["gulab jamun", "ice cream"], | |
"soft drinks": ["cola", "lemon soda"] | |
} | |
prices = { | |
"samosa": 9, | |
"onion pakoda": 10, | |
"chilli gobi": 12, | |
"chicken biryani": 14, | |
"mutton biryani": 16, | |
"veg biryani": 12, | |
"paneer butter": 10, | |
"fish curry": 12, | |
"chicken manchurian": 14, | |
"veg manchurian": 12, | |
"chilli chicken": 14, | |
"paneer biryani": 13, | |
"chicken curry": 14, | |
"gulab jamun": 8, | |
"ice cream": 6, | |
"cola": 5, | |
"lemon soda": 6 | |
} | |
menus = { | |
"all": list(prices.keys()), | |
"vegetarian": [ | |
"samosa", "onion pakoda", "chilli gobi", "veg biryani", "paneer butter", "veg manchurian", "paneer biryani", "gulab jamun", "ice cream", "cola", "lemon soda" | |
], | |
"non-vegetarian": [ | |
"chicken biryani", "mutton biryani", "fish curry", "chicken manchurian", "chilli chicken", "chicken curry", "gulab jamun", "ice cream", "cola", "lemon soda" | |
] | |
} | |
def index(): | |
return render_template_string(html_code) | |
def reset_cart(): | |
global cart, menu_preferences, section_preferences | |
cart = [] | |
menu_preferences = None | |
section_preferences = None | |
return "Cart reset successfully." | |
def process_audio(): | |
try: | |
# Handle audio input | |
audio_file = request.files.get("audio") | |
if not audio_file: | |
return jsonify({"response": "No audio file provided."}), 400 | |
# Save audio file and convert to WAV format | |
temp_file = NamedTemporaryFile(delete=False, suffix=".webm") | |
audio_file.save(temp_file.name) | |
converted_file = NamedTemporaryFile(delete=False, suffix=".wav") | |
ffmpeg.input(temp_file.name).output( | |
converted_file.name, acodec="pcm_s16le", ac=1, ar="16000" | |
).run(overwrite_output=True) | |
# Recognize speech | |
recognizer = sr.Recognizer() | |
recognizer.dynamic_energy_threshold = True | |
recognizer.energy_threshold = 60 | |
with sr.AudioFile(converted_file.name) as source: | |
recognizer.adjust_for_ambient_noise(source, duration=1) | |
audio_data = recognizer.record(source) | |
raw_command = recognizer.recognize_google(audio_data).lower() | |
logging.info(f"Raw recognized command: {raw_command}") | |
# Preprocess the command | |
command = preprocess_command(raw_command) | |
# Process the command and get a response | |
response = process_command(command) | |
except sr.UnknownValueError: | |
response = "Sorry, I couldn't understand. Please try again." | |
except Exception as e: | |
response = f"An error occurred: {str(e)}" | |
finally: | |
os.unlink(temp_file.name) | |
os.unlink(converted_file.name) | |
return jsonify({"response": response}) | |
def preprocess_command(command): | |
""" | |
Normalize the user command to improve matching. | |
""" | |
command = command.strip().lower() | |
return command | |
def process_command(command): | |
global cart, menu_preferences, section_preferences | |
# Log command and current preferences for debugging | |
logging.info(f"Command: {command}, Menu Preferences: {menu_preferences}, Section Preferences: {section_preferences}") | |
# Handle menu preferences | |
if menu_preferences is None: | |
preferences = ["non-vegetarian", "vegetarian", "all"] | |
# Check for exact matches first | |
if command in preferences: | |
menu_preferences = command | |
else: | |
# Perform fuzzy matching as a fallback | |
closest_match = process.extractOne(command, preferences, scorer=fuzz.partial_ratio) | |
if closest_match and closest_match[1] > 75: # Stricter threshold | |
menu_preferences = closest_match[0] | |
if menu_preferences == "non-vegetarian": | |
return "You have chosen the Non-Vegetarian menu. Which section would you like? (biryanis, starters, curries, desserts, soft drinks)" | |
elif menu_preferences == "vegetarian": | |
return "You have chosen the Vegetarian menu. Which section would you like? (biryanis, starters, curries, desserts, soft drinks)" | |
elif menu_preferences == "all": | |
return "You have chosen the complete menu. Which section would you like? (biryanis, starters, curries, desserts, soft drinks)" | |
return "Please specify your preference: Non-Vegetarian, Vegetarian, or All." | |
# Handle section preferences | |
# Handle section preferences | |
if section_preferences is None or any(section in command for section in default_sections.keys()): | |
sections = list(default_sections.keys()) # ["biryanis", "starters", "curries", "desserts", "soft drinks"] | |
# Check if the user wants to change the section | |
for section in sections: | |
if section in command: | |
section_preferences = section | |
break | |
else: | |
# Use fuzzy matching as a fallback | |
closest_match = process.extractOne(command, sections, scorer=fuzz.partial_ratio) | |
if closest_match and closest_match[1] > 75: # Stricter threshold | |
section_preferences = closest_match[0] | |
if section_preferences: | |
# Provide available items in the selected section based on menu preference | |
available_items = [item for item in default_sections[section_preferences] if item in menus[menu_preferences]] | |
return f"Here are the items in {section_preferences}: {', '.join(available_items)}. Say the item name to add to the cart." | |
return "Please specify a section: biryanis, starters, curries, desserts, or soft drinks." | |
# Handle item addition | |
# Handle item addition | |
available_items = [item for item in default_sections[section_preferences] if item in menus[menu_preferences]] | |
if available_items: | |
# Use exact match or fuzzy matching to find the item | |
for item in available_items: | |
if item in command: | |
quantity = extract_quantity(command) | |
if quantity is not None: | |
for cart_item in cart: | |
if cart_item[0] == item: | |
cart_item[2] += quantity | |
break | |
else: | |
cart.append([item, prices[item], quantity]) | |
cart_summary = ", ".join([f"{i[0]} x{i[2]} (${i[1] * i[2]})" for i in cart]) | |
return f"Added {quantity} x {item} to your cart. Current cart: {cart_summary}. Would you like to choose another section or finalize your order?" | |
return "Please specify a quantity between 1 and 10. Say 'Add 2' or 'Add three' followed by the item name." | |
# Fuzzy matching for items | |
closest_match = process.extractOne(command, available_items, scorer=fuzz.partial_ratio) | |
if closest_match and closest_match[1] > 75: # Stricter threshold for matching | |
matched_item = closest_match[0] | |
quantity = extract_quantity(command) | |
if quantity is not None: | |
for cart_item in cart: | |
if cart_item[0] == matched_item: | |
cart_item[2] += quantity | |
break | |
else: | |
cart.append([matched_item, prices[matched_item], quantity]) | |
cart_summary = ", ".join([f"{i[0]} x{i[2]} (${i[1] * i[2]})" for i in cart]) | |
return f"Added {quantity} x {matched_item} to your cart. Current cart: {cart_summary}. Would you like to choose another section or finalize your order?" | |
return f"Matched {matched_item}. Please specify a quantity between 1 and 10." | |
# Handle remove command | |
# Handle remove command | |
if "remove" in command: | |
# Exact match: Check if the item exists in the cart | |
for item in cart: | |
if item[0] in command: | |
cart.remove(item) | |
cart_summary = ", ".join([f"{i[0]} x{i[2]} (${i[1] * i[2]})" for i in cart]) if cart else "Your cart is now empty." | |
return f"Removed {item[0]} from your cart. Current cart: {cart_summary}." | |
# Fuzzy match: Try to find the closest match in the cart | |
closest_match = process.extractOne(command, [i[0] for i in cart], scorer=fuzz.partial_ratio) | |
if closest_match and closest_match[1] > 75: # Stricter threshold | |
matched_item = closest_match[0] | |
cart = [item for item in cart if item[0] != matched_item] | |
cart_summary = ", ".join([f"{i[0]} x{i[2]} (${i[1] * i[2]})" for i in cart]) if cart else "Your cart is now empty." | |
return f"Removed {matched_item} from your cart. Current cart: {cart_summary}." | |
return "The item you are trying to remove is not in your cart." | |
# Handle final order | |
if "final order" in command: | |
if cart: | |
total = sum(i[1] * i[2] for i in cart) | |
order_summary = ", ".join([f"{i[0]} x{i[2]} (${i[1] * i[2]})" for i in cart]) | |
cart.clear() | |
menu_preferences = None | |
section_preferences = None | |
return f"Your final order is: {order_summary}. Total bill: ${total}. Thank you for ordering! You can start a new order by specifying your preference." | |
return "Your cart is empty. Please add items to your cart first." | |
return "Sorry, I couldn't understand that. Please try again." | |
def extract_quantity(command): | |
""" | |
Extract quantity from the user command. | |
""" | |
# Map numeric words to digits | |
number_words = { | |
"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, | |
"six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, | |
"1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8, "9": 9, "10": 10 | |
} | |
# Normalize command and split into words | |
command_words = command.split() | |
# Check for numeric words or digits in the command | |
for word in command_words: | |
if word in number_words: | |
return number_words[word] | |
return None | |
html_code = """ | |
<!DOCTYPE html> | |
<html lang="en"> | |
<head> | |
<meta charset="UTF-8"> | |
<meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
<title>AI Dining Assistant</title> | |
<style> | |
body { | |
display: flex; | |
flex-direction: column; | |
align-items: center; | |
justify-content: center; | |
min-height: 100vh; | |
margin: 0; | |
font-family: Arial, sans-serif; | |
background-color: #f4f4f9; | |
} | |
h1 { | |
color: #333; | |
} | |
.mic-button { | |
font-size: 2rem; | |
padding: 1rem 2rem; | |
color: white; | |
background-color: #007bff; | |
border: none; | |
border-radius: 50px; | |
cursor: pointer; | |
transition: background-color 0.3s; | |
} | |
.mic-button:hover { | |
background-color: #0056b3; | |
} | |
.status, .response { | |
margin-top: 1rem; | |
text-align: center; | |
color: #555; | |
font-size: 1.2rem; | |
} | |
.response { | |
background-color: #e8e8ff; | |
padding: 1rem; | |
border-radius: 10px; | |
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); | |
display: none; | |
} | |
</style> | |
</head> | |
<body> | |
<h1>AI Dining Assistant</h1> | |
<button class="mic-button" id="mic-button">🎤</button> | |
<div class="status" id="status">Press the mic button to start...</div> | |
<div class="response" id="response">Response will appear here...</div> | |
<script> | |
const micButton = document.getElementById('mic-button'); | |
const status = document.getElementById('status'); | |
const response = document.getElementById('response'); | |
let mediaRecorder; | |
let audioChunks = []; | |
let isConversationActive = false; | |
micButton.addEventListener('click', () => { | |
if (!isConversationActive) { | |
isConversationActive = true; | |
startConversation(); | |
} | |
}); | |
function startConversation() { | |
const utterance = new SpeechSynthesisUtterance('Please choose your preference: All, Vegetarian, or Non-Vegetarian.'); | |
speechSynthesis.speak(utterance); | |
utterance.onend = () => { | |
status.textContent = 'Listening...'; | |
startListening(); | |
}; | |
} | |
function startListening() { | |
navigator.mediaDevices.getUserMedia({ audio: true }).then(stream => { | |
mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' }); | |
mediaRecorder.start(); | |
audioChunks = []; | |
mediaRecorder.ondataavailable = event => audioChunks.push(event.data); | |
mediaRecorder.onstop = async () => { | |
const audioBlob = new Blob(audioChunks, { type: 'audio/webm' }); | |
const formData = new FormData(); | |
formData.append('audio', audioBlob); | |
status.textContent = 'Processing...'; | |
try { | |
const result = await fetch('/process-audio', { method: 'POST', body: formData }); | |
const data = await result.json(); | |
response.textContent = data.response; | |
response.style.display = 'block'; | |
const utterance = new SpeechSynthesisUtterance(data.response); | |
speechSynthesis.speak(utterance); | |
utterance.onend = () => { | |
console.log("Speech synthesis completed."); | |
if (data.response.includes("final order") || data.response.includes("Thank you for ordering")) { | |
status.textContent = 'Order completed. Press the mic button to start again.'; | |
isConversationActive = false; | |
} else { | |
status.textContent = 'Listening...'; | |
setTimeout(() => { | |
startListening(); | |
}, 100); | |
} | |
}; | |
utterance.onerror = (e) => { | |
console.error("Speech synthesis error:", e.error); | |
status.textContent = 'Error with speech output.'; | |
isConversationActive = false; | |
}; | |
} catch (error) { | |
response.textContent = 'Sorry, I could not understand. Please try again.'; | |
response.style.display = 'block'; | |
status.textContent = 'Press the mic button to restart the conversation.'; | |
isConversationActive = false; | |
} | |
}; | |
setTimeout(() => mediaRecorder.stop(), 5000); | |
}).catch(() => { | |
status.textContent = 'Microphone access denied.'; | |
isConversationActive = false; | |
}); | |
} | |
</script> | |
</body> | |
</html> | |
""" | |
if __name__ == "__main__": | |
app.run(host="0.0.0.0", port=7860) | |