app.py

import gradio as gr
from gradio_rich_textbox import RichTextbox
from PIL import Image
from surya.ocr import run_ocr
from surya.model.detection.segformer import load_model as load_det_model, load_processor as load_det_processor
from surya.model.recognition.model import load_model as load_rec_model
from surya.model.recognition.processor import load_processor as load_rec_processor
from gradio_client import Client
from dotenv import load_dotenv
import requests
from io import BytesIO
import cohere
import os
import re


title = "# Welcome to AyaTonic"
description = "Learn a New Language With Aya"

# Load environment variables
load_dotenv()
COHERE_API_KEY = os.getenv('CO_API_KEY')
SEAMLESSM4T = os.getenv('SEAMLESSM4T')


# Regular expression patterns for each color
patterns = {
    "red": r'<span style="color: red;">(.*?)</span>',
    "blue": r'<span style="color: blue;">(.*?)</span>',
    "green": r'<span style="color: green;">(.*?)</span>',
}

# Dictionaries to hold the matches
matches = {
    "red": [],
    "blue": [],
    "green": [],
}
class TaggedPhraseExtractor:
    def __init__(self, text=''):
        self.text = text
        self.patterns = {}

    def set_text(self, text):
        """Set the text to search within."""
        self.text = text

    def add_pattern(self, color, pattern):
        """Add a new color and its associated pattern."""
        self.patterns[color] = pattern

    def extract_phrases(self):
        """Extract phrases for all colors and patterns added."""
        matches = {color: re.findall(pattern, self.text) for color, pattern in self.patterns.items()}
        return matches

    def print_phrases(self):
        """Extract phrases and print them."""
        matches = self.extract_phrases()
        for color, phrases in matches.items():
            print(f"Phrases with color {color}:")
            for phrase in phrases:
                print(f"- {phrase}")
            print()  
            
co = cohere.Client(COHERE_API_KEY)
audio_client = Client(SEAMLESSM4T)

def process_audio_to_text(audio_path):
    """
    Convert audio input to text using the Gradio client.
    """
    result = audio_client.predict(
        audio_path,
        "English",  
        "English",  
        api_name="/s2tt"
    )
    print("Audio Result: ", result)
    return result['text']  # Adjust based on the actual response

def process_text_to_audio(text, target_language="English"):
    """
    Convert text input to audio using the Gradio client.
    """
    result = audio_client.predict(
        text,
        "English",  
        target_language,  
        api_name="/t2st"
    )
    return result['audio']  # Adjust based on the actual response

class OCRProcessor:
    def __init__(self, langs=["en"]):
        self.langs = langs
        self.det_processor, self.det_model = load_det_processor(), load_det_model()
        self.rec_model, self.rec_processor = load_rec_model(), load_rec_processor()

    def process_image(self, image):
        """
        Process a PIL image and return the OCR text.
        """
        predictions = run_ocr([image], [self.langs], self.det_model, self.det_processor, self.rec_model, self.rec_processor)
        return predictions[0]  # Assuming the first item in predictions contains the desired text

    def process_pdf(self, pdf_path):
        """
        Process a PDF file and return the OCR text.
        """
        predictions = run_ocr([pdf_path], [self.langs], self.det_model, self.det_processor, self.rec_model, self.rec_processor)
        return predictions[0]  # Assuming the first item in predictions contains the desired text
    
def process_input(image=None, file=None, audio=None, text=""):
    ocr_processor = OCRProcessor()
    final_text = text
    if image is not None:
        ocr_prediction = ocr_processor.process_image(image)
        # gettig text from ocr object
        for idx in range(len((list(ocr_prediction)[0][1]))):
            final_text += " "
            final_text += list((list(ocr_prediction)[0][1])[idx])[1][1]
    if file is not None:
        if file.name.lower().endswith(('.png', '.jpg', '.jpeg')):
            pil_image = Image.open(file)
            ocr_prediction = ocr_processor.process_image(pil_image)
            # gettig text from ocr object
            for idx in range(len((list(ocr_prediction)[0][1]))):
                final_text += " "
                final_text += list((list(ocr_prediction)[0][1])[idx])[1][1]
        elif file.name.lower().endswith('.pdf'):
            ocr_prediction = ocr_processor.process_pdf(file.name)
            # gettig text from ocr object
            for idx in range(len((list(ocr_prediction)[0][1]))):
                final_text += " "
                final_text += list((list(ocr_prediction)[0][1])[idx])[1][1]
        else:
            final_text += "\nUnsupported file type."
    print("OCR Text: ", final_text)
    if audio is not None:
        audio_text = process_audio_to_text(audio)
        final_text += "\n" + audio_text

    response = co.generate(
        model='c4ai-aya',
        prompt=final_text,
        max_tokens=1024,
        temperature=0.5
    )
    generated_text = response.generations[0].text
    print("Generated Text: ", generated_text)

    # Process generated text with command-nightly model
    response = co.generate(
        model='command-nightly',
        prompt=generated_text,
        max_tokens=1024,
        temperature=0.5
    )
    processed_text = response.generations[0].text

    audio_output = process_text_to_audio(processed_text)

    return processed_text, audio_output

# Define Gradio interface
iface = gr.Interface(
    fn=process_input,
    inputs=[
        gr.Image(type="pil", label="Camera Input"),
        gr.File(label="File Upload"),
        gr.Audio(sources="microphone", type="filepath", label="Mic Input"),
        gr.Textbox(lines=2, label="Text Input")
    ],
    outputs=[
        RichTextbox(label="Processed Text"),
        gr.Audio(label="Audio Output")
    ],
    title=title,
    description=description
)

if __name__ == "__main__":
    iface.launch()


# co = cohere.Client('yhA228YGeZSl1ctten8LQxw2dky2nngHetXFjV2Q') # This is your trial API key
# response = co.generate(
#   model='c4ai-aya',
#   prompt='एक यांत्रिक घड़ी दिन के समय को प्रदान करने के लिए एक गैर-इलेक्ट्रॉनिक तंत्र का उपयोग करती है। एक मुख्य स्प्रिंग का उपयोग यांत्रिक तंत्र को ऊर्जा संग्रहीत करने के लिए किया जाता है। एक यांत्रिक घड़ी में दांतों का एक कुंडल होता है जो धीरे-धीरे मुख्य स्प्रिंग से संचालित होता है। दांतों के कुंडल को एक यांत्रिक तंत्र में स्थानांतरित करने के लिए पहियों की एक श्रृंखला का उपयोग किया जाता है जो हाथों को घड़ी के चेहरे पर दाईं ओर ले जाता है। घड़ी के तंत्र को स्थिर करने और यह सुनिश्चित करने के लिए कि हाथ सही दिशा में घूमते हैं, एक कंपन का उपयोग किया जाता है।\n\nProduce a complete blog post in FRENCH based on the above : ',
#   max_tokens=3674,
#   temperature=0.9,
#   k=0,
#   stop_sequences=[],
#   return_likelihoods='NONE')
# print('Prediction: {}'.format(response.generations[0].text))

# client = Client("https://facebook-seamless-m4t-v2-large.hf.space/--replicas/nq5nn/")
# result = client.predict(
# 		https://github.com/gradio-app/gradio/raw/main/test/test_files/audio_sample.wav,	# filepath  in 'Input speech' Audio component
# 		Afrikaans,	# Literal[Afrikaans, Amharic, Armenian, Assamese, Basque, Belarusian, Bengali, Bosnian, Bulgarian, Burmese, Cantonese, Catalan, Cebuano, Central Kurdish, Croatian, Czech, Danish, Dutch, Egyptian Arabic, English, Estonian, Finnish, French, Galician, Ganda, Georgian, German, Greek, Gujarati, Halh Mongolian, Hebrew, Hindi, Hungarian, Icelandic, Igbo, Indonesian, Irish, Italian, Japanese, Javanese, Kannada, Kazakh, Khmer, Korean, Kyrgyz, Lao, Lithuanian, Luo, Macedonian, Maithili, Malayalam, Maltese, Mandarin Chinese, Marathi, Meitei, Modern Standard Arabic, Moroccan Arabic, Nepali, North Azerbaijani, Northern Uzbek, Norwegian Bokmål, Norwegian Nynorsk, Nyanja, Odia, Polish, Portuguese, Punjabi, Romanian, Russian, Serbian, Shona, Sindhi, Slovak, Slovenian, Somali, Southern Pashto, Spanish, Standard Latvian, Standard Malay, Swahili, Swedish, Tagalog, Tajik, Tamil, Telugu, Thai, Turkish, Ukrainian, Urdu, Vietnamese, Welsh, West Central Oromo, Western Persian, Yoruba, Zulu]  in 'Source language' Dropdown component
# 		Bengali,	# Literal[Bengali, Catalan, Czech, Danish, Dutch, English, Estonian, Finnish, French, German, Hindi, Indonesian, Italian, Japanese, Korean, Maltese, Mandarin Chinese, Modern Standard Arabic, Northern Uzbek, Polish, Portuguese, Romanian, Russian, Slovak, Spanish, Swahili, Swedish, Tagalog, Telugu, Thai, Turkish, Ukrainian, Urdu, Vietnamese, Welsh, Western Persian]  in 'Target language' Dropdown component
# 							api_name="/s2st"
# )
# print(result)

# co = cohere.Client('yhA228YGeZSl1ctten8LQxw2dky2nngHetXFjV2Q')
# response = co.generate(
#   model='command-nightly',
#   prompt='Les mécanismes de montres mécaniques\n\nLes mécanismes de montres mécaniques sont des mécanismes qui indiquent la journée, mais pas l\'électronique. Elles utilisent un ressort principal pour stocker l\'énergie nécessaire au fonctionnement des mécanismes. Un train d\'engrenages est utilisé pour transférer l\'énergie du ressort principal à un ensemble de roues qui font tourner les aiguilles dans le sens horaire sur le cadran de la montre.\n\nLes mécanismes de montres mécaniques sontdakshineswar omkarnathji, qui sont des lieux de culte qui sont construits dans le temple. Les engrenages sont des roues qui sont utilisées pour transférer l\'énergie du ressort principal à un ensemble de roues qui font tourner les aiguilles dans le sens horaire sur le cadran de la montre.\n\nLe ressort principal est un ressort qui est utilisé pour stocker l\'énergie nécessaire au fonctionnement des mécanismes de la montre. Le ressort principal est un ressort qui est utilisé pour stocker l\'énergie nécessaire au fonctionnement des mécanismes de la montre, et il est utilisé pour transférer l\'énergie aux engrenages, qui sont des roues qui sont utilisées pour faire tourner les aiguilles dans le sens horaire sur le cadran de la montre.\n\nLes engrenages sont des roues qui sont utilisées pour faire tourner les aiguilles dans le sens horaire sur le cadran de la montre, et elles sont utilisées pour transférer l\'énergie du ressort principal aux roues qui font tourner les aiguilles dans le sens horaire sur le cadran de la montre.\n\nLes mécanismes de montres mécaniques sont des mécanismes qui indiquent la journée, et elles sont utilisées pour transférer l\'énergie du ressort principal à un ensemble de roues qui font tourner les aiguilles dans le sens horaire sur le cadran de la montre.\n\nLes mécanismes de montres mécaniques sont des mécanismes qui indiquent la journée, et elles sont utilisées pour transférer l\'énergie du ressort principal à un ensemble de roues qui font tourner les aiguilles dans le sens horaire sur le cadran de la montre, et elles sont utilisées pour stabiliser le mécanisme de la montre, et pour s\'assurer que les aiguilles tournent dans le bon sens.\n\nthe above text is a learning aid. you must use rich text format to rewrite the above and add 1 . a red color tags for nouns 2. a blue color tag for verbs 3. a green color tag for adjectives and adverbs:',
#   max_tokens=7294,
#   temperature=0.6,
#   k=0,
#   stop_sequences=[],
#   return_likelihoods='NONE')
# print('Prediction: {}'.format(response.generations[0].text))
# example = RichTextbox().example_inputs()



iface = gr.Interface(
    fn=process_input,
    inputs=[
        gr.Image(type="pil", label="Camera Input"),
        gr.File(label="File Upload"),
        gr.Audio(sources="microphone", type="filepath", label="Mic Input"),
        gr.Textbox(lines=2, label="Text Input")
    ],
    outputs=[
        gr.RichTextbox(label="Processed Text"),
        gr.Audio(label="Audio Output")
    ],
    title="OCR and Speech Processing App",
    description="This app processes images, PDFs, and audio inputs to generate text and audio outputs."
)

if __name__ == "__main__":
    iface.launch()

demo = gr.Interface(
    lambda x:x,
    RichTextbox(),  # interactive version of your component
    RichTextbox(),  # static version of your component
    examples=[[example]],  # uncomment this line to view the "example version" of your component
)


if __name__ == "__main__":
    demo.launch()