app.py

from langchain_together import ChatTogether

"""# ⭐ LLM model with togithor Ai

"""

from langchain_community.llms import Together

import os
os.environ['TOGETHER_API_KEY'] = 'e83925ff068ab5e4598a56f68385fd37144469f50eec94f5c2e6647798f1be9e'

response = Together(
    model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
    max_tokens=1524,
    temperature=0.2,
    # top_p=1.1,
    # top_k=40,

    repetition_penalty=1.1,
    together_api_key=os.environ.get('TOGETHER_API_KEY')
)

"""# ⭐ Pinecone Vectore Database

"""


from langchain_pinecone import PineconeVectorStore
from langchain_openai import OpenAIEmbeddings
import os
os.environ['PINECONE_API_KEY']='f7413055-9b13-4bbc-8c92-56132e034bff'

em=OpenAIEmbeddings(api_key='sk-Q43XYIJudIE0Q7e5t23U5CjA5dMNYRGlMOhfm6VTA2T3BlbkFJn3a9zqCCIdjRcV7QKmkok3n0F1BL_KS0OzkLEbjXgA',model="text-embedding-3-small")
pc=PineconeVectorStore(index_name="learnverse",embedding=em)

"""#⭐ Summarization"""

import gradio as gr
from transformers import BitsAndBytesConfig, pipeline
from langchain_huggingface import HuggingFacePipeline
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser

"""# ⭐ Summary Prompt"""

prompt = """
    You are an expert AI summarization model tasked with creating a comprehensive summary for 10 years old kids of the provided context. The summary should be approximately one pages long and well-structured.

    this is the context:
    ```{context}```

    Please follow these specific guidelines for the summary:

    ### Detailed Summary
    - **Section 1: Key Concepts**
      - Introduce the first major topic or theme.
      - Use bullet points to list important details and insights.

    - **Section 2: Supporting Details**
      - Discuss secondary topics or supporting arguments.
      - Use bullet points to outline critical information and findings.

    ### Conclusion
    - Suggest any potential actions, solutions, or recommendations.



    this is the summary:

    """
summary_prompt = ChatPromptTemplate.from_template(
    prompt
)

summary_llm_chain = summary_prompt | response | StrOutputParser()

# create a function that upload pdf file and the summary chain get the file
from langchain_core.runnables import RunnablePassthrough

summary_pdf_chain = {"context": RunnablePassthrough()} | summary_llm_chain

"""# Query Prompt"""

q_prompt = """
            you are the greatest Question answering model ,you will get a question and answer the question based on the context.

      this is the context:
      ```{context}```

      this is the questions: {question}
    """
query_prompt = ChatPromptTemplate.from_template(
    q_prompt
)

query_llm_chain = query_prompt | response | StrOutputParser()

from langchain_core.runnables import RunnablePassthrough

retriever = pc.as_retriever(
    search_type="similarity",
    search_kwargs={'k': 4}
)

query_rag_chain = {"context": retriever, "question": RunnablePassthrough()}|query_llm_chain

"""# ⭐ Extract Text From Pdf

"""

from langchain_community.document_loaders import PyPDFLoader
import time

def extract_text_from_pdf(file):
    loader = PyPDFLoader(file)
    pages = loader.load_and_split()
    pc.from_documents(pages,index_name='learnverse',embedding=em)

    text = ""
    for page in pages:

      text += page.page_content
    return text

"""# ⭐ Text-to-Speech"""

from io import BytesIO
from elevenlabs import VoiceSettings, play
from elevenlabs.client import ElevenLabs
import ffmpeg
import IPython.display as ipd
import os

# Make sure to import the required classes
# sk_dcb140eeca914ac72a06ae91c4e9742b2c559c7451831c71
def text_to_speech_stream(text: str):
    ELEVENLABS_API_KEY = 'sk_3ec0ff46017e49189870e2dc9c51f87939d6e2d8cc823316'
    client = ElevenLabs(
        api_key=ELEVENLABS_API_KEY,
    )

    response = client.text_to_speech.convert(
        voice_id="jBpfuIE2acCO8z3wKNLl",
        optimize_streaming_latency="0",
        output_format="mp3_44100_64",
        text=text,
        model_id="eleven_multilingual_v2",
        voice_settings=VoiceSettings(
            stability=0.5,
            similarity_boost=0.75,
            style=0,
            use_speaker_boost=True,
        ),
    )

    audio_data = BytesIO()
    for chunk in response:
        if chunk:
            audio_data.write(chunk)

    audio_data.seek(0)

    # Create 'samples' directory if it doesn't exist
    if not os.path.exists('samples'):
        os.makedirs('samples')

    # Write the audio data to a file
    with open('samples/output.mp3', 'wb') as f:
        f.write(audio_data.read())

    return 'samples/output.mp3'

"""# ⭐ Get Topics

"""

# prompt for extracting three topics

topics_prompt="""

Extract the Main Topics:

Analyze the following text and identify the one main clear topic that related to AI like robot and VR etc Then, translate the topic into a simplified format that can be understood . The goal is to ensure that the  topic would be easy and clear so the model can accurately generate a 3d shape based on the simplified concepts.

Text: {context}
Answer:

"""

tp = ChatPromptTemplate.from_template(topics_prompt)

topic_chain = tp | response | StrOutputParser()

"""# Evauluation summary"""


import wandb
wandb.login()
# 956c40e3fd97485d68ec80c6841faec28368fd34

from rouge import Rouge

def evaluate_summary(generated_summary):
    wandb.init(
    # set the wandb project where this run will be logged
    project="learnverse")
    """
    Evaluates the generated summary against a list of reference summaries using the ROUGE metric.

    Parameters:
    - reference_summaries (list of str): A list of reference summaries (ground truth).
    - generated_summary (str): The summary generated by the model.

    Returns:
    - dict: A dictionary containing the average ROUGE-1, ROUGE-2, and ROUGE-L scores.
    """
    # Variable 1
    summary1 = """
    Introduction: The context discusses the concept of Artificial Intelligence (AI), its evolution, and its applications in various fields. AI is a branch of computer science that aims to create intelligent machines capable of performing tasks that typically require human intelligence.

    Section 1: Key Concepts
    * Definition and Types of AI: AI can be classified into three types: Weak or Narrow AI, General AI, and Strong AI. Weak AI is the most widely used type, which can perform a pre-defined narrow set of instructions without exhibiting any thinking capability.
    * Machine Learning and Deep Learning: Machine Learning (ML) is a subset of AI that enables computers to learn from data and past experiences. Deep Learning (DL) is a subdomain of ML that mimics the human nervous system and is used for image recognition, pattern recognition, and feature extraction.
    * Applications of AI: AI has various applications in fields such as agriculture, business, education, entertainment, healthcare, and space exploration.

    Section 2: Supporting Details
    * Agriculture: AI is used in soil analysis, crop sowing, pest control, and crop harvesting. It has improved crop yields and reduced the use of chemical fertilizers.
    * Healthcare: AI is used in medical diagnosis, image analysis, and patient monitoring. It has improved the accuracy of diagnosis and reduced the workload of healthcare professionals.
    * Education: AI is used in personalized learning, adaptive assessments, and intelligent tutoring systems. It has improved student outcomes and increased access to education.

    Section 3: Analysis and Interpretation
    * Impact of AI: AI has the potential to transform various industries and improve the quality of life. However, it also raises concerns about job displacement, data privacy, and security.
    * Challenges and Limitations: AI requires large amounts of data, computational power, and expertise. It also faces challenges related to interpretability, transparency, and accountability.

    Conclusion: In conclusion, AI is a rapidly evolving field with various applications in different industries. While it has the potential to transform the world, it also raises concerns about its impact on society. To fully harness the benefits of AI, it is essential to address its challenges and limitations and ensure that its development and deployment are responsible and ethical.
    """

    # Variable 2
    summary2 = """
    Introduction: The provided context is an introduction to Artificial Intelligence (AI), its subsets, and applications in various fields. The main purpose is to explore the capabilities, types, and domains of AI, as well as its impact on modern society.

    Detailed Summary

    Section 1: Key Concepts
    * Definition and Types of AI: AI is a branch of computer science that enables computers to mimic human behavior. There are three types of AI: Weak or Narrow AI, General AI, and Strong AI.
    * Domains of AI: The major domains of AI include neural networks, robotics, expert systems, fuzzy logic systems, and natural language processing (NLP).
    * Subsets of AI: The two major subsets of AI are Machine Learning (ML) and Deep Learning (DL).

    Section 2: Supporting Details
    * Machine Learning: ML is a subset of AI that enables computers to learn from data and past experiences. There are three types of ML: Supervised Learning, Unsupervised Learning, and Reinforcement Learning.
    * Deep Learning: DL is a subdomain of ML that mimics the human nervous system. It has various applications, including image recognition, natural language processing, and speech recognition.
    * Applications of AI: AI has numerous applications in agriculture, business, education, entertainment, healthcare, and space exploration.

    Section 3: Analysis and Interpretation
    * Impact of AI: AI has transformed various industries and has the potential to revolutionize healthcare, education, and other sectors.
    * Challenges and Limitations: AI faces challenges such as data accuracy, security, and interpretability. It also raises concerns about job displacement and bias.
    * Future Directions: AI is expected to continue growing and improving, with potential applications in areas like genome editing, personalized medicine, and smart cities.

    Conclusion: In conclusion, AI is a rapidly evolving field with numerous applications and potential benefits. However, it also raises concerns about data accuracy, security, and job displacement. As AI continues to grow and improve, it is essential to address these challenges and ensure that its benefits are equitably distributed.
    """

    # Variable 3
    summary3 = """
    Introduction: The context discusses the concept of Artificial Intelligence (AI) and its applications in various fields. AI is a branch of computer science that enables computers to mimic human behavior, assisting humans in performance and decision-making. The context highlights the importance of AI in modern society, its subsets, and its impact on healthcare, education, business, and other sectors.

    Section 1: Key Concepts
    * Artificial Intelligence (AI): AI is a domain of computer science that deals with the development of intelligent computer systems capable of perceiving, analyzing, and reacting to inputs.
    * Types of AI: AI can be classified into three types based on capabilities: Weak or Narrow AI, General AI, and Strong AI.
    * Subsets of AI: Machine Learning (ML) and Deep Learning (DL) are two subsets of AI used to solve problems using high-performance algorithms and multilayer neural networks.

    Section 2: Supporting Details
    * Applications of AI: AI has various applications in healthcare, education, business, and other sectors, including medical diagnosis, image processing, web search engines, and finance.
    * Machine Learning (ML): ML is a subset of AI that enables computers to learn from data and past experiences, improving performance and prediction accuracy.
    * Deep Learning (DL): DL is a subdomain of ML that mimics the human nervous system, using neural networks to analyze and interpret data.

    Section 3: Analysis and Interpretation
    * Impact of AI: AI has transformed various sectors, including healthcare, education, and business, by improving efficiency, accuracy, and decision-making.
    * Challenges and Limitations: AI faces challenges such as data accuracy, security, and interpretability, which need to be addressed to ensure its effective implementation.
    * Future Directions: AI is expected to continue transforming various sectors, with potential applications in space exploration, smart cities, and transportation.

    Conclusion: In conclusion, AI is a rapidly evolving field with significant implications for various sectors. Its subsets, ML and DL, have transformed the way we approach problems and make decisions. While AI faces challenges and limitations, its potential applications and benefits make it an essential technology for the future.
    """


    # Create the list of reference summaries
    reference_summaries = [summary1, summary2, summary3]

    # Initialize the ROUGE evaluator
    rouge = Rouge()

    # Initialize accumulators for ROUGE scores
    rouge_1 = {'r': 0, 'p': 0, 'f': 0}
    rouge_2 = {'r': 0, 'p': 0, 'f': 0}
    rouge_l = {'r': 0, 'p': 0, 'f': 0}

    # Evaluate each reference summary
    i=0
    for reference in reference_summaries:
        scores = rouge.get_scores(generated_summary, reference)
        i+=1

        rouge_1['r'] += scores[0]['rouge-1']['r']
        rouge_1['p'] += scores[0]['rouge-1']['p']
        rouge_1['f'] += scores[0]['rouge-1']['f']

        rouge_2['r'] += scores[0]['rouge-2']['r']
        rouge_2['p'] += scores[0]['rouge-2']['p']
        rouge_2['f'] += scores[0]['rouge-2']['f']

        rouge_l['r'] += scores[0]['rouge-l']['r']
        rouge_l['p'] += scores[0]['rouge-l']['p']
        rouge_l['f'] += scores[0]['rouge-l']['f']
        # print('\n')
        print("score #"+str(i))
        print(scores)
        # print(rouge_1)
        # print(rouge_2)
        # print(rouge_l)

    # Calculate the average scores
    num_references = len(reference_summaries)
    rouge_1 = {key: value / num_references for key, value in rouge_1.items()}
    rouge_2 = {key: value / num_references for key, value in rouge_2.items()}
    rouge_l = {key: value / num_references for key, value in rouge_l.items()}

    # Return the average scores in a dictionary
    print('\n')
    print("The Average Scores")
    print('')

    print(rouge_1)
    print(rouge_2)
    print(rouge_l)
    print('\n\n\n')



    wandb.log(rouge_1)
    wandb.log(rouge_2)
    wandb.log(rouge_l)


    if rouge_1['p'] < 0.1 and rouge_2['p'] < 0.1 and rouge_l['p'] < 0.1:
        wandb.alert(title='Low Precesion', text=f'Precesion {rouge_1["p"]},{rouge_2["p"]},{rouge_l["p"]} is below the acceptable theshold')

    wandb.finish()
    return {
        'ROUGE-1': rouge_1,
        'ROUGE-2': rouge_2,
        'ROUGE-L': rouge_l
    }

"""# 🦾 Function Integrator"""

def process_question(file):

    #pd_file is for giving the ai asist somthing short to create
    # pd_file = "AI is very good"


    pdffile = extract_text_from_pdf(file)
    three_topics = topic_chain.invoke({"context": pdffile})
    print("--------Three Topics------")
    print(three_topics)

    summary = summary_pdf_chain.invoke(pdffile)
    print("\n--------Summary---------")
    print(summary)

    print("--------Evaluation Summary---------")
    evaluation = evaluate_summary(summary)




    audio_file = text_to_speech_stream(summary)
    prompt = topics_prompt
    shape = generate_gif(prompt)
    ai_asistant = animate_image(audio_file)


    return summary,evaluation,ai_asistant,shape

# process_question()

"""#llm guard"""



from transformers import AutoTokenizer, BitsAndBytesConfig, AutoModelForCausalLM
import torch

model_id = "meta-llama/LlamaGuard-7b"
guard_tokenizer = AutoTokenizer.from_pretrained(model_id)

bnb_config_guard = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    # Allow offloading to CPU for parts of the model
    load_in_8bit_fp32_cpu_offload=True
)
guard_model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config_guard,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    )

def moderate_with_template(chat):
    input_ids = guard_tokenizer.apply_chat_template(chat, return_tensors="pt")
    output = guard_model.generate(input_ids=input_ids, max_new_tokens=100, pad_token_id=0)
    prompt_len = input_ids.shape[-1]
    return guard_tokenizer.decode(output[0][prompt_len:], skip_special_tokens=True)

def invoking(question):
  return query_rag_chain.invoke(question)

def answer_question(question):

       # Check if the question is safe using Llama guard
    chat = [ {"role": "user", "content": question} ]

    if not moderate_with_template(chat) == 'safe':
        return "I'm sorry, but I can't respond to that question as it may contain inappropriate content."

    ai_msg = invoking(question)  # Generate AI response


    system_response = [
    {"role": "user", "content": question},
    {"role": "assistant", "content": ai_msg},
]
    if not moderate_with_template(system_response) == 'safe':
        return "I generated a response, but it may contain inappropriate content. Let me try again with a more appropriate answer."
    else:
        return ai_msg

# answer_question("how to kill everybody")

"""# 🤖 *AI* assistent"""


# Commented out IPython magic to ensure Python compatibility.
!update-alternatives --install /usr/local/bin/python3 python3 /usr/bin/python3.8 2
!update-alternatives --install /usr/local/bin/python3 python3 /usr/bin/python3.9 1
!sudo apt install python3.8

!sudo apt-get install python3.8-distutils

!python --version

!apt-get update

!apt install software-properties-common

!sudo dpkg --remove --force-remove-reinstreq python3-pip python3-setuptools python3-wheel

!apt-get install python3-pip

print('Git clone project and install requirements...')
!git clone https://github.com/Winfredy/SadTalker &> /dev/null
# %cd SadTalker
!export PYTHONPATH=/content/SadTalker:$PYTHONPATH
!python3.8 -m pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113
!apt update
!apt install ffmpeg &> /dev/null
!python3.8 -m pip install -r requirements.txt

print('Download pre-trained models...')
!rm -rf checkpoints
!bash scripts/download_models.sh

"""# ⛳ 3D Shape"""

# Commented out IPython magic to ensure Python compatibility.
!git clone https://github.com/openai/shap-e
# %cd shap-e


import torch
from shap_e.diffusion.sample import sample_latents
from shap_e.diffusion.gaussian_diffusion import diffusion_from_config
from shap_e.models.download import load_model, load_config
from shap_e.util.notebooks import create_pan_cameras, decode_latent_images, gif_widget

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

import imageio
import os
import hashlib

def generate_gif(prompt):
    # device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    # Load models and diffusion configuration
    xm = load_model('transmitter', device=device)
    model = load_model('text300M', device=device)
    diffusion = diffusion_from_config(load_config('diffusion'))

    # Generate latents
    batch_size = 1
    guidance_scale = 8.95
    render_mode = 'nerf'
    size = 128

    latents = sample_latents(
        batch_size=batch_size,
        model=model,
        diffusion=diffusion,
        guidance_scale=guidance_scale,
        model_kwargs=dict(texts=[prompt] * batch_size),
        progress=True,
        clip_denoised=True,
        use_fp16=True,
        use_karras=True,
        karras_steps=64,
        sigma_min=1e-3,
        sigma_max=160,
        s_churn=0,
    )

    # Create cameras
    cameras = create_pan_cameras(size, device)

    # Render images and create GIF
    for i, latent in enumerate(latents):
        images = decode_latent_images(xm, latent, cameras, rendering_mode=render_mode)

    # Ensure the directory exists
    gif_dir = "generated_gifs"
    os.makedirs(gif_dir, exist_ok=True)

    # Generate a short, unique file name using a hash of the prompt
    prompt_hash = hashlib.md5(prompt.encode()).hexdigest()[:10]
    gif_path = os.path.join(gif_dir, f"{prompt_hash}.gif")

    # Save the images as a GIF
    imageio.mimsave(gif_path, images, fps=10)  # Save images as GIF

    return gif_path



"""#big ⛏ func"""

import ipywidgets as widgets
import glob
import matplotlib.pyplot as plt
from IPython.display import display, HTML
from base64 import b64encode
import os
import sys
import subprocess

from google.colab import drive
drive.mount('/content/drive')

import os
import subprocess
import glob

def animate_image(audio_file):
    # Display the selected image (optional if using in Gradio)
    img_path = '/content/drive/MyDrive/img_9.png'
    # print(f"Image Has Been Seleceted: ")

    # Run the animation generation script
    result = subprocess.run([
        "python3.8", "inference.py", "--driven_audio", audio_file,
        "--source_image", img_path, "--result_dir", "./results", "--still", "--preprocess", "full", "--enhancer", "gfpgan"
    ], capture_output=True, text=True)

    # Check for errors
    if result.stderr:
        print("Errors:", result.stderr, file=sys.stderr)

    # Find the generated video file
    mp4_files = glob.glob('./results/*.mp4')

    if mp4_files:
        mp4_path = mp4_files[0]
        print(f"Generated animation: {mp4_path}")
        return mp4_path
    else:
        print("No results found.")
        return None

"""# 🚀 Gradio"""

import gradio as gr

with gr.Blocks() as demo:
    gr.Markdown("## Summarization and Animation Tool")

    with gr.Row():
        with gr.Column():
            input_file = gr.File(label="Upload File", type='filepath')
            summary = gr.Textbox(label="Summary", lines=3)
            # evaluation_summary = gr.Textbox(label="Evaluation Summary", lines=3)
            animation_video = gr.Video(label="Animation Video")
            shape = gr.Image(label="3D Shape GIF")
            question = gr.Textbox(label="Question", lines=3)
            answer = gr.Textbox(label="Answer", lines=3)


    summarize_button = gr.Button("Summarize")
    summarize_button.click(process_question, inputs=input_file, outputs=[summary,animation_video,shape])


    question_button = gr.Button("Ask Question")
    question_button.click(lambda q: answer_question(q.strip()), inputs=[question], outputs=[answer])

demo.launch(debug=True)