Spaces:
Running
Running
File size: 3,919 Bytes
883c025 6cd989d 883c025 a46024d 883c025 6cd989d 883c025 fdde8c6 883c025 dbb3600 883c025 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
import os
from dotenv import load_dotenv
from scrapegraphai.graphs import SmartScraperGraph
from scrapegraphai.utils import prettify_exec_info
from langchain_community.llms import HuggingFaceEndpoint
from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings
import gradio as gr
import subprocess
import json
import re
# Ensure Playwright installs required browsers and dependencies
subprocess.run(["playwright", "install"])
# subprocess.run(["playwright", "install-deps"])
# Load environment variables
load_dotenv()
HUGGINGFACEHUB_API_TOKEN = os.getenv('HF_TOKEN')
# Initialize the model instances
repo_id = "mistralai/Mistral-7B-Instruct-v0.2"
# repo_id = "Qwen/Qwen2.5-72B-Instruct"
llm_model_instance = HuggingFaceEndpoint(
repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN
)
embedder_model_instance = HuggingFaceInferenceAPIEmbeddings(
api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2"
)
graph_config = {
"llm": {
"model_instance": llm_model_instance,
"model_tokens": 100000,
},
"embeddings": {"model_instance": embedder_model_instance}
}
#######
def clean_json_string(json_str):
"""
Removes any comments or prefixes before the actual JSON content.
Returns the cleaned JSON string.
"""
# Find the first occurrence of '{'
json_start = json_str.find('{')
if json_start == -1:
# If no '{' is found, try with '[' for arrays
json_start = json_str.find('[')
if json_start == -1:
return json_str # Return original if no JSON markers found
# Extract everything from the first JSON marker
cleaned_str = json_str[json_start:]
# Verify it's valid JSON
try:
json.loads(cleaned_str)
return cleaned_str
except json.JSONDecodeError:
return json_str # Return original if cleaning results in invalid JSON
def scrape_and_summarize(prompt, source):
smart_scraper_graph = SmartScraperGraph(
prompt=prompt,
source=source,
config=graph_config
)
result = smart_scraper_graph.run()
# Clean the result if it's a string
if isinstance(result, str):
result = clean_json_string(result)
exec_info = smart_scraper_graph.get_execution_info()
return result, prettify_exec_info(exec_info)
#######
# def scrape_and_summarize(prompt, source):
# smart_scraper_graph = SmartScraperGraph(
# prompt=prompt,
# source=source,
# config=graph_config
# )
# result = smart_scraper_graph.run()
# exec_info = smart_scraper_graph.get_execution_info()
# return result, prettify_exec_info(exec_info)
# Gradio interface
with gr.Blocks() as demo:
gr.Markdown("# Scrape websites, no-code version")
gr.Markdown("""
Scrape and summarize web content using advanced AI models on the Hugging Face Hub without writing any code. Input your desired prompt and source URL to get started.
""")
with gr.Row():
with gr.Column():
model_dropdown = gr.Textbox(label="Model", value="mistralai/Mistral-7B-Instruct-v0.2")
prompt_input = gr.Textbox(label="Prompt", value="List all the press releases with their headlines and urls. Output only the results; do not add any comments or include 'JSON OUTPUT' or similar phrases.")
source_input = gr.Textbox(label="Source URL", value="https://www.whitehouse.gov/")
scrape_button = gr.Button("Scrape and Summarize")
with gr.Column():
result_output = gr.JSON(label="Result")
exec_info_output = gr.Textbox(label="Execution Info")
scrape_button.click(
scrape_and_summarize,
inputs=[prompt_input, source_input],
outputs=[result_output, exec_info_output]
)
# Launch the Gradio app
if __name__ == "__main__":
demo.launch() |