SmartScraper / app.py
sergioska's picture
add scrape graph script
5465fb7
raw
history blame
No virus
1.32 kB
import streamlit as st
from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM
from scrapegraphai.graphs import SmartScraperGraph
from scrapegraphai.utils import prettify_exec_info
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1")
modelNomic = AutoModel.from_pretrained("nomic-ai/nomic-embed-text-v1.5", trust_remote_code=True)
graph_config = {
"llm": {
"model-instance": model,
"temperature": 1,
"format": "json", # Ollama needs the format to be specified explicitly
"model_tokens": 4096, # depending on the model set context length
},
"embeddings": {
"model-instance": modelNomic,
"temperature": 0,
}
}
# ************************************************
# Create the SmartScraperGraph instance and run it
# ************************************************
smart_scraper_graph = SmartScraperGraph(
prompt="List me shoes in first page with names, prices and image urls",
# also accepts a string with the already downloaded HTML code
source="https://www.footlocker.co.uk/en/category/sale/men.html",
config=graph_config
)
result = smart_scraper_graph.run()
print(result)
x = st.slider('Select a value')
st.write(x, 'squared is', x * x)