Spaces:
Running
Running
| # -*- coding: utf-8 -*- | |
| """ | |
| Created on Tue Feb 20 2023 | |
| @author: cyberandy | |
| """ | |
| # ---------------------------------------------------------------------------- # | |
| # Imports | |
| # ---------------------------------------------------------------------------- # | |
| from io import StringIO # for redirect_stdout | |
| from functools import wraps # for caching | |
| import contextlib # for redirect_stdout | |
| import tldextract | |
| import requests | |
| import streamlit as st | |
| import pandas as pd | |
| import streamlit.components.v1 as components | |
| import json | |
| import os | |
| from openai import OpenAI | |
| # Unset any proxy environment variables that might be causing issues | |
| proxy_vars = ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy'] | |
| for var in proxy_vars: | |
| if var in os.environ: | |
| del os.environ[var] | |
| # ---------------------------------------------------------------------------- # | |
| # App Config. & Styling | |
| # ---------------------------------------------------------------------------- # | |
| PAGE_CONFIG = { | |
| "page_title": "Structured Data Audit - a Free SEO Tool by WordLift", | |
| "page_icon": "img/fav-ico.png", | |
| "layout": "centered" | |
| } | |
| def local_css(file_name): | |
| with open(file_name) as f: | |
| st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True) | |
| st.set_page_config(**PAGE_CONFIG) | |
| local_css("style.css") | |
| # ---------------------------------------------------------------------------- # | |
| # Web Application | |
| # ---------------------------------------------------------------------------- # | |
| # st.title("🔥 Schema Audit 🔥") | |
| # ---------------------------------------------------------------------------- # | |
| # Sidebar | |
| # ---------------------------------------------------------------------------- # | |
| # st.sidebar.image("img/logo-wordlift.png", width=200) | |
| # st.sidebar.info("Run the schema audit on any website to quickly get an overview of the available markup. \ | |
| # Simply add the naked domain without 'www.' (eg. etsy.com or etsy.com/about) URL and click on ""ANALYZE"" to get the results.") | |
| # st.sidebar.subheader("Configuration") | |
| # ---------------------------------------------------------------------------- # | |
| # Functions | |
| # ---------------------------------------------------------------------------- # | |
| # Set the API endpoint and the API key | |
| API_ENDPOINT = "https://api2.woorank.com/reviews" | |
| API_KEY = os.environ.get("woorank_api_key") | |
| openai_api_key = os.environ.get("openai_api_key") | |
| if not API_KEY: | |
| st.error("The API keys are not properly configured. Check your environment variables!") | |
| elif not openai_api_key: | |
| st.error("The OpenAI API key is not properly configured. Check your environment variables!") | |
| else: | |
| # Generate the report by calling the ChatGPT Turbo API and the WooRank API | |
| # First, let's create a simple PromptTemplate class since it's not imported | |
| class PromptTemplate: | |
| def __init__(self, template, input_variables): | |
| self.template = template | |
| self.input_variables = input_variables | |
| def format(self, **kwargs): | |
| return self.template.format(**kwargs) | |
| def analyze_data(_advice, _items, _topics, _issues, _technologies, openai_api_key): | |
| """ | |
| Analyzes website data and generates a structured report using OpenAI's GPT model. | |
| Args: | |
| _advice (list): A list of strings, each string is a piece of advice | |
| _items (list): A list of items that are being analyzed | |
| _topics (list): A list of topics that the user is interested in | |
| _issues (list): A list of issues that the user has selected | |
| _technologies (list): A list of technologies that the user has selected | |
| openai_api_key (str): The OpenAI API key | |
| Returns: | |
| str: A JSON-formatted string containing the analysis report | |
| """ | |
| try: | |
| # Create the system message for ChatGPT | |
| prefix_messages = [{ | |
| "role": "system", | |
| "content": '''You are an SEO expert specializing in structured data analysis. Your task is to create JSON-formatted reports about websites' structured data. | |
| Key requirements: | |
| 1. Always format output as a valid JSON object | |
| 2. Use the exact structure provided in the template | |
| 3. Include HTML formatting (<i>, <b>, <u>) as specified | |
| 4. Add relevant links to structured data (https://wordlift.io/blog/en/entity/structured-data/) and schema.org (https://wordlift.io/blog/en/entity/schema-org/) in the first section | |
| 5. Keep responses concise but informative | |
| 6. Ensure proper JSON escaping for quotes and special characters | |
| Remember: The output must be a single, valid JSON object that can be parsed without additional processing.''' | |
| }] | |
| # Initialize OpenAI client with basic configuration | |
| client = OpenAI( | |
| api_key=openai_api_key, | |
| base_url="https://api.openai.com/v1" | |
| ) | |
| # Construct messages for the chat API | |
| messages = [] | |
| messages.extend(prefix_messages) | |
| # Create the prompt template and run statement based on conditions | |
| if not _issues and len(_items) > 0: | |
| # Case 1: When there are NO issues but there ARE items | |
| template = """ | |
| Create a JSON object based on the following data: | |
| 1. {advice} and schema classes: {items} | |
| 2. Entities found: {topics} | |
| 3. Technologies: {technologies} | |
| Structure your response as a valid JSON object with this exact format: | |
| {{ | |
| "first": "Analysis of schema classes with classes marked in <i>italic</i>", | |
| "second": "Description of entities marked in <b>bold</b>", | |
| "third": "Description of technologies in <i>italic</i>" | |
| }}""" | |
| prompt = PromptTemplate( | |
| template=template, | |
| input_variables=["advice", "items", "topics", "technologies"] | |
| ) | |
| run_statement = { | |
| "advice": _advice, | |
| "items": _items, | |
| "topics": _topics, | |
| "technologies": _technologies | |
| } | |
| elif not _items: | |
| # Case 2: When there are NO schema classes | |
| template = """ | |
| Create a JSON object for a website with no schema classes, based on: | |
| 1. Entities found: {topics} | |
| 2. Technologies: {technologies} | |
| Structure your response as a valid JSON object with this exact format: | |
| {{ | |
| "first": "Notice about missing schema classes", | |
| "second": "Description of entities marked in <b>bold</b>", | |
| "third": "Description of technologies in <i>italic</i>" | |
| }}""" | |
| prompt = PromptTemplate( | |
| template=template, | |
| input_variables=["topics", "technologies"] | |
| ) | |
| run_statement = { | |
| "topics": _topics, | |
| "technologies": _technologies | |
| } | |
| else: | |
| # Case 3: When there ARE issues | |
| template = """ | |
| Create a JSON object based on the following data: | |
| 1. {advice} and schema classes: {items} | |
| 2. Markup issues: {issues} | |
| 3. Entities found: {topics} | |
| 4. Technologies: {technologies} | |
| Structure your response as a valid JSON object with this exact format: | |
| {{ | |
| "first": "Analysis of schema classes with classes marked in <i>italic</i>", | |
| "second": "Description of issues marked in <u>underline</u>", | |
| "third": "Description of entities marked in <b>bold</b>", | |
| "fourth": "Description of technologies in <i>italic</i>" | |
| }}""" | |
| prompt = PromptTemplate( | |
| template=template, | |
| input_variables=["advice", "items", "topics", "issues", "technologies"] | |
| ) | |
| run_statement = { | |
| "advice": _advice, | |
| "items": _items, | |
| "topics": _topics, | |
| "issues": _issues, | |
| "technologies": _technologies | |
| } | |
| # Format the prompt and add it to messages | |
| user_message = prompt.format(**run_statement) | |
| messages.append({"role": "user", "content": user_message}) | |
| # Make the API call with better error handling | |
| try: | |
| response = client.chat.completions.create( | |
| model="gpt-4", | |
| messages=messages, | |
| temperature=0.7, | |
| max_tokens=1500 | |
| ) | |
| if hasattr(response.choices[0].message, 'content'): | |
| out = response.choices[0].message.content | |
| else: | |
| out = "Error: No content in response" | |
| except Exception as e: | |
| error_msg = str(e) | |
| print(f"OpenAI API Error: {error_msg}") | |
| if "proxies" in error_msg: | |
| out = "Error: Proxy configuration issue. Please check your environment settings." | |
| else: | |
| out = f"Sorry, there was an error with the OpenAI API: {error_msg}" | |
| return out | |
| except Exception as e: | |
| error_message = f"An unexpected error occurred: {str(e)}" | |
| print(error_message) | |
| return error_message | |
| # Call WooRank API to get the data (cached) | |
| def get_woorank_data(url): | |
| """ | |
| It takes a URL as input, and returns a dictionary of the data from the Woorank API | |
| :param url: The URL of the website you want to get data for | |
| """ | |
| # Extract the domain from the URL | |
| extracted = tldextract.extract(url) | |
| url = f"{extracted.domain}.{extracted.suffix}" | |
| # Build the API URL | |
| api_url = f"{API_ENDPOINT}?url={url}" | |
| # Set the API key in the headers | |
| headers = {"x-api-key": API_KEY, | |
| "Accept": "application/json"} | |
| # Call the API using HTTP GET and parse the JSON response to extract what we need | |
| response = requests.get(api_url, headers=headers) | |
| data = response.json() | |
| result = data.get("criteria", {}).get("schema_org", {}) | |
| advice = result.get("advice", {}) | |
| items = result.get("data", {}).get("counts", {}) | |
| issues = result.get("data", {}).get("issues", {}) | |
| topics_raw = data.get("criteria", {}).get("topics", {}).get("data", {}) | |
| technologies_raw = data.get("criteria", {}).get( | |
| "technologies", {}).get("data", {}).get("technologies", {}) | |
| # extract the unique English labels into a list | |
| topics = list( | |
| set([label for item in topics_raw for label in item['dbpediaLabelsEn']])) | |
| # extract the technologies that are related to seo and search-engines | |
| technologies = [] | |
| for item in technologies_raw: | |
| if "seo" in item["categories"] or "search-engines" in item["categories"]: | |
| technologies.append(item["app"]) | |
| # Return now all the items we need | |
| return result, advice, items, issues, topics, technologies | |
| # Here capture the output of the function and write it to the Streamlit app for debugging purposes | |
| def capture_output(func): | |
| """Capture output from running a function and write using streamlit.""" | |
| def wrapper(*args, **kwargs): | |
| # Redirect output to string buffers | |
| stdout, stderr = StringIO(), StringIO() | |
| try: | |
| with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): | |
| return func(*args, **kwargs) | |
| except Exception as err: | |
| print(f"Failure while executing: {err}") | |
| finally: | |
| if _stdout := stdout.getvalue(): | |
| print("Execution stdout:") | |
| print(_stdout) | |
| if _stderr := stderr.getvalue(): | |
| print("Execution stderr:") | |
| print(_stderr) | |
| return wrapper | |
| # ---------------------------------------------------------------------------- # | |
| # Main Function | |
| # ---------------------------------------------------------------------------- # | |
| def main(): | |
| # Set up the Streamlit app | |
| # Adding the input for the URL | |
| url = st.text_input("Enter a URL to analyze") | |
| if st.button("RUN THE STRUCTURED DATA AUDIT"): | |
| # Call the Woorank API | |
| schema_org, advice, items, issues, topics, technologies = get_woorank_data( | |
| url) | |
| if not advice: | |
| st.warning("Whoops, sorry, our bot didn't find any data. It might be that the URL is not accessible (a firewall is in place, for example), or the website does not have structured data.", icon="⚠️") | |
| else: | |
| msg = analyze_data(advice, items, topics, issues, technologies, openai_api_key) | |
| # Display the results when the button is clicked and the data is available | |
| if schema_org and msg: | |
| st.write("##### Your Findings 📈") | |
| try: | |
| data_out = json.loads(msg) | |
| # here is the first block of text with the advice | |
| first_block_text = data_out['first'] | |
| # here is the second block of text (opportunities if there are no issues, issues if there are) | |
| second_block_text = data_out['second'] | |
| # here we create the HTML string for the first block of text (advice) | |
| htmlstr1 = f"""<div class="success">{first_block_text}</div>""" | |
| st.markdown(htmlstr1, unsafe_allow_html=True) | |
| # adding a disclosure message | |
| st.markdown( | |
| """<div class="disclosure">*These findings are based on the analysis of your website as seen from the "eyes" of a crawler.</div>""", unsafe_allow_html=True) | |
| # if there are no issues, we only have three blocks of text (advice, opportunities, technologies) | |
| if not issues: | |
| # here we get the third block of text with the technologies | |
| third_block_text = data_out['third'] | |
| # here we create the HTML string for the second block of text (opportunities) | |
| htmlstr2 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{second_block_text}</p>""" | |
| st.markdown(htmlstr2, unsafe_allow_html=True) | |
| # here we create the HTML string for the third block of text (technologies) | |
| htmlstr3 = f"""<p class="technology">👩🏽💻 <b>Technologies</b></br>{third_block_text}</p>""" | |
| st.markdown(htmlstr3, unsafe_allow_html=True) | |
| # if there are issues, we have four blocks of text (advice, issues, opportunities, technologies) | |
| else: | |
| # here we get the third block of text with the opportunities | |
| third_block_text = data_out['third'] | |
| # here we get the fourth block of text with the technologies | |
| fourth_block_text = data_out['fourth'] | |
| # here we create the HTML string for the second block of text (issues) | |
| htmlstr2 = f"""<p class="warning">⚠️ <b>Warnings</b></br>{second_block_text}</p>""" | |
| st.markdown(htmlstr2, unsafe_allow_html=True) | |
| # here we create the HTML string for the third block of text (opportunities) | |
| htmlstr3 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{third_block_text}</p>""" | |
| st.markdown(htmlstr3, unsafe_allow_html=True) | |
| # here we create the HTML string for the fourth block of text (technologies) | |
| htmlstr4 = f"""<p class="technology">👩🏽💻 <b>Technologies</b></br>{fourth_block_text}</p>""" | |
| st.markdown(htmlstr4, unsafe_allow_html=True) | |
| except Exception as e: | |
| st.warning( | |
| "Sorry, something went wrong. Please try again later.", icon="⚠️") | |
| # Adding debug info | |
| stprint = capture_output(print) | |
| stprint(e) | |
| stprint(msg) | |
| st.write("---") | |
| # Adding an expandable section to display the full response | |
| with st.expander("INSPECT THE REPORT"): | |
| # st.write("#### Advice") | |
| # st.markdown(advice, unsafe_allow_html=True) | |
| st.write("##### Items") | |
| st.write(items) | |
| if not issues: | |
| st.write("No issues found on the structured data") | |
| else: | |
| st.write("#### Issues") | |
| st.write(issues) | |
| st.write("##### Entities") | |
| st.write(topics) | |
| st.write("##### Technologies") | |
| st.write(technologies) | |
| st.write("##### Full response") | |
| st.write(schema_org) | |
| # If the API call fails, display an error message | |
| else: | |
| if len(url) == 0: | |
| st.warning("Please enter a URL to analyze") | |
| if __name__ == "__main__": | |
| main() |