Update app.py

app.py

@@ -2,148 +2,126 @@ import gradio as gr
 import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin
-from gradio_client import Client
+import os
 import json
 import csv
-import pandas
-import groq
-import os
 
+# Load environment variables
 api_key = os.environ.get('groq')
 read_key = os.environ.get('HF_TOKEN', None)
-client = groq.Client(api_key=api_key)
+
+# Initialize Groq client
+if api_key:
+    from groq import Client as GroqClient
+    client = GroqClient(api_key=api_key)
+else:
+    client = None
 
 # Use Llama 3 70B powered by Groq for answering
 def ask_llm(ort):
+    if not client:
+        return "Groq API key not set."
     
-    try:        
+    try:
         completion = client.chat.completions.create(
             model="llama3-70b-8192",
             messages=[
                 {"role": "system", "content": "You are a helpful assistant."},
                 {"role": "user", "content": f"{ort}. \n instruction: antworte kurz und knapp. antworte immer auf deutsch"}
             ],
-        )       
+        )
         return completion.choices[0].message.content
     except Exception as e:
         return f"Error in response generation: {str(e)}"
 
 def parse_links_and_content(ort):
-    
     base_url = "https://vereine-in-deutschland.net"
     all_links = []
     all_links_text = []
     initial_url = f"{base_url}/vereine/Bayern/{ort}"
-    
+
     try:
         response = requests.get(initial_url)
-        response.raise_for_status()  # Überprüfen, ob die Anfrage erfolgreich war
-        
-        # Parse the HTML content using BeautifulSoup
+        response.raise_for_status()
+
         soup = BeautifulSoup(response.content, 'html.parser')
-        
-        # Ermittle die letzte Seite
+
+        # Determine the last page
         link_element = soup.select_one('li.page-item:nth-child(8) > a:nth-child(1)')
-        
+        last_page = 1
         if link_element and 'href' in link_element.attrs:
             href = link_element['href']
-            # Extrahiere die letzten beiden Zeichen der URL
-            last_two_chars = href[-2:].strip()          
-            # Konvertiere die letzten beiden Zeichen in einen Integer
-            last_two_chars_int = int(last_two_chars)
-            print(last_two_chars_int)
-        else:
-            last_two_chars_int = 10  # Falls die letzte Seite nicht gefunden wird, nimm an, dass es nur eine Seite gibt
-
-        # Schleife durch alle Seiten und sammle Links
-        for page_number in range(1, last_two_chars_int +1):
+            last_page = int(href.split('/')[-1])
+
+        # Loop through all pages and collect links
+        for page_number in range(1, last_page + 1):
             page_url = f"{base_url}/vereine/Bayern/{ort}/p/{page_number}"
             response = requests.get(page_url)
-            response.raise_for_status()            
+            response.raise_for_status()
             soup = BeautifulSoup(response.content, 'html.parser')
             target_div = soup.select_one('div.row-cols-1:nth-child(4)')
-            
+
             if target_div:
                 links = [urljoin(base_url, a['href']) for a in target_div.find_all('a', href=True)]
-                texts = [a.text for a in target_div.find_all('a', href=True)] 
-                #print(texts)
+                texts = [a.text for a in target_div.find_all('a', href=True)]
                 all_links.extend(links)
                 all_links_text.extend(texts)
             else:
                 print(f"Target div not found on page {page_number}")
-        
+
     except Exception as e:
         return str(e), []
-        
+
     all_links = all_links[0::2]
     all_links_text = all_links_text[0::2]
-   
+
     return all_links_text, all_links
 
-def scrape_links(links):        
-    links=links
+def scrape_links(links):
     details = []
-    for contacts in links:     
-        contact_element = soup.select_one('.nav')
-        response = requests.get(contacts)
-        response.raise_for_status()
-        soup = BeautifulSoup(response.content, 'html.parser')
-        target_nav = soup.select_one('.nav')
-        details.append(target_nav.text)      
+    for link in links:
+        try:
+            response = requests.get(link)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.content, 'html.parser')
+            target_nav = soup.select_one('.nav')
+            if target_nav:
+                details.append(target_nav.text.strip())
+            else:
+                details.append("No contact information found")
+        except Exception as e:
+            details.append(f"Error: {str(e)}")
 
     return details
 
-    client = Client("mgokg/PerplexicaApi")
-    for verein in links:   
-        result = client.predict(
-            
-    		prompt=f"{verein}",
-    		api_name="/parse_links"
-    )
-        #print(result)
-        contact_details.append(result)
-        
-    return contact_details
-
-# Speichere die JSON-Daten in eine CSV-Datei 
-def save_to_csv(data, filename): 
-    keys = data[0].keys() 
-    with open(filename, 'w', newline='', encoding='utf-8') as output_file: 
-        dict_writer = csv.DictWriter(output_file, fieldnames=keys) 
-        dict_writer.writeheader() 
+def save_to_csv(data, filename):
+    keys = data[0].keys() if data else []
+    with open(filename, 'w', newline='', encoding='utf-8') as output_file:
+        dict_writer = csv.DictWriter(output_file, fieldnames=keys)
+        dict_writer.writeheader()
         dict_writer.writerows(data)
 
-# Erstelle die Gradio-Schnittstelle
+# Create the Gradio interface
 with gr.Blocks() as demo:
-    gr.Markdown("# ")
+    gr.Markdown("# Vereine in Deutschland")
     with gr.Row():
         ort_input = gr.Textbox(label="Ort", placeholder="Gib den Namen des Ortes ein")
     with gr.Row():
-        links_output = gr.JSON(label="Antwort")
-        rechts_output = gr.JSON(label="Antwort")
-    #links_output = gr.DataFrame(label="Ergebnisse")
-    #json_output = gr.JSON(label="Ergebnisse")
+        links_output = gr.JSON(label="Links")
+        details_output = gr.JSON(label="Details")
 
     def process_ort(ort):
-        #antwort = ask_llm(ort)
-        #antwort=gr.Markdown()
-        #return antwort
-        links = parse_links_and_content(ort)
-        #return links
-        contact= scrape_links(links)      
-        json_data = [json.loads(item) for item in contact] 
-        #save_to_csv(json_data, './contact_details.csv')
-        #return f"[Download CSV](contact_details.csv)", json_data
-        #return json_data
-        #return contact
-        return json_data, links
-        #return json_data
-        
-    # Button zum Starten der Parsung
-    button = gr.Button("senden")
-    
-    # Verbinde den Button mit der Funktion
-    button.click(fn=process_ort, inputs=ort_input, outputs=[links_output, rechts_output])
+        links_text, links = parse_links_and_content(ort)
+        contact_details = scrape_links(links)
+        json_data = [json.loads(detail) for detail in contact_details if detail.startswith("{")]
+        save_to_csv(json_data, './contact_details.csv')
+        return links_text, contact_details
+
+    # Button to start the parsing
+    button = gr.Button("Senden")
+
+    # Connect the button to the function
+    button.click(fn=process_ort, inputs=ort_input, outputs=[links_output, details_output])
 
-# Starte die Gradio-Anwendung
-demo.launch()
+# Launch the Gradio application
+demo.launch()