Spaces:
Running
Running
| import requests | |
| from bs4 import BeautifulSoup | |
| def download_text_and_title(url): | |
| try: | |
| # Remove the query string from the URL | |
| url = url.strip() | |
| url = url.split("?")[0] | |
| # Remove emojis and other special characters | |
| url = url.encode("ascii", "ignore").decode("ascii") | |
| # Send a GET request to the URL | |
| headers = { | |
| "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) " | |
| "AppleWebKit/537.36 (KHTML, like Gecko) " | |
| "Chrome/50.0.2661.102 Safari/537.36" | |
| } | |
| response = requests.get(url, headers=headers, allow_redirects=True) | |
| # While response is a redirect, follow it | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| title = soup.title.string if soup.title else "No Title Found" | |
| while title.startswith("http:/") or title.startswith("https:/"): | |
| url = title | |
| response = requests.get(url, headers=headers, allow_redirects=True) | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| title = soup.title.string if soup.title else "No Title Found" | |
| # Check if the request was successful | |
| if response.status_code == 200: | |
| # Parse the HTML content using BeautifulSoup | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| # Extract the title | |
| title = soup.title.string if soup.title else "No Title Found" | |
| # Extract all the text from the webpage | |
| text = [p.get_text() for p in soup.find_all("p")] | |
| text = [ | |
| p.replace("\n", " ").replace("\r", " ").replace("\t", " ") for p in text | |
| ] | |
| text = [" ".join(p.strip().split()) for p in text] | |
| text = [p for p in text if len(p) > 0 and len(p.split()) > 5] | |
| # Clean text | |
| text = "\n".join(text) | |
| title = title.replace("\n", " ").replace("\r", " ").replace("\t", " ") | |
| title = " ".join(title.strip().split()) | |
| return title, text, url | |
| else: | |
| print("Failed to retrieve the web page. Status code:", response.status_code) | |
| print("URL:", url) | |
| return None, None, None | |
| except Exception as e: | |
| print("An error occurred:", str(e)) | |
| print("URL:", url) | |
| return None, None, None | |
| # Example usage | |
| if __name__ == "__main__": | |
| url = "https://www.huffingtonpost.es/sociedad/esta-palabra-mas-prescindible-espanol-cambia-entiende.html" # Replace with the URL you want to scrape | |
| title, text = download_text_and_title(url) | |
| if title and text: | |
| print("Title:", title) | |
| print("Text:", text) | |
| else: | |
| print("Unable to retrieve text and title.") | |