Spaces:
Sleeping
Sleeping
File size: 8,622 Bytes
3c77d98 7ba3a06 3c77d98 13f6685 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 7ba3a06 3c77d98 8dbd54d 997bc87 8dbd54d 997bc87 8dbd54d 997bc87 8dbd54d 997bc87 3c77d98 7ba3a06 3c77d98 8dbd54d 997bc87 8dbd54d 3c77d98 8dbd54d 3c77d98 7ba3a06 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 |
import requests
import pandas as pd
from bs4 import BeautifulSoup
def extract_div_contents_from_url(url):
response = requests.get(url)
if response.status_code != 200:
print(f"Error: Received status code {response.status_code} for URL: {url}")
return pd.DataFrame(columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
soup = BeautifulSoup(response.content, 'html.parser')
div_classes = ["mw-heading mw-heading3",'boilerplate afd vfd xfd-closed', 'boilerplate afd vfd xfd-closed archived mw-archivedtalk']
divs = []
for div_class in div_classes:
divs.extend(soup.find_all('div', class_=div_class))
url_fragment = url.split('#')[-1].replace('_', ' ')
data = []
for div in divs:
try:
title = None
text_url = None
# Extract title and text_url
title_tag = div.find('a')
if title_tag:
title_span = div.find('span', {'data-mw-comment-start': True})
if title_span:
title_anchor = title_span.find_next_sibling('a')
if title_anchor:
title = title_anchor.text
text_url = 'https://en.wikipedia.org' + title_anchor['href']
else:
title = title_tag.text
text_url = 'https://en.wikipedia.org' + title_tag['href']
if title == 'talk page' or title is None:
heading_tag = div.find('div', class_='mw-heading mw-heading3')
if heading_tag:
title_tag = heading_tag.find('a')
if title_tag:
title = title_tag.text
text_url = 'https://en.wikipedia.org' + title_tag['href']
if not title:
continue
if title.lower() != url_fragment.lower():
continue
deletion_discussion = div.prettify()
# Extract label
label = ''
verdict_tag = div.find('p')
if verdict_tag:
label_b_tag = verdict_tag.find('b')
if label_b_tag:
label = label_b_tag.text.strip()
# Extract confirmation
confirmation = ''
discussion_tag = div.find('dd')
if discussion_tag:
discussion_tag_i = discussion_tag.find('i')
if discussion_tag_i:
confirmation_b_tag = discussion_tag_i.find('b')
if confirmation_b_tag:
confirmation = confirmation_b_tag.text.strip()
# Split deletion_discussion into discussion and verdict
parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
discussion = parts[0] if len(parts) > 0 else ''
verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
data.append([title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
except Exception as e:
print(f"Error processing div: {e}")
continue
df = pd.DataFrame(data, columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'verdict', 'discussion'])
df = df[['title', 'discussion', 'verdict', 'label']]
print(f"DataFrame created with {len(df)} rows")
return df
def extract_div_contents_from_url_new(url):
response = requests.get(url)
if response.status_code != 200:
print(f"Error: Received status code {response.status_code} for URL: {url}")
return pd.DataFrame(columns=['date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
soup = BeautifulSoup(response.content, 'html.parser')
div_classes = ["mw-heading mw-heading3"]
divs = []
for div_class in div_classes:
divs.extend(soup.find_all('div', class_=div_class))
url_fragment = url.split('#')[-1].replace('_', ' ')
log_date = url.split('/')[-1]
data = []
for i, div in enumerate(divs):
try:
title = None
text_url = None
title_tag = div.find('a')
if title_tag:
title_span = div.find('span', {'data-mw-comment-start': True})
if title_span:
title_anchor = title_span.find_next_sibling('a')
if title_anchor:
title = title_anchor.text
text_url = 'https://en.wikipedia.org' + title_anchor['href']
else:
title = title_tag.text
text_url = 'https://en.wikipedia.org' + title_tag['href']
if title == 'talk page' or title is None:
heading_tag = div.find('div', class_='mw-heading mw-heading3')
if heading_tag:
title_tag = heading_tag.find('a')
if title_tag:
title = title_tag.text
text_url = 'https://en.wikipedia.org' + title_tag['href']
if not title:
continue
if title.lower() != url_fragment.lower():
continue
next_div = div.find_next('div', class_='mw-heading mw-heading3')
deletion_discussion = ''
sibling = div.find_next_sibling()
while sibling and sibling != next_div:
deletion_discussion += str(sibling)
sibling = sibling.find_next_sibling()
label = ''
verdict_tag = div.find('p')
if verdict_tag:
label_b_tag = verdict_tag.find('b')
if label_b_tag:
label = label_b_tag.text.strip()
confirmation = ''
discussion_tag = div.find('dd')
if discussion_tag:
discussion_tag_i = discussion_tag.find('i')
if discussion_tag_i:
confirmation_b_tag = discussion_tag_i.find('b')
if confirmation_b_tag:
confirmation = confirmation_b_tag.text.strip()
parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
discussion = parts[0] if len(parts) > 0 else ''
verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
data.append([ title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
except Exception as e:
print(f"Error processing div: {e}")
continue
df = pd.DataFrame(data, columns=[ 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
return df
def extract_post_links_text(discussion_html):
split_point = '<span class="plainlinks">'
if split_point in discussion_html:
parts = discussion_html.split(split_point)
if len(parts) > 1:
return parts[1]
return discussion_html
def process_discussion(df):
df['discussion_cleaned'] = df['verdict'].apply(extract_post_links_text)
return df
def html_to_plaintext(html_content):
soup = BeautifulSoup(html_content, 'html.parser')
for tag in soup.find_all(['p', 'li', 'dd', 'dl']):
tag.insert_before('\n')
tag.insert_after('\n')
for br in soup.find_all('br'):
br.replace_with('\n')
text = soup.get_text(separator=' ', strip=True)
text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
return text
def process_html_to_plaintext(df):
df['discussion_cleaned'] = df['discussion_cleaned'].apply(html_to_plaintext)
df = df[['title', 'discussion_cleaned', 'label']]
return df
import pysbd
def split_text_into_sentences(text):
seg = pysbd.Segmenter(language="en", clean=False)
sentences = seg.segment(text)
return ' '.join(sentences[1:])
def process_split_text_into_sentences(df):
df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
return df
def process_data(url):
df = extract_div_contents_from_url(url)
if df.at[0,'discussion'] == '':
df = extract_div_contents_from_url_new(url)
#print(df.head())
df = process_discussion(df)
print(df.at[0,'discussion'])
df = process_html_to_plaintext(df)
df = process_split_text_into_sentences(df)
if not df.empty:
return df.at[0,'title']+ ' : '+df.at[0, 'discussion_cleaned']
else:
return 'Empty DataFrame'
|