Spaces:
Runtime error
Runtime error
noelfranthomas
commited on
Commit
•
01ef47d
1
Parent(s):
e9ebb6e
NLP
Browse files- app.py +20 -5
- web_scraper.py +190 -0
app.py
CHANGED
@@ -1,9 +1,24 @@
|
|
1 |
import gradio as gr
|
2 |
|
3 |
-
from web_scraper import minutes_scraper
|
4 |
|
5 |
-
|
6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
|
8 |
-
|
9 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
import gradio as gr
|
2 |
|
3 |
+
from web_scraper import minutes_scraper, minutes_scraper_no_sum
|
4 |
|
5 |
+
with gr.Blocks() as app:
|
6 |
+
gr.Markdown("Get the meeting minutes from a URL")
|
7 |
+
with gr.Tab("Without Summary"):
|
8 |
+
text_input = gr.Textbox()
|
9 |
+
text_output = gr.JSON()
|
10 |
+
text_button = gr.Button("Scrape")
|
11 |
+
with gr.Tab("With Summary (Slower)"):
|
12 |
+
text_input = gr.Textbox()
|
13 |
+
text_output = gr.JSON()
|
14 |
+
text_button = gr.Button("Scrape & Summarize")
|
15 |
|
16 |
+
with gr.Accordion("Note on Summary"):
|
17 |
+
gr.Markdown("The summary is generated using the [Facebook BART model](https://huggingface.co/facebook/bart-large-cnn). The summary is not perfect, but it is a good starting point for a quick overview of the meeting. Please bear in mind that this process may take longer depending on the amount of text to summarize.")
|
18 |
+
|
19 |
+
|
20 |
+
|
21 |
+
text_button.click(minutes_scraper, inputs=text_input, outputs=text_output)
|
22 |
+
image_button.click(minutes_scraper_no_sum(), inputs=image_input, outputs=image_output)
|
23 |
+
|
24 |
+
app.launch()
|
web_scraper.py
CHANGED
@@ -204,6 +204,196 @@ def minutes_scraper(URL=""):
|
|
204 |
|
205 |
|
206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
207 |
# # Serialize and write to "meeting_minutes.json"
|
208 |
# with open(f"{out_dir}/meeting_minutes.json", "w") as out:
|
209 |
# json.dump(JSON_obj, out, indent=4)
|
|
|
204 |
|
205 |
|
206 |
|
207 |
+
# # Serialize and write to "meeting_minutes.json"
|
208 |
+
# with open(f"{out_dir}/meeting_minutes.json", "w") as out:
|
209 |
+
# json.dump(JSON_obj, out, indent=4)
|
210 |
+
|
211 |
+
# Add this to data base
|
212 |
+
|
213 |
+
return JSON_obj
|
214 |
+
|
215 |
+
def minutes_scraper_no_sum(URL=""):
|
216 |
+
if not isValidURL(URL):
|
217 |
+
print("Invalid or missing URL input")
|
218 |
+
print("Please enter a URL now:")
|
219 |
+
|
220 |
+
return "Invalid URL"
|
221 |
+
|
222 |
+
# Get output directory
|
223 |
+
out_dir = ""
|
224 |
+
out_dir = os.getcwd()
|
225 |
+
|
226 |
+
###
|
227 |
+
|
228 |
+
s = summarizer() # Summarizer object
|
229 |
+
|
230 |
+
# Object to be seriliazed
|
231 |
+
JSON_obj = {}
|
232 |
+
|
233 |
+
# Get meeting ID
|
234 |
+
page = requests.get(URL)
|
235 |
+
o = urlparse(URL)
|
236 |
+
query = parse_qs(o.query)
|
237 |
+
|
238 |
+
JSON_obj["meeting_id"] = query["Id"][0]
|
239 |
+
|
240 |
+
# Complete HTML File
|
241 |
+
soup = BeautifulSoup(page.content, "html.parser")
|
242 |
+
|
243 |
+
# Most of the page content is found in this container
|
244 |
+
page_content = soup.find(id="package-container")
|
245 |
+
|
246 |
+
###
|
247 |
+
|
248 |
+
# MM Header
|
249 |
+
agenda_header = page_content.find("header", class_="AgendaHeader")
|
250 |
+
|
251 |
+
## Header information
|
252 |
+
|
253 |
+
# Get the Agenda Header
|
254 |
+
try:
|
255 |
+
JSON_obj["agenda_header_subtitle"] = agenda_header.find("p", class_="AgendaHeaderSubTitle").text
|
256 |
+
except AttributeError:
|
257 |
+
JSON_obj["agenda_header_subtitle"] = ""
|
258 |
+
|
259 |
+
|
260 |
+
# Get the start time
|
261 |
+
JSON_obj["start_time"] = agenda_header.find("time").text
|
262 |
+
|
263 |
+
# Get the location
|
264 |
+
try:
|
265 |
+
JSON_obj["location"] = agenda_header_subtitle = agenda_header.find("div", class_="Value LocationValue").text ### This does not get all location info
|
266 |
+
except AttributeError:
|
267 |
+
JSON_obj["location"] = ""
|
268 |
+
|
269 |
+
# Get the attendence (seperated by who can and can't vote)
|
270 |
+
attendance_table = agenda_header.find(class_="AgendaHeaderAttendanceTable").find_all("div")
|
271 |
+
try:
|
272 |
+
present = [x.text for x in attendance_table[2].find_all("li")]
|
273 |
+
except IndexError:
|
274 |
+
present = []
|
275 |
+
|
276 |
+
try:
|
277 |
+
also_present = [x.text for x in attendance_table[5].find_all("li")]
|
278 |
+
except IndexError:
|
279 |
+
also_present = []
|
280 |
+
|
281 |
+
JSON_obj["attendance"] = {'present': present, 'also_present': also_present}
|
282 |
+
|
283 |
+
###
|
284 |
+
|
285 |
+
# MM Body
|
286 |
+
agenda_items = page_content.find("div", class_="AgendaItems")
|
287 |
+
|
288 |
+
## Body information
|
289 |
+
|
290 |
+
# Get item containers
|
291 |
+
agenda_item_containers = agenda_items.find_all("div", class_="AgendaItemContainer indent")
|
292 |
+
|
293 |
+
# Get roll call
|
294 |
+
try:
|
295 |
+
roll_call = agenda_item_containers[0].find_all("p", class_="Body1")
|
296 |
+
JSON_obj["roll_call"] = roll_call[2].text.rstrip('.').replace(', and ', ', ').split(', ')
|
297 |
+
except IndexError:
|
298 |
+
JSON_obj["roll_call"] = []
|
299 |
+
|
300 |
+
if DEBUG:
|
301 |
+
print(JSON_obj["roll_call"])
|
302 |
+
|
303 |
+
# Get generator of item containers
|
304 |
+
agenda_item_containers = agenda_items.children
|
305 |
+
|
306 |
+
item_number = 1
|
307 |
+
for agenda_item in agenda_item_containers:
|
308 |
+
|
309 |
+
# Get titles
|
310 |
+
titles = [x.text for x in agenda_item.find_all("div", class_="AgendaItemTitle")]
|
311 |
+
|
312 |
+
# Get each motion in each item
|
313 |
+
motions = agenda_item.find_all("ul", class_="AgendaItemMotions")
|
314 |
+
|
315 |
+
if DEBUG:
|
316 |
+
print(item_number)
|
317 |
+
|
318 |
+
if motions != None:
|
319 |
+
item_sub_number = 1
|
320 |
+
|
321 |
+
for motion in motions:
|
322 |
+
|
323 |
+
# Dictionary to store all motion info
|
324 |
+
motion_obj = {}
|
325 |
+
|
326 |
+
if DEBUG:
|
327 |
+
print(str(item_number) + '.' + str(item_sub_number))
|
328 |
+
|
329 |
+
# Place "anchor"
|
330 |
+
motion_anchor = [x.parent.parent.parent.parent for x in motion.find_all("div", class_="MotionText RichText")]
|
331 |
+
|
332 |
+
# Get motion title
|
333 |
+
motion_titles = [x.find("div", class_="AgendaItemTitle").text.strip() for x in motion_anchor]
|
334 |
+
|
335 |
+
# Get list of who the motion is moved by
|
336 |
+
moved_by_list = [x.find("span", class_="Value") for x in motion.find_all("div", class_="MovedBy")]
|
337 |
+
moved_by_list = [x.text for x in moved_by_list]
|
338 |
+
|
339 |
+
# Get motion description
|
340 |
+
motion_description_list = [x.text for x in motion.find_all("div", class_="MotionText RichText")]
|
341 |
+
|
342 |
+
# Get motion result
|
343 |
+
motion_result_list = [x.text for x in motion.find_all("div", class_="MotionResult")]
|
344 |
+
|
345 |
+
# Get motion votes
|
346 |
+
motion_votes_list = [x.text[x.text.find(')') + 1:].split(', and ') for x in motion.find_all("table", class_="MotionVoters")]
|
347 |
+
|
348 |
+
# Get motion attachments
|
349 |
+
motion_attachments_list = [x.find_all("a", class_="Link") for x in motion_anchor]
|
350 |
+
motion_attachments_list_names = []
|
351 |
+
motion_attachments_list_links = []
|
352 |
+
for x in motion_attachments_list:
|
353 |
+
motion_attachments_list_names.append([y.text for y in x]) # ?
|
354 |
+
motion_attachments_list_links.append([y['href'] for y in x])
|
355 |
+
|
356 |
+
motion_obj["titles"] = motion_titles
|
357 |
+
motion_obj["moved_by"] = moved_by_list
|
358 |
+
motion_obj["details"] = motion_description_list
|
359 |
+
motion_obj["results"] = motion_result_list
|
360 |
+
motion_obj["votes"] = motion_votes_list
|
361 |
+
motion_obj['attachment_names'] = motion_attachments_list_names[0]
|
362 |
+
motion_obj['attachment_links'] = motion_attachments_list_links[0]
|
363 |
+
motion_obj['attachment_count'] = len(motion_attachments_list_names[0])
|
364 |
+
|
365 |
+
# for desc in motion_description_list:
|
366 |
+
# if len(desc.split()) > s.max_length:
|
367 |
+
# motion_obj['summary'] = s.summarize(text=desc)[0]
|
368 |
+
# else:
|
369 |
+
# motion_obj['summary'] = "Too short to summarize"
|
370 |
+
|
371 |
+
|
372 |
+
if DEBUG:
|
373 |
+
print(str(item_number) + '.' + str(item_sub_number))
|
374 |
+
print(motion_titles) # title
|
375 |
+
print("Moved by: " + str(moved_by_list)) # Moved by
|
376 |
+
print(motion_description_list) # Other details
|
377 |
+
print("Result: " + str(motion_result_list)) # Result
|
378 |
+
print("Votes: " + str(motion_votes_list)) # Votes
|
379 |
+
print(motion_attachments_list_names[0]) # attachment names
|
380 |
+
print(motion_attachments_list_links[0]) # attachment links
|
381 |
+
print()
|
382 |
+
|
383 |
+
# Append to JSON object
|
384 |
+
JSON_obj[f'{item_number}.{item_sub_number}'] = motion_obj
|
385 |
+
|
386 |
+
item_sub_number+=1
|
387 |
+
|
388 |
+
if DEBUG:
|
389 |
+
print('-----------------------------------\n\n\n')
|
390 |
+
|
391 |
+
item_number+=1
|
392 |
+
|
393 |
+
|
394 |
+
|
395 |
+
|
396 |
+
|
397 |
# # Serialize and write to "meeting_minutes.json"
|
398 |
# with open(f"{out_dir}/meeting_minutes.json", "w") as out:
|
399 |
# json.dump(JSON_obj, out, indent=4)
|