| | import requests |
| | import json |
| | import wikitextparser as wtp |
| | from lxml import etree |
| | from lxml import html |
| |
|
| | |
| | |
| |
|
| | class MyWikiAPI: |
| | WIKI_BASE_URL = "https://en.wikipedia.org/w/api.php" |
| | WIKI_FEATURED_URL = "https://en.wikipedia.org/wiki/Wikipedia:Featured_article_candidates/Featured_log" |
| | user_agent_headers = {"user-Agent": |
| | "AgentsCourseAssignment/1.0 (https://huggingface.co/spaces/krzsam/Agents-Course-Assignment)"} |
| |
|
| | def __init__(self): |
| | print(f"***KS*** Initializing Wiki API") |
| |
|
| | def __find_section_on_page__(self, page_title, section_title): |
| | response = requests.get( |
| | self.WIKI_BASE_URL, |
| | headers=self.user_agent_headers, |
| | params={ |
| | 'action': 'parse', |
| | 'format': 'json', |
| | 'page': page_title, |
| | 'prop': 'sections', |
| | }).json() |
| | sections = response['parse']['sections'] |
| | section_ix_found = None |
| | for sec in sections: |
| | if sec["line"] == section_title: |
| | section_ix_found = sec["index"] |
| | return section_ix_found |
| |
|
| | def __get_page_section_content__(self, page_title, section_id, format="wikitext"): |
| | response = requests.get( |
| | self.WIKI_BASE_URL, |
| | headers=self.user_agent_headers, |
| | params={ |
| | 'action': 'parse', |
| | 'format': 'json', |
| | 'page': page_title, |
| | 'prop': format, |
| | 'section': section_id |
| | }).json() |
| | return response["parse"][format]["*"] |
| |
|
| | def __get_featured_log__(self, month, year): |
| | featured_url = f"{self.WIKI_FEATURED_URL}/{month}_{year}" |
| | print(f"Getting content for: {featured_url}") |
| | response = requests.get( |
| | featured_url, |
| | headers=self.user_agent_headers, |
| | ).text |
| | return response |
| |
|
| | def __process_featured_log__(self, html_content): |
| | tree = html.fromstring(html_content) |
| | article_heading = "mw-heading mw-heading3" |
| | elements = tree.xpath(f"//div[@class='{article_heading}']") |
| | element_texts = [] |
| | for element in elements: |
| | n1 = element.getnext() |
| | n1_text = " ".join(n1.itertext()) |
| |
|
| | n2 = n1.getnext() |
| | n2_text = " ".join(n2.itertext()) |
| | element_text = f"{n1_text} {n2_text}" |
| |
|
| | element_texts.append(element_text) |
| |
|
| | return element_texts |
| |
|
| | def __is_int__(self, s): |
| | try: |
| | int(s) |
| | except ValueError: |
| | return False |
| | else: |
| | return True |
| |
|
| | def get_category(self, category, year): |
| | ret = self.__get_category_pages__(category, year) |
| | print(f"Got category: {category}\n{ret}") |
| |
|
| | def get_page_section(self, page_title, section_title): |
| | section_id = self.__find_section_on_page__(page_title, section_title) |
| | _ret = "" |
| | if section_id is not None: |
| | _ret = self.__get_page_section_content__(page_title, section_id) |
| | |
| | return _ret |
| |
|
| | def filter_section_and_table(self, section_content, sub_section_name, year_start, year_end): |
| | parsed = wtp.parse(section_content) |
| | sections = parsed.sections |
| | section_found = None |
| | for sec in sections: |
| | if sec.title is not None and sec.title.find(sub_section_name) >= 0: |
| | section_found = sec |
| |
|
| | print(f"Found matching subsection: {section_found.title}") |
| |
|
| | rows_collected = [] |
| | if section_found is not None and section_found.tables is not None and len(section_found.tables) > 0: |
| | table_data = section_found.tables[0].data() |
| | for row in table_data: |
| | if self.__is_int__(row[0]) and year_start <= int(row[0]) <= year_end: |
| | rows_collected.append(row) |
| |
|
| | return rows_collected |
| |
|
| | def get_featured_articles(self, month, year): |
| | |
| | |
| | full_html = self.__get_featured_log__(month, year) |
| |
|
| | ret = self.__process_featured_log__(full_html) |
| |
|
| | |
| | return ret |
