routers/bunnkatu/src/crawler.py

import requests
from bs4 import BeautifulSoup
from models.api import API

class Crawler:
    def __init__(self, url):
        self.url = url
        self.soup = self.get_soup()

    def get_soup(self):
        response = requests.get(self.url)
        return BeautifulSoup(response.text, 'html.parser')

    def crawl(self):
        api_list = []
        api_table = self.soup.find('table', {'class': 'markdown-table'})
        for row in api_table.find_all('tr')[1:]:
            cols = row.find_all('td')
            api = API(
                name=cols[0].text.strip(),
                description=cols[1].text.strip(),
                category=cols[2].text.strip(),
                link=cols[3].find('a')['href']
            )
            api_list.append(api)
        return api_list