netkeibaの出馬表から過去出走レースのデータを引っ張ってみた

netkeibaの出馬表から各馬のページに遷移し
データを見比べるのが面倒なため、予想に必要な情報をcsvで出力するためのプログラムを組んでみた
書き方が汚いので改良の余地はありそう


    import pandas as pd
    import urllib
    import requests
    import re
    import unicodedata
    import csv

    from webdriver_manager.chrome import ChromeDriverManager
    from selenium import webdriver
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import Select
    from bs4 import BeautifulSoup
    from tqdm import tqdm

    def main():

        site_url = input("データを取得したい出馬表のURLを入力してください(中央競馬のレースのみ対応):")
        browser = webdriver.Chrome(ChromeDriverManager().install())

        browser.get(site_url)

        browser.implicitly_wait(10) 
        link_list = []              

        link_list = make_horseURL(browser, link_list)

        df_RaceResult = pd.DataFrame()

        for url in tqdm(link_list):
        df_RaceResult = pd.concat([df_RaceResult, get_result(url)], axis=0)

        df_RaceResult.to_csv('./中間.csv', encoding='cp932', header=False, index=False, errors="ignore")

        file = open('./中間.csv',"r")
        out_file = open('./最終.csv',"w")

        lines = file.readlines()
        for line1 in lines:
            line = line1.split(',')
            if line[6] != "":
                line.pop(12)
                line.pop(16)
                line.pop(16)
            weight = line[21]
            if weight != "":
                horse_weight = re.split('[()]', weight)
                line.pop(21)
                line = line + horse_weight

            elif line[6] == "":
                line.pop(6)
                line.pop(6)
                line.pop(13)

            race_name = line[5]
            if '(' in race_name:
                grade_name = re.split('[()]', race_name)
                grade = grade_name[-2]
                line.insert(6,grade)
            else:
                line.insert(6,"")

            race_track2 = line[2]
            if "札幌" in race_track2:
                race_track = "札幌"
                line.pop(2)
                line.insert(2,race_track)
            elif "函館" in race_track2:
                race_track = "函館"
                line.pop(2)
                line.insert(2,race_track)
            elif "新潟" in race_track2:
                race_track = "新潟"
                line.pop(2)
                line.insert(2,race_track)
            elif "福島" in race_track2:
                race_track = "福島"
                line.pop(2)
                line.insert(2,race_track)
            elif "中山" in race_track2:
                race_track = "中山"
                line.pop(2)
                line.insert(2,race_track)
            elif "東京" in race_track2:
                race_track = "東京"
                line.pop(2)
                line.insert(2,race_track)
            elif "中京" in race_track2:
                race_track = "中京"
                line.pop(2)
                line.insert(2,race_track)
            elif "京都" in race_track2:
                race_track = "京都"
                line.pop(2)
                line.insert(2,race_track)
            elif "阪神" in race_track2:
                race_track = "阪神"
                line.pop(2)
                line.insert(2,race_track)
            elif "小倉" in race_track2:
                race_track = "小倉"
                line.pop(2)
                line.insert(2,race_track)


            result = ','.join(str(s) for s in line)
            out_file.write(result)

    def make_horseURL(browser, link_list):
        print('取得したHTMLからレース結果のURLを抽出します')
        html = browser.page_source.encode('utf-8')      
        soup = BeautifulSoup(html, 'html.parser')       
        table_data = soup.find(class_='Shutuba_Table')  

        for element in table_data.find_all('a'):
            url = element.get('href') 

            link_url = urllib.parse.urljoin('https://db.netkeiba.com', url)

            word_list = ['jockey', 'trainer', 'popup', 'bookmark']

            tmp_list = link_url.split('/') 
            and_list = set(word_list) & set(tmp_list) 

            if len(and_list) == 0:
                link_list.append(link_url)

        return link_list


    def get_result(url):
        res = requests.get(url)
        soup = BeautifulSoup(res.content, 'lxml')

        title = (soup.title.string.split(' '))
        horse_name = title[0]

        tables = soup.find('table', attrs={'class': 'db_h_race_results'})
        tables = tables.find_all('tr')

        indexs = tables[0].text.split('\n')

        tmp = []
        df = pd.DataFrame()
        df_tmp1 = pd.DataFrame()

        for table in tables[1:]:
            tmp = table.text.split('\n')
            df_tmp1 = pd.Series(tmp)
            df = pd.concat([df, df_tmp1], axis=1)

        df_tmp2 = pd.DataFrame()
        df_tmp3 = pd.DataFrame()
        for i in ( 1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 26, 27, 33, 34, 35, 36,):
            df_tmp2 = df.iloc[i]
            df_tmp3['馬名'] = horse_name
            df_tmp3 = pd.concat([df_tmp3, df_tmp2], axis=1)

        return df_tmp3

    if __name__ == "__main__":
        main()