cpc_iri_web.py

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import pandas as pd
import streamlit as st

#st.set_page_config(layout='wide')

options = Options()
options.add_argument('--headless')
#@st.cache_resource
def get_driver():
    return webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

driver = get_driver()
driver.get("https://iri.columbia.edu/our-expertise/climate/forecasts/enso/current/?enso_tab=enso-sst_table")
page = driver.page_source
soup = BeautifulSoup(page, 'lxml')

#cabeçalho
header = soup.find('h4')
header = header.text.split(':')[1]

# table
table = soup.find('table')
df_table = pd.read_html(str(table))[0]
df_table = pd.DataFrame(df_table.to_records())
df_table = df_table[df_table.columns[1:]]
df_table.set_index('Season', inplace=True)

# Divs
div = soup.find('div', {"id": "ENSO-Plume-Models-Table"})
df = pd.read_html(str(div))[0]
df = pd.DataFrame(df.to_records())
df = df[df.columns[1:]]
colunas = [x.split(',')[1].split(')')[0] for x in df.columns]
colunas = [x.replace(' ', '').replace("'", "") for x in colunas]
df.columns = colunas

# modelos dinamicos
modelos_dinamicos = df[1:18]
modelos_dinamicos.set_index('Model', inplace=True)
modelos_dinamicos = modelos_dinamicos.astype(float)
modelos_dinamicos = modelos_dinamicos.T
modelos_dinamicos['Média'] = modelos_dinamicos.mean(axis=1)

# modelos estatisticos
modelos_estatisticos = df[20:36]
modelos_estatisticos.set_index('Model', inplace=True)
modelos_estatisticos = modelos_estatisticos.astype(float)
modelos_estatisticos = modelos_estatisticos.T
modelos_estatisticos['Média'] = modelos_estatisticos.mean(axis=1)