|
import csv
|
|
from selenium import webdriver
|
|
from selenium.webdriver.common.by import By
|
|
import time
|
|
|
|
|
|
driver = webdriver.Chrome()
|
|
|
|
|
|
url1 = "https://www.ynhhs.org/find-a-doctor#q=psychiatry&sort=relevancy&numberOfResults=100&f:affiliations=[YaleMedNortheast]"
|
|
url2 = "https://www.ynhhs.org/find-a-doctor#q=psychiatry&first=100&sort=relevancy&numberOfResults=100&f:affiliations=[YaleMedNortheast]"
|
|
|
|
|
|
def scrape_doctors(url):
|
|
driver.get(url)
|
|
time.sleep(5)
|
|
doctor_cards = driver.find_elements(By.CLASS_NAME, "doctor-card")
|
|
doctors_data = []
|
|
for doctor_card in doctor_cards:
|
|
try:
|
|
name = doctor_card.find_element(By.CLASS_NAME, "info-container").text.strip()
|
|
doctors_data.append([name])
|
|
except Exception as e:
|
|
print(f"Error processing card: {e}")
|
|
return doctors_data
|
|
|
|
|
|
doctors_info = scrape_doctors(url1)
|
|
|
|
|
|
doctors_info.extend(scrape_doctors(url2))
|
|
|
|
|
|
with open('doctors_info.csv', mode='w', newline='', encoding='utf-8') as file:
|
|
writer = csv.writer(file)
|
|
writer.writerow(['Name'])
|
|
writer.writerows(doctors_info)
|
|
|
|
|
|
driver.quit() |