Question: I've been trying to download 300 linked files from SEC's AAER site. Most of the links are pdf's, but some are websites that I would need to save to pdf instead of just downloading. I'm teaching myself some python web scraping and this didn't seem like too hard a task, but I havent been able to get past the 403 error when downloading. This code is working fine to scrape the links to the files and the 4 digit code I would like to name the files: from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.chrome.options import Options from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import time import os import requests # Set up Chrome options to allow direct PDF download (for the download step) download_path = "C:/Users/taylo/Downloads/sec_aaer_downloads" chrome_options = Options() chrome_options.add_experimental_option("prefs", { "download.default_directory": download_path, # Specify your preferred download directory "download.prompt_for_download": False, # Disable download prompt "plugins.always_open_pdf_externally": True, # Automatically open PDF in browser "safebrowsing.enabled": False, # Disable Chrome’s safe browsing check that can block downloads "profile.default_content_settings.popups": 0 # Disable popups }) # Set up the webdriver with options driver = webdriver.Chrome(executable_path="C:/chromedriver/chromedriver", options=chrome_options) # URLs for pages 1, 2, and 3 urls = [ "https://www.sec.gov/enforcement-litigation/accounting-auditing-enforcement-releases?page=0", "https://www.sec.gov/enforcement-litigation/accounting-auditing-enforcement-releases?page=1", "https://www.sec.gov/enforcement-litigation/accounting-auditing-enforcement-releases?page=2" ] # Initialize an empty list to store the URLs and AAER numbers pdf_data = [] # Loop through each URL (pages 1, 2, and 3) for url in urls: print(f"Scraping URL: {url}...") driver.get(url) # Wait for the table rows containing links to be loaded WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, '//*[@id="block-uswds-sec-content"]/div/div/div[3]/div/table/tbody/tr[1]'))) # Extract the link and AAER number from each row on the current page rows = driver.find_elements(By.XPATH, '//*[@id="block-uswds-sec-content"]/div/div/div[3]/div/table/tbody/tr') for row in rows: try: # Extract the link from the first column (PDF link) link_element = row.find_element(By.XPATH, './/td[2]/div[1]/a') link_href = link_element.get_attribute('href') # Extract the AAER number from the second column aaer_text_element = row.find_element(By.XPATH, './/td[2]/div[2]/span[2]') aaer_text = aaer_text_element.text aaer_number = aaer_text.split("AAER-")[1].split()[0] # Extract the number after AAER- # Store the data in a list of dictionaries pdf_data.append({'link': link_href, 'aaer_number': aaer_number}) except Exception as e: print(f"Error extracting data from row: {e}") # Print the scraped data (optional for verification) for entry in pdf_data: print(f"Link: {entry['link']}, AAER Number: {entry['aaer_number']}")  But when I try to do something like this, I can't get the downloads to go through: import os import time import requests # Set the download path download_path = "C:/Users/taylo/Downloads/sec_aaer_downloads" os.makedirs(download_path, exist_ok=True) # Loop through each entry in the pdf_data list for entry in pdf_data: try: # Extract the PDF link and AAER number link_href = entry['link'] aaer_number = entry['aaer_number'] # Send a GET request to download the PDF pdf_response = requests.get(link_href, stream=True, headers={ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" }) # Check if the request was successful if pdf_response.status_code == 200: # Save the PDF to the download folder, using the AAER number as the filename pdf_file_path = os.path.join(download_path, f"{aaer_number}.pdf") with open(pdf_file_path, "wb") as pdf_file: for chunk in pdf_response.iter_content(chunk_size=8192): pdf_file.write(chunk) print(f"Downloaded: {aaer_number}.pdf") else: print(f"Failed to download the file from {link_href}, status code: {pdf_response.status_code}") except Exception as e: print(f"Error downloading the PDF for AAER {aaer_number}: {e}")  At this point it would have been faster to manually download the files but I want to know what I'm doing wrong. I've tried Setting User-Agent Header and Simulating User Click with Selenium. Thanks for any advice you may have!

Answer: After copying all the Headers inside the request header when you manually open the link containing the PDF:   pdf_response = requests.get(link_href, headers={ "Host": "www.sec.gov", "User-Agent": "YOUR_USER_AGENT", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br, zstd", "Connection": "keep-alive", "Cookie": "YOUR_COOKIE", "Upgrade-Insecure-Requests": "1", "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Sec-Fetch-User": "?1", "Priority": "u=0, i", "Pragma": "no-cache", })  I was able to download the files:  You also need to remove the stream=true argument inside the requests. These answers why Status Code 403 Forbidden is occurs, you need all the headers to access the URLs. Hope this helps!