I have an issue with the EGazette scraper. When we click the download option on the website, it redirects us to a new page where we have to manually click the download icon. This is a Chrome feature that Selenium can't handle automatically. To ensure the PDFs download automatically without opening another page, I set some options like "plugins.always_open_pdf_externally": True. However, even after passing all the arguments, It still redirects to a new page and the download is not starting.

# %%
%pip install selenium webdriver-manager
#Download ChromeDriver Before running
#download this for win64 -> https://storage.googleapis.com/chrome-for-testing-public/129.0.6668.89/win64/chromedriver-win64.zip
#download this for mac -> https://storage.googleapis.com/chrome-for-testing-public/129.0.6668.89/mac-x64/chromedriver-mac-x64.zip

# %%
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service as ChromeService
from webdriver_manager.chrome import ChromeDriverManager
import time 

# %%
options = webdriver.ChromeOptions()
options.add_experimental_option('prefs', {
   "download.default_directory": "F:/Lamarr/egazette/docs", #Destination directory path
    "download.prompt_for_download": False,
    "download.directory_upgrade": True,
    "safebrowsing.enabled": True,
    "plugins.always_open_pdf_externally": True,
    "profile.default_content_setting_values.automatic_downloads": 1,
    "profile.default_content_setting_values.popups": 0,
    "plugins.plugins_disabled": ["Chrome PDF Viewer"]
})
# chrome_driver_path = "F:/Lamarr/egazette/chromedriver-win64/chromedriver-win64/chromedriver.exe" #ChromeDriver Path
driver = webdriver.Chrome(options=options)
# driver = webdriver.Chrome(service=ChromeService(chrome_driver_path), options=options)
# driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=options) This isn't working it states incompatibility with chrome versiona and driver version
url = "https://egazette.gov.in"
driver.get(url)
driver.maximize_window()
driver.implicitly_wait(10)

cookie_div = driver.find_element(By.XPATH, '//*[@id="ImgMessage_OK"]')
if cookie_div:
    cookie_div.click()
    print("Button clicked, waiting for redirection...")
    time.sleep(5)
    print("Waited for 5 seconds after redirection")
else:
  print("No cookie")

print("Home page")

try:
    cross_button = driver.find_element(By.XPATH, '//*[@id="Div_ImageScroller"]/div[1]/a')
    if cross_button:
        cross_button.click()
        print("Cross button clicked.")        
        time.sleep(5)
        
except NoSuchElementException:
    print("Cross button not found, proceeding with search.")
    
try:
    search_button = driver.find_element(By.XPATH, '//*[@id="sgzt"]')
    search_button.click()
    print("Search button clicked, waiting for redirection...")
    time.sleep(5)
    page_source = driver.page_source
    print("Page source retrieved after second redirection")
    # print(page_source)

except Exception as e:
    print(f"An error occurred: {e}")

print("Search page")
try:
    search_button = driver.find_element(By.XPATH, '//*[@id="btnPublish"]')
    search_button.click()
    print("Search by Publish button clicked, waiting for redirection...")
    time.sleep(5)
    page_source = driver.page_source
    print("Page source retrieved after second redirection")
    # print(page_source)

except Exception as e:
    print(f"An error occurred: {e}")

print("Search by publish date")
try:
    from_date = driver.find_element(By.XPATH, '//*[@id="txtDateFrom"]')
    from_date.click()
    from_date.clear()
    from_date.send_keys("01-Oct-2024")

    to_date = driver.find_element(By.XPATH, '//*[@id="txtDateTo"]')
    to_date.click()
    to_date.clear()
    to_date.send_keys("05-Oct-2024")

    print("Dates input successfully.")

    submit_button = driver.find_element(By.XPATH, '//*[@id="ImgSubmit"]')
    submit_button.click()
    print("Submit button clicked, waiting for redirection...")

    time.sleep(5)

    page_source = driver.page_source
    print("Page source retrieved after date search redirection")
    # print(page_source)

except Exception as e:
    print(f"An error occurred: {e}")

try:
    total_gazettes_element = driver.find_element(By.XPATH, '//*[@id="lbl_Result"]')
    total_gazettes_text = total_gazettes_element.text
    print(f"Total gazettes text: {total_gazettes_text}")

    n = int(total_gazettes_text.split(":")[1].strip())
    print(f"Total number of gazettes: {n}")

    print("Downloading the pdfs")
    
    for i in range(7):   #randomly assigned number for testing first
            try:
                download_button_xpath = f'//*[@id="gvGazetteList_imgbtndownload_{i}"]'
                download_button = driver.find_element(By.XPATH, download_button_xpath)
                download_button.click()
                print(f"Clicked to download gazette number {i + 1}")
                # Switch to the new tab if it opens a new window
                # windows = driver.window_handles
                # if len(windows) > 1:
                #     driver.switch_to.window(windows[1])  
                #     print("Switched to new tab for download.")
                #     try:
                #         open_button = driver.find_element(By.ID, "open-button")
                #         open_button.click()
                #         print("Clicked on Open button.")
                #         time.sleep(5)  
                #     except Exception as e:
                #         print("Error clicking on Open button:", e)

                #     time.sleep(5)
                #     driver.close()  
                #     driver.switch_to.window(windows[0])  # Switch back to the main window
    
            except Exception as download_exception:
                print(f"Error downloading gazette number {i + 1}: {download_exception}")
except Exception as e:
    print(f"An error occurred: {e}")
print("Download process completed.")