Title: WebDriverException: unhandled inspector error - No node with given id found at a specific iteration point
Tags: python,selenium-webdriver,web-scraping,selenium-chromedriver

Question: I have written a Python script using Selenium and ChromeDriver to scrape data. The script navigates through several pages and clicks on various buttons to retrieve the data. However, I am encountering the following error:
WebDriverException: Message: unknown error: unhandled inspector error: {"code":-32000,"message":"No node with given id found"}

The error seems to occur at a specific point in the iteration, rather than being random. I have tried to troubleshoot the issue, but I am not sure what is causing it or how to fix it.
I am using Python 3.10.5 and the Selenium library with ChromeDriver version 113.0.5672.63 on a Windows 10 machine. Any help with resolving this issue would be greatly appreciated.
I'm still a beginner and this is my first time trying selenium. I have tried adding time.sleep(1) to make sure the web is loaded, check the visibility of the element, and the element is clickable but the problem still occurs.
This is the current script that I have written
url = '.../'
path = Service(r'...\chromedriver_win32')

options = Options()
options.add_experimental_option("debuggerAddress", "localhost:9222")
driver = webdriver.Chrome(service=path, options=options)
driver.get(url)
wait = WebDriverWait(driver, 10)

def scrape_left_table(prob, kab, kec):
    data = [] 
    rows = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(1) > table > tbody > tr')
    for row in rows:
        wilayah = row.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > button').text
        persentasi = row.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > span').text
        class_1= row.find_element(By.CSS_SELECTOR, 'td:nth-child(2)').text
        class_2= row.find_element(By.CSS_SELECTOR, 'td:nth-child(3)').text

        data.append([prob, kab, kec, wilayah, persentasi, class_1, class_2])
    
    return data

def scrape_right_table(prob, kab, kec):
    data = [] 
    rows = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(2) > table > tbody > tr')
    for row in rows:
        wilayah = row.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > button').text
        persentasi = row.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > span').text
        class_1= row.find_element(By.CSS_SELECTOR, 'td:nth-child(2)').text
        class_2= row.find_element(By.CSS_SELECTOR, 'td:nth-child(3)').text

        data.append([prob, kab, kec, wilayah, persentasi, class_1, class_2])
    
    return data

data = []
 
provinsi = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(1) > table > tbody > tr')
button = provinsi[1].find_element(By.TAG_NAME, 'button')
pro = button.text
wait.until(EC.element_to_be_clickable(button)).click()

wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div:nth-child(1) > table > tbody > tr')))
for i in [1,2]:
    time.sleep(1)
    kabupaten = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')
    for kab in kabupaten:
        time.sleep(1)
        wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')))
        kab_button = kab.find_element(By.TAG_NAME, 'button')
        kab_name = kab_button.text
        driver.execute_script("arguments[0].scrollIntoView();", kab_button)
        driver.execute_script("arguments[0].click();", kab_button)

        for i in [1,2]:
            time.sleep(1)
            kecamatan = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')
            for kec in kecamatan:
                wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')))

                kec_button = kec.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > button')
                kec_name = kec_button.text
                driver.execute_script("arguments[0].scrollIntoView();", kec_button)
                driver.execute_script("arguments[0].click();", kec_button)

                kelurahan = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(1) > table > tbody > tr')
                time.sleep(1)
                left_table = scrape_left_table(pro, kab_name, kec_name)
                right_table = scrape_right_table(pro, kab_name, kec_name)
                data += left_table + right_table

                back = driver.find_element(By.CSS_SELECTOR, '#app > div.sticky-top.bg-white > div > div:nth-child(2) > div > div > div > div:nth-child(5) > div > div > div.vs__actions > button')
                driver.execute_script("arguments[0].scrollIntoView();", back)
                driver.execute_script("arguments[0].click();", back)
                
        back = driver.find_element(By.CSS_SELECTOR, '#app > div.sticky-top.bg-white > div > div:nth-child(2) > div > div > div > div:nth-child(4) > div > div > div.vs__actions > button')
        driver.execute_script("arguments[0].scrollIntoView();", back)
        driver.execute_script("arguments[0].click();", back)

After a certain iteration i.e. for provinsi[0] errors occur after 689 iterations for provinsi[1] errors occur after 35 iterations.
WebDriverException                        Traceback (most recent call last)
c:\...\web_scraping.ipynb Cell 4 in ()
     23 for kec in kecamatan:
     24     wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')))
---> 26     kec_button = kec.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > button')
     27     kec_name = kec_button.text
     28     driver.execute_script("arguments[0].scrollIntoView();", kec_button)

WebDriverException: Message: unknown error: unhandled inspector error: {"code":-32000,"message":"No node with given id found"}

Answer: This happens only in certain DOM elements like .gif and alert messages from a different iframe. Using driver.switchTo().defaultContent(); after inspecting the element helped.