Title: Using Pythons Selenium to scrape some website gives some errors
Tags: python,selenium-webdriver

Question: I have a slightly strange setup. I have two scripts in Python and R.
I was originally using R's RSelenium which worked but then stopped working so my original code was all in R - now I had to switch to Python and use undetected_chromedriver and a few other options that Selenium has available in Python. So I have 2 scripts.

The R script - uses rvest to process the web scraped data and sends via the command line the info to the Python script which runs the Selenium part.
The Python Selenium script which does the web scraping and I want to run the scripts via the terminal.

Problem:
When I set headless = True in Python I get all the errors. Here is my code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import random
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import ElementClickInterceptedException
import undetected_chromedriver as uc
import sys
import random
import pandas as pd

from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options

from selenium.webdriver.chrome.service import Service as ChromeService
from seleniumwire import webdriver
import re
import uuid
import datetime

from selenium.common.exceptions import TimeoutException



RUN_HEADLESS = True

PATH = '/home/matt/bin/chromedriver/chromedriver'

options = uc.ChromeOptions()
options.add_argument('--user-data-directory=/home/matt/.config/google-chrome-beta/Default')


if RUN_HEADLESS:
    options.add_argument("--headless=new") # ('--headless')
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    # #options.add_argument("--disable-gpu")  # If running on Linux/Unix system
    # options.add_argument("--disable-extensions")
    # options.add_argument("--start-maximized") 
    # # options.add_argument('--disable-javascript')
else:
    pass

How can I make the headless version work? when I set headless = False everything works but of course I have the browser open when scraping.
Error:
[7238:7238:0621/230217.106355:ERROR:process_singleton_posix.cc(334)] Failed to create /home/matt/.config/google-chrome/SingletonLock: El fichero ya existe (17)
MESA-INTEL: warning: Performance support disabled, consider sysctl dev.i915.perf_stream_paranoid=0

[0621/230217.125157:ERROR:nacl_helper_linux.cc(355)] NaCl helper process running without a sandbox!
Most likely you need to configure your SUID sandbox correctly
Traceback (most recent call last):
  File "/run/media/matt/A34E-C6B8/inmobiliarioProject2/rscripts/production/collectingTheData/v2/collectIndividualPages_Comprar.py", line 151, in <module>
    driver = uc.Chrome(
             ^^^^^^^^^^
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/undetected_chromedriver/__init__.py", line 466, in __init__
    super(Chrome, self).__init__(
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/chrome/webdriver.py", line 84, in __init__
    super().__init__(
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/chromium/webdriver.py", line 104, in __init__
    super().__init__(
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/remote/webdriver.py", line 286, in __init__
    self.start_session(capabilities, browser_profile)
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/undetected_chromedriver/__init__.py", line 729, in start_session
    super(selenium.webdriver.chrome.webdriver.WebDriver, self).start_session(
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/remote/webdriver.py", line 378, in start_session
    response = self.execute(Command.NEW_SESSION, parameters)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/remote/webdriver.py", line 440, in execute
    self.error_handler.check_response(response)
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/remote/errorhandler.py", line 245, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: cannot connect to chrome at 127.0.0.1:35111
from chrome not reachable
Stacktrace:
#0 0x561e4749c4e3 <unknown>
#1 0x561e471cbb00 <unknown>
#2 0x561e471b9436 <unknown>
#3 0x561e471f89be <unknown>
#4 0x561e471f0884 <unknown>
#5 0x561e4722fccc <unknown>
#6 0x561e4722f47f <unknown>
#7 0x561e47226de3 <unknown>
#8 0x561e471fc2dd <unknown>
#9 0x561e471fd34e <unknown>
#10 0x561e4745c3e4 <unknown>
#11 0x561e474603d7 <unknown>
#12 0x561e4746ab20 <unknown>
#13 0x561e47461023 <unknown>
#14 0x561e4742f1aa <unknown>
#15 0x561e474856b8 <unknown>
#16 0x561e47485847 <unknown>
#17 0x561e47495243 <unknown>
#18 0x7fa7187a16ea start_thread

EDIT:
using --headless instead of headless=new I get the following:
[1] "NA link. Skipping...."
[6782:6782:0622/000945.392409:ERROR:process_singleton_posix.cc(334)] Failed to create /home/matt/.config/google-chrome/SingletonLock: El fichero ya existe (17)
[0622/000945.407709:ERROR:nacl_helper_linux.cc(355)] NaCl helper process running without a sandbox!
Most likely you need to configure your SUID sandbox correctly
MESA-INTEL: warning: Performance support disabled, consider sysctl dev.i915.perf_stream_paranoid=0

Traceback (most recent call last):
  File "/run/media/matt/A34E-C6B8/inmobiliarioProject2/rscripts/production/collectingTheData/v2/collectIndividualPages_Comprar.py", line 151, in <module>
    driver = uc.Chrome(
             ^^^^^^^^^^
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/undetected_chromedriver/__init__.py", line 466, in __init__
    super(Chrome, self).__init__(
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/chrome/webdriver.py", line 84, in __init__
    super().__init__(
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/chromium/webdriver.py", line 104, in __init__
    super().__init__(
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/remote/webdriver.py", line 286, in __init__
    self.start_session(capabilities, browser_profile)
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/undetected_chromedriver/__init__.py", line 729, in start_session
    super(selenium.webdriver.chrome.webdriver.WebDriver, self).start_session(
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/remote/webdriver.py", line 378, in start_session
    response = self.execute(Command.NEW_SESSION, parameters)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/remote/webdriver.py", line 440, in execute
    self.error_handler.check_response(response)
  File "/home/matt/.asdf/installs/python/3.11.3/lib/python3.11/site-packages/selenium/webdriver/remote/errorhandler.py", line 245, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: cannot connect to chrome at 127.0.0.1:33765
from chrome not reachable
Stacktrace:
#0 0x56307a71f4e3 <unknown>
#1 0x56307a44eb00 <unknown>
#2 0x56307a43c436 <unknown>
#3 0x56307a47b9be <unknown>
#4 0x56307a473884 <unknown>
#5 0x56307a4b2ccc <unknown>
#6 0x56307a4b247f <unknown>
#7 0x56307a4a9de3 <unknown>
#8 0x56307a47f2dd <unknown>
#9 0x56307a48034e <unknown>
#10 0x56307a6df3e4 <unknown>
#11 0x56307a6e33d7 <unknown>
#12 0x56307a6edb20 <unknown>
#13 0x56307a6e4023 <unknown>
#14 0x56307a6b21aa <unknown>
#15 0x56307a7086b8 <unknown>
#16 0x56307a708847 <unknown>
#17 0x56307a718243 <unknown>
#18 0x7ff1875e96ea start_thread

EDIT: Code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import random
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import ElementClickInterceptedException
import undetected_chromedriver as uc
import sys
import random
import pandas as pd

from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options

from selenium.webdriver.chrome.service import Service as ChromeService
from seleniumwire import webdriver
import re
import uuid
import datetime

from selenium.common.exceptions import TimeoutException

import time
from seleniumbase import Driver




RUN_HEADLESS = False

PATH = '/home/matt/bin/chromedriver/chromedriver'

options = uc.ChromeOptions()
options.add_argument('--user-data-directory=/home/matt/.config/google-chrome-beta/Default')

################ New code added to fix chromedriver and chrome version mismatch ################
#options = Options()
if RUN_HEADLESS:
    options.add_argument("--headless") # ('--headless')
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    # #options.add_argument("--disable-gpu")  # If running on Linux/Unix system
    # options.add_argument("--disable-extensions")
    # options.add_argument("--start-maximized") 
    # # options.add_argument('--disable-javascript')
else:
    pass
  
  
driver = uc.Chrome(options=options)
  #seleniumwire_options = seleniumwire_options # here is where I pass random proxies to the options
  
  
link = "https://www.fotocasa.es/es/comprar/vivienda/madrid-capital/calefaccion-terraza-trastero-ascensor-no-amueblado/176698848/d?from=list"
driver.get(link)

This gives me:
TypeError: WebDriver.__init__() got an unexpected keyword argument 'executable_path'

Answer: Have you tried using --headless instead of --headless=new?