diff --git a/Dockerfile b/Dockerfile index 6bc561d4066..dc6a0586b9c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,34 @@ # SeleniumBase Docker Image FROM ubuntu:22.04 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +#====================== +# Locale Configuration +#====================== +RUN apt-get update +RUN apt-get install -y --no-install-recommends tzdata locales +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV TZ="America/New_York" + +#====================== +# Install Common Fonts +#====================== +RUN apt-get update +RUN apt-get install -y \ + fonts-liberation \ + fonts-open-sans \ + fonts-mononoki \ + fonts-roboto \ + fonts-lato #============================ # Install Linux Dependencies #============================ -RUN apt-get update && apt-get install -y \ - fonts-liberation \ +RUN apt-get update +RUN apt-get install -y \ libasound2 \ libatk-bridge2.0-0 \ libatk1.0-0 \ @@ -17,60 +40,57 @@ RUN apt-get update && apt-get install -y \ libgtk-3-0 \ libnspr4 \ libnss3 \ + libu2f-udev \ + libvulkan1 \ libwayland-client0 \ libxcomposite1 \ libxdamage1 \ libxfixes3 \ libxkbcommon0 \ - libxrandr2 \ - libu2f-udev \ - libvulkan1 \ - xdg-utils + libxrandr2 + +#========================== +# Install useful utilities +#========================== +RUN apt-get update +RUN apt-get install -y xdg-utils #================================= # Install Bash Command Line Tools #================================= +RUN apt-get update RUN apt-get -qy --no-install-recommends install \ curl \ sudo \ unzip \ vim \ wget \ - xvfb \ - && rm -rf /var/lib/apt/lists/* + xvfb #================ # Install Chrome #================ -RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -RUN apt-get install -y ./google-chrome-stable_current_amd64.deb +RUN apt-get update +RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +RUN dpkg -i google-chrome-stable_current_amd64.deb +RUN apt-get -fy --no-install-recommends install RUN rm google-chrome-stable_current_amd64.deb #================ # Install Python #================ -RUN apt-get update -y -RUN apt-get install -y python3 python3-pip python3-setuptools python3-dev +RUN apt-get update +RUN apt-get install -y python3 python3-pip python3-setuptools python3-dev python3-tk RUN alias python=python3 RUN echo "alias python=python3" >> ~/.bashrc RUN apt-get -qy --no-install-recommends install python3.10 RUN rm /usr/bin/python3 RUN ln -s python3.10 /usr/bin/python3 -#============================================= -# Allow Special Characters in Python Programs -#============================================= -RUN export PYTHONIOENCODING=utf8 -RUN echo "export PYTHONIOENCODING=utf8" >> ~/.bashrc - -#=========================== -# Configure Virtual Display -#=========================== -RUN set -e -RUN echo "Starting X virtual framebuffer (Xvfb) in background..." -RUN Xvfb -ac :99 -screen 0 1280x1024x16 > /dev/null 2>&1 & -RUN export DISPLAY=:99 -RUN exec "$@" +#=============== +# Cleanup Lists +#=============== +RUN rm -rf /var/lib/apt/lists/* #===================== # Set up SeleniumBase @@ -89,6 +109,7 @@ RUN find . -name '*.pyc' -delete RUN pip install --upgrade pip setuptools wheel RUN cd /SeleniumBase && ls && pip install -r requirements.txt --upgrade RUN cd /SeleniumBase && pip install . +RUN pip install pyautogui #======================= # Download chromedriver diff --git a/examples/raw_cdp_logging.py b/examples/raw_cdp_logging.py index 7a58f8a14e8..1f9c7659665 100644 --- a/examples/raw_cdp_logging.py +++ b/examples/raw_cdp_logging.py @@ -5,8 +5,7 @@ try: url = "seleniumbase.io/apps/turnstile" driver.uc_open_with_reconnect(url, 2) - driver.switch_to_frame("iframe") - driver.uc_click("span") + driver.uc_gui_handle_cf() driver.sleep(3) pprint(driver.get_log("performance")) finally: diff --git a/examples/raw_form_turnstile.py b/examples/raw_form_turnstile.py index a722534cccf..9fbd85464dc 100644 --- a/examples/raw_form_turnstile.py +++ b/examples/raw_form_turnstile.py @@ -2,7 +2,7 @@ with SB(uc=True, test=True) as sb: url = "seleniumbase.io/apps/form_turnstile" - sb.driver.uc_open_with_reconnect(url, 2) + sb.uc_open_with_reconnect(url, 2) sb.press_keys("#name", "SeleniumBase") sb.press_keys("#email", "test@test.test") sb.press_keys("#phone", "1-555-555-5555") @@ -12,8 +12,8 @@ sb.click('span:contains("9:00 PM")') sb.highlight_click('input[value="AR"] + span') sb.click('input[value="cc"] + span') - sb.switch_to_frame("iframe") - sb.driver.uc_click("span") + sb.scroll_to("iframe") + sb.uc_gui_handle_cf() sb.highlight("img#captcha-success", timeout=3) sb.highlight_click('button:contains("Request & Pay")') sb.highlight("img#submit-success") diff --git a/examples/raw_nopecha.py b/examples/raw_nopecha.py index bbe377fbdd5..17eab52cbc0 100644 --- a/examples/raw_nopecha.py +++ b/examples/raw_nopecha.py @@ -1,19 +1,11 @@ from seleniumbase import SB with SB(uc=True, test=True) as sb: - sb.driver.uc_open_with_reconnect("nopecha.com/demo/turnstile", 4) - if sb.is_element_visible("#example-container0 iframe"): - sb.switch_to_frame("#example-container0 iframe") - if not sb.is_element_visible("circle.success-circle"): - sb.driver.uc_click("span", reconnect_time=3) - sb.switch_to_frame("#example-container0 iframe") - sb.switch_to_default_content() - - sb.switch_to_frame("#example-container5 iframe") - sb.driver.uc_click("span", reconnect_time=2.5) - sb.switch_to_frame("#example-container5 iframe") - sb.assert_element("svg#success-icon", timeout=3) - sb.switch_to_parent_frame() + sb.uc_open_with_disconnect("nopecha.com/demo/turnstile", 3.5) + sb.uc_gui_press_keys("\t\t ") + sb.sleep(3.5) + sb.connect() + sb.uc_gui_handle_cf("#example-container5 iframe") if sb.is_element_visible("#example-container0 iframe"): sb.switch_to_frame("#example-container0 iframe") diff --git a/examples/raw_turnstile.py b/examples/raw_turnstile.py index ba2a172831c..3f45b83f33a 100644 --- a/examples/raw_turnstile.py +++ b/examples/raw_turnstile.py @@ -1,23 +1,9 @@ from seleniumbase import SB - -def open_the_turnstile_page(sb): +with SB(uc=True, test=True) as sb: url = "seleniumbase.io/apps/turnstile" - sb.driver.uc_open_with_reconnect(url, reconnect_time=2) - - -def click_turnstile_and_verify(sb): - sb.driver.switch_to_frame("iframe") - sb.driver.uc_click("span") + sb.uc_open_with_reconnect(url, reconnect_time=2) + sb.uc_gui_handle_cf() sb.assert_element("img#captcha-success", timeout=3) - - -with SB(uc=True, test=True) as sb: - open_the_turnstile_page(sb) - try: - click_turnstile_and_verify(sb) - except Exception: - open_the_turnstile_page(sb) - click_turnstile_and_verify(sb) sb.set_messenger_theme(location="top_left") sb.post_message("SeleniumBase wasn't detected", duration=3) diff --git a/examples/uc_cdp_events.py b/examples/uc_cdp_events.py index 51283d8cc58..d869c7513fc 100644 --- a/examples/uc_cdp_events.py +++ b/examples/uc_cdp_events.py @@ -13,8 +13,7 @@ def add_cdp_listener(self): ) def click_turnstile_and_verify(sb): - sb.switch_to_frame("iframe") - sb.driver.uc_click("span") + sb.uc_gui_handle_cf() sb.assert_element("img#captcha-success", timeout=3) sb.highlight("img#captcha-success", loops=8) @@ -22,7 +21,7 @@ def test_display_cdp_events(self): if not (self.undetectable and self.uc_cdp_events): self.get_new_driver(undetectable=True, uc_cdp_events=True) url = "seleniumbase.io/apps/turnstile" - self.driver.uc_open_with_reconnect(url, 2) + self.uc_open_with_reconnect(url, 2) self.add_cdp_listener() self.click_turnstile_and_verify() self.sleep(1) diff --git a/help_docs/uc_mode.md b/help_docs/uc_mode.md index 7931f38456f..b89ba282f83 100644 --- a/help_docs/uc_mode.md +++ b/help_docs/uc_mode.md @@ -20,7 +20,7 @@ * Automatically changing user agents to prevent detection. * Automatically setting various chromium args as needed. -* Has special methods. Eg. `driver.uc_click(selector)` +* Has special `uc_*()` methods. 👤 Here's an example with the Driver manager: @@ -67,22 +67,11 @@ with SB(uc=True, test=True) as sb: ```python from seleniumbase import SB -def open_the_turnstile_page(sb): +with SB(uc=True, test=True) as sb: url = "seleniumbase.io/apps/turnstile" - sb.driver.uc_open_with_reconnect(url, reconnect_time=2) - -def click_turnstile_and_verify(sb): - sb.switch_to_frame("iframe") - sb.driver.uc_click("span") + sb.uc_open_with_reconnect(url, reconnect_time=2) + sb.uc_gui_handle_cf() sb.assert_element("img#captcha-success", timeout=3) - -with SB(uc=True, test=True) as sb: - open_the_turnstile_page(sb) - try: - click_turnstile_and_verify(sb) - except Exception: - open_the_turnstile_page(sb) - click_turnstile_and_verify(sb) sb.set_messenger_theme(location="top_left") sb.post_message("SeleniumBase wasn't detected", duration=3) ``` @@ -129,6 +118,27 @@ with SB(uc=True, test=True, ad_block_on=True) as sb: +👤 On Linux, use `sb.uc_gui_handle_cf()` to handle Cloudflare Turnstiles: + +```python +from seleniumbase import SB + +with SB(uc=True, test=True) as sb: + url = "https://www.virtualmanager.com/en/login" + sb.uc_open_with_reconnect(url, 4) + print(sb.get_page_title()) + sb.uc_gui_handle_cf() # Ready if needed! + print(sb.get_page_title()) + sb.assert_element('input[name*="email"]') + sb.assert_element('input[name*="login"]') + sb.set_messenger_theme(location="bottom_center") + sb.post_message("SeleniumBase wasn't detected!") +``` + +uc_gui_handle_cf on Linux + +The 2nd `print()` should output "Virtual Manager", which means that the automation successfully passed the Turnstile. + -------- 👤 In UC Mode, driver.get(url) has been modified from its original version: If anti-bot services are detected from a requests.get(url) call that's made before navigating to the website, then driver.uc_open_with_reconnect(url) will be used instead. To open a URL normally in UC Mode, use driver.default_get(url). @@ -144,6 +154,7 @@ with SB(uc=True, test=True, ad_block_on=True) as sb: ### 👤 Here are some UC Mode examples that bypass CAPTCHAs when clicking is required: +* [SeleniumBase/examples/raw_pyautogui.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/raw_pyautogui.py) * [SeleniumBase/examples/raw_turnstile.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/raw_turnstile.py) * [SeleniumBase/examples/raw_form_turnstile.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/raw_form_turnstile.py) * [SeleniumBase/examples/uc_cdp_events.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/uc_cdp_events.py) @@ -214,11 +225,6 @@ driver.reconnect("breakpoint") (Note that while the special UC Mode breakpoint is active, you can't use Selenium commands in the browser, and the browser can't detect Selenium.) -👤 The two main causes of getting detected in UC Mode (which are both easily handled) are: - -
  • Timing. (UC Mode methods let you customize default values that aren't good enough for your environment.)
    • -
  • Not using driver.uc_click(selector) when you need to remain undetected while clicking something.
    • - 👤 On Linux, you may need to use `driver.uc_gui_handle_cf()` to successfully bypass a Cloudflare CAPTCHA. If there's more than one iframe on that website (and Cloudflare isn't the first one) then put the CSS Selector of that iframe as the first arg to `driver.uc_gui_handle_cf()`. This method uses `pyautogui`. In order for `pyautogui` to focus on the correct element, use `xvfb=True` / `--xvfb` to activate a special virtual display on Linux. 👤 To find out if UC Mode will work at all on a specific site (before adjusting for timing), load your site with the following script: @@ -268,46 +274,15 @@ with ThreadPoolExecutor(max_workers=len(urls)) as executor: -------- -👥 Double Duty: Here's an example of handling two CAPTCHAs on one page: - - - -```python -from seleniumbase import SB - -with SB(uc=True, test=True) as sb: - sb.driver.uc_open_with_reconnect("nopecha.com/demo/turnstile", 3.4) - if sb.is_element_visible("#example-container0 iframe"): - sb.switch_to_frame("#example-container0 iframe") - if not sb.is_element_visible("circle.success-circle"): - sb.driver.uc_click("span", reconnect_time=3) - sb.switch_to_frame("#example-container0 iframe") - sb.switch_to_default_content() - - sb.switch_to_frame("#example-container5 iframe") - sb.driver.uc_click("span", reconnect_time=2.5) - sb.switch_to_frame("#example-container5 iframe") - sb.assert_element("svg#success-icon", timeout=3) - sb.switch_to_parent_frame() - - if sb.is_element_visible("#example-container0 iframe"): - sb.switch_to_frame("#example-container0 iframe") - sb.assert_element("circle.success-circle") - sb.switch_to_parent_frame() - - sb.set_messenger_theme(location="top_center") - sb.post_message("SeleniumBase wasn't detected!", duration=3) -``` - --------- - 👤 What makes UC Mode work? Here are the 3 primary things that UC Mode does to make bots appear human: + For example, if the Chrome DevTools Console variables aren't renamed, you can expect to find them easily when using selenium for browser automation: @@ -321,13 +296,17 @@ While chromedriver is connected to raw Selenium method definitions have been provided for reference (but you don't need to call those methods directly): + Also note that chromedriver isn't detectable in a browser tab if it never touches that tab. Here's a JS command that lets you open a URL in a new tab (from your current tab): + The above JS method is used within SeleniumBase UC Mode methods for opening URLs in a stealthy way. Since some websites try to detect if your browser is a bot on the initial page load, this allows you to bypass detection in those situations. After a few seconds (customizable), UC Mode tells chromedriver to connect to that tab so that automated commands can now be issued. At that point, chromedriver could be detected if websites are looking for it (but generally websites only look for it during specific events, such as page loads, form submissions, and button clicks). diff --git a/requirements.txt b/requirements.txt index 56b90063bb4..a85f6e42ec9 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ pip>=24.0;python_version<"3.8" -pip>=24.1;python_version>="3.8" +pip>=24.1.1;python_version>="3.8" packaging>=24.0;python_version<"3.8" packaging>=24.1;python_version>="3.8" setuptools>=68.0.0;python_version<"3.8" -setuptools>=70.1.0;python_version>="3.8" +setuptools>=70.1.1;python_version>="3.8" wheel>=0.42.0;python_version<"3.8" wheel>=0.43.0;python_version>="3.8" attrs>=23.2.0 diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index bab6b9fe392..0f4cb9997d3 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.28.0" +__version__ = "4.28.1" diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index fe9023a4ab2..0d21607aced 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -1634,6 +1634,10 @@ def get_driver( if headless2 and browser_name == constants.Browser.FIREFOX: headless2 = False # Only for Chromium headless = True + if not hasattr(sb_config, "headless"): + sb_config.headless = headless + if not hasattr(sb_config, "headless2"): + sb_config.headless2 = headless2 if ( binary_location and isinstance(binary_location, str) diff --git a/seleniumbase/fixtures/shared_utils.py b/seleniumbase/fixtures/shared_utils.py index 4bf07657ff9..578d2bab6c1 100644 --- a/seleniumbase/fixtures/shared_utils.py +++ b/seleniumbase/fixtures/shared_utils.py @@ -81,46 +81,46 @@ def format_exc(exception, message): from seleniumbase.common.exceptions import TextNotVisibleException from seleniumbase.common import exceptions - if exception == Exception: + if exception is Exception: exc = Exception return exc, message - elif exception == ElementNotVisibleException: + elif exception is ElementNotVisibleException: exc = exceptions.ElementNotVisibleException elif exception == "ElementNotVisibleException": exc = exceptions.ElementNotVisibleException - elif exception == LinkTextNotFoundException: + elif exception is LinkTextNotFoundException: exc = exceptions.LinkTextNotFoundException elif exception == "LinkTextNotFoundException": exc = exceptions.LinkTextNotFoundException - elif exception == NoSuchElementException: + elif exception is NoSuchElementException: exc = exceptions.NoSuchElementException elif exception == "NoSuchElementException": exc = exceptions.NoSuchElementException - elif exception == TextNotVisibleException: + elif exception is TextNotVisibleException: exc = exceptions.TextNotVisibleException elif exception == "TextNotVisibleException": exc = exceptions.TextNotVisibleException - elif exception == NoAlertPresentException: + elif exception is NoAlertPresentException: exc = exceptions.NoAlertPresentException elif exception == "NoAlertPresentException": exc = exceptions.NoAlertPresentException - elif exception == NoSuchAttributeException: + elif exception is NoSuchAttributeException: exc = exceptions.NoSuchAttributeException elif exception == "NoSuchAttributeException": exc = exceptions.NoSuchAttributeException - elif exception == NoSuchFrameException: + elif exception is NoSuchFrameException: exc = exceptions.NoSuchFrameException elif exception == "NoSuchFrameException": exc = exceptions.NoSuchFrameException - elif exception == NoSuchWindowException: + elif exception is NoSuchWindowException: exc = exceptions.NoSuchWindowException elif exception == "NoSuchWindowException": exc = exceptions.NoSuchWindowException - elif exception == NoSuchFileException: + elif exception is NoSuchFileException: exc = exceptions.NoSuchFileException elif exception == "NoSuchFileException": exc = exceptions.NoSuchFileException - elif exception == NoSuchOptionException: + elif exception is NoSuchOptionException: exc = exceptions.NoSuchOptionException elif exception == "NoSuchOptionException": exc = exceptions.NoSuchOptionException diff --git a/seleniumbase/plugins/driver_manager.py b/seleniumbase/plugins/driver_manager.py index 598077f362c..9ee7f4529aa 100644 --- a/seleniumbase/plugins/driver_manager.py +++ b/seleniumbase/plugins/driver_manager.py @@ -125,7 +125,9 @@ def Driver( uc_cdp=None, # Shortcut / Duplicate of "uc_cdp_events". uc_sub=None, # Shortcut / Duplicate of "uc_subprocess". log_cdp=None, # Shortcut / Duplicate of "log_cdp_events". + ad_block=None, # Shortcut / Duplicate of "ad_block_on". server=None, # Shortcut / Duplicate of "servername". + guest=None, # Shortcut / Duplicate of "guest_mode". wire=None, # Shortcut / Duplicate of "use_wire". pls=None, # Shortcut / Duplicate of "page_load_strategy". ): @@ -263,6 +265,8 @@ def Driver( incognito = True else: incognito = False + if guest is not None and guest_mode is None: + guest_mode = guest if guest_mode is None: if "--guest" in sys_argv: guest_mode = True @@ -515,6 +519,8 @@ def Driver( swiftshader = True else: swiftshader = False + if ad_block is not None and ad_block_on is None: + ad_block_on = ad_block if ad_block_on is None: if "--ad-block" in sys_argv or "--ad_block" in sys_argv: ad_block_on = True diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py index b823d564c2d..938b0f3fd0a 100644 --- a/seleniumbase/plugins/sb_manager.py +++ b/seleniumbase/plugins/sb_manager.py @@ -103,7 +103,9 @@ def SB( uc_cdp=None, # Shortcut / Duplicate of "uc_cdp_events". uc_sub=None, # Shortcut / Duplicate of "uc_subprocess". log_cdp=None, # Shortcut / Duplicate of "log_cdp_events". + ad_block=None, # Shortcut / Duplicate of "ad_block_on". server=None, # Shortcut / Duplicate of "servername". + guest=None, # Shortcut / Duplicate of "guest_mode". wire=None, # Shortcut / Duplicate of "use_wire". pls=None, # Shortcut / Duplicate of "page_load_strategy". sjw=None, # Shortcut / Duplicate of "skip_js_waits". @@ -296,6 +298,8 @@ def SB( incognito = True else: incognito = False + if guest is not None and guest_mode is None: + guest_mode = guest if guest_mode is None: if "--guest" in sys_argv: guest_mode = True @@ -659,6 +663,8 @@ def SB( swiftshader = True else: swiftshader = False + if ad_block is not None and ad_block_on is None: + ad_block_on = ad_block if ad_block_on is None: if "--ad-block" in sys_argv or "--ad_block" in sys_argv: ad_block_on = True diff --git a/setup.py b/setup.py index fb73d9e309a..02cd9a60108 100755 --- a/setup.py +++ b/setup.py @@ -147,11 +147,11 @@ python_requires=">=3.7", install_requires=[ 'pip>=24.0;python_version<"3.8"', - 'pip>=24.1;python_version>="3.8"', + 'pip>=24.1.1;python_version>="3.8"', 'packaging>=24.0;python_version<"3.8"', 'packaging>=24.1;python_version>="3.8"', 'setuptools>=68.0.0;python_version<"3.8"', - 'setuptools>=70.1.0;python_version>="3.8"', + 'setuptools>=70.1.1;python_version>="3.8"', 'wheel>=0.42.0;python_version<"3.8"', 'wheel>=0.43.0;python_version>="3.8"', 'attrs>=23.2.0',