diff --git a/requirements.txt b/requirements.txt index 58faf54d3f8..f144a905b20 100755 --- a/requirements.txt +++ b/requirements.txt @@ -60,7 +60,8 @@ pytest-metadata==3.1.1 pytest-ordering==0.6 pytest-rerunfailures==14.0;python_version<"3.9" pytest-rerunfailures==15.1;python_version>="3.9" -pytest-xdist==3.6.1 +pytest-xdist==3.6.1;python_version<"3.9" +pytest-xdist==3.7.0;python_version>="3.9" parameterized==0.9.0 behave==1.2.6 soupsieve==2.7 diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 5173bc46dcb..9958ccb6fb9 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.39.0" +__version__ = "4.39.1" diff --git a/seleniumbase/console_scripts/sb_mkrec.py b/seleniumbase/console_scripts/sb_mkrec.py index 01ca9cfe431..7f72eb7fde7 100644 --- a/seleniumbase/console_scripts/sb_mkrec.py +++ b/seleniumbase/console_scripts/sb_mkrec.py @@ -192,8 +192,9 @@ def main(): elif "'" not in start_page: used_sp = "'%s'" % start_page data.append( - " self.uc_open_with_disconnect(\n" - " %s\n" + " self.activate_cdp_mode(\n" + " %s,\n" + " recorder=True,\n" " )" % used_sp ) else: diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index eb30a46053e..963042aab12 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -108,7 +108,9 @@ def make_driver_executable_if_not(driver_path): shared_utils.make_executable(driver_path) -def extend_driver(driver, proxy_auth=False, use_uc=True): +def extend_driver( + driver, proxy_auth=False, use_uc=True, recorder_ext=False +): # Extend the driver with new methods driver.default_find_element = driver.find_element driver.default_find_elements = driver.find_elements @@ -234,6 +236,14 @@ def extend_driver(driver, proxy_auth=False, use_uc=True): driver.switch_to_tab = DM.switch_to_tab driver.switch_to_frame = DM.switch_to_frame driver.reset_window_size = DM.reset_window_size + if recorder_ext: + from seleniumbase.js_code.recorder_js import recorder_js + recorder_code = ( + """window.onload = function() { %s };""" % recorder_js + ) + driver.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", {"source": recorder_code} + ) if hasattr(driver, "proxy"): driver.set_wire_proxy = DM.set_wire_proxy if proxy_auth: @@ -2542,6 +2552,7 @@ def _set_chrome_options( included_disabled_features.append("PrivacySandboxSettings4") included_disabled_features.append("SidePanelPinning") included_disabled_features.append("UserAgentClientHint") + included_disabled_features.append("DisableLoadExtensionCommandLineSwitch") for item in extra_disabled_features: if item not in included_disabled_features: included_disabled_features.append(item) @@ -2819,6 +2830,8 @@ def get_driver( if headless2 and browser_name == constants.Browser.FIREFOX: headless2 = False # Only for Chromium headless = True + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() if ( is_using_uc(undetectable, browser_name) and binary_location @@ -3014,6 +3027,8 @@ def get_driver( proxy_pass = None proxy_scheme = "http" if proxy_string: + # (The code below was for the Chrome 137 extension fix) + # sb_config._cdp_proxy = proxy_string username_and_password = None if "@" in proxy_string: # Format => username:password@hostname:port @@ -4450,6 +4465,9 @@ def get_local_driver( included_disabled_features.append("PrivacySandboxSettings4") included_disabled_features.append("SidePanelPinning") included_disabled_features.append("UserAgentClientHint") + included_disabled_features.append( + "DisableLoadExtensionCommandLineSwitch" + ) for item in extra_disabled_features: if item not in included_disabled_features: included_disabled_features.append(item) @@ -5328,7 +5346,9 @@ def get_local_driver( driver = webdriver.Chrome( service=service, options=chrome_options ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) if not auto_upgrade_chromedriver: raise # Not an obvious fix. else: @@ -5580,11 +5600,15 @@ def get_local_driver( 'Emulation.setDeviceMetricsOverride', set_device_metrics_override ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) else: # Running headless on Linux (and not using --uc) try: driver = webdriver.Chrome(options=chrome_options) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) except Exception as e: if not hasattr(e, "msg"): raise @@ -5606,7 +5630,9 @@ def get_local_driver( driver = webdriver.Chrome( service=service, options=chrome_options ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) mcv = None # Major Chrome Version if "Current browser version is " in e.msg: line = e.msg.split("Current browser version is ")[1] @@ -5649,7 +5675,9 @@ def get_local_driver( service=service, options=chrome_options, ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) # Use the virtual display on Linux during headless errors logging.debug( "\nWarning: Chrome failed to launch in" @@ -5667,7 +5695,9 @@ def get_local_driver( driver = webdriver.Chrome( service=service, options=chrome_options ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) except Exception as original_exception: if use_uc: raise @@ -5677,7 +5707,9 @@ def get_local_driver( driver = webdriver.Chrome( service=service, options=chrome_options ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) if user_data_dir: print("\nUnable to set user_data_dir while starting Chrome!\n") raise @@ -5704,7 +5736,9 @@ def get_local_driver( ) try: driver = webdriver.Chrome(service=service) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) except Exception: raise original_exception else: diff --git a/seleniumbase/extensions/recorder.zip b/seleniumbase/extensions/recorder.zip index 257fcd562d8..79a4870afae 100644 Binary files a/seleniumbase/extensions/recorder.zip and b/seleniumbase/extensions/recorder.zip differ diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index 1a991f84f57..3d99d0222ff 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -292,6 +292,7 @@ async def get( _cdp_locale = None _cdp_platform = None _cdp_geolocation = None + _cdp_recorder = None if ( hasattr(sb_config, "_cdp_timezone") and sb_config._cdp_timezone ): @@ -330,6 +331,8 @@ async def get( _cdp_geolocation = kwargs["geolocation"] elif "geoloc" in kwargs: _cdp_geolocation = kwargs["geoloc"] + if "recorder" in kwargs: + _cdp_recorder = kwargs["recorder"] if _cdp_timezone: await connection.send(cdp.page.navigate("about:blank")) await connection.set_timezone(_cdp_timezone) @@ -344,9 +347,37 @@ async def get( await connection.send(cdp.page.navigate("about:blank")) await connection.set_geolocation(_cdp_geolocation) # Use the tab to navigate to new url + if ( + hasattr(sb_config, "_cdp_proxy") + and "@" in sb_config._cdp_proxy + and sb_config._cdp_proxy + and "auth" not in kwargs + ): + username_and_password = sb_config._cdp_proxy.split("@")[0] + proxy_user = username_and_password.split(":")[0] + proxy_pass = username_and_password.split(":")[1] + await connection.set_auth( + proxy_user, proxy_pass, self.tabs[0] + ) + time.sleep(0.25) + elif "auth" in kwargs and kwargs["auth"] and ":" in kwargs["auth"]: + username_and_password = kwargs["auth"] + proxy_user = username_and_password.split(":")[0] + proxy_pass = username_and_password.split(":")[1] + await connection.set_auth( + proxy_user, proxy_pass, self.tabs[0] + ) + time.sleep(0.25) frame_id, loader_id, *_ = await connection.send( cdp.page.navigate(url) ) + if _cdp_recorder: + pass # (The code below was for the Chrome 137 extension fix) + '''from seleniumbase.js_code.recorder_js import recorder_js + recorder_code = ( + """window.onload = function() { %s };""" % recorder_js + ) + await connection.send(cdp.runtime.evaluate(recorder_code))''' # Update the frame_id on the tab connection.frame_id = frame_id connection.browser = self diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index fda6ce9451f..945f45871d3 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -360,8 +360,10 @@ async def start( except Exception: time.sleep(0.15) driver = await Browser.create(config) - if proxy and "@" in str(proxy): - time.sleep(0.15) + if proxy: + sb_config._cdp_proxy = proxy + if "@" in str(proxy): + time.sleep(0.15) if lang: sb_config._cdp_locale = lang elif "locale" in kwargs: @@ -402,10 +404,14 @@ async def start_async(*args, **kwargs) -> Browser: binary_location = None if "browser_executable_path" in kwargs: binary_location = kwargs["browser_executable_path"] + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() else: binary_location = detect_b_ver.get_binary_location("google-chrome") - if binary_location and not os.path.exists(binary_location): - binary_location = None + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() + if not os.path.exists(binary_location): + binary_location = None if ( shared_utils.is_chrome_130_or_newer(binary_location) and "user_data_dir" in kwargs @@ -438,10 +444,14 @@ def start_sync(*args, **kwargs) -> Browser: binary_location = None if "browser_executable_path" in kwargs: binary_location = kwargs["browser_executable_path"] + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() else: binary_location = detect_b_ver.get_binary_location("google-chrome") - if binary_location and not os.path.exists(binary_location): - binary_location = None + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() + if not os.path.exists(binary_location): + binary_location = None if ( shared_utils.is_chrome_130_or_newer(binary_location) and "user_data_dir" in kwargs diff --git a/seleniumbase/undetected/cdp_driver/config.py b/seleniumbase/undetected/cdp_driver/config.py index b11228ea59d..e701bb4a801 100644 --- a/seleniumbase/undetected/cdp_driver/config.py +++ b/seleniumbase/undetected/cdp_driver/config.py @@ -144,7 +144,8 @@ def __init__( "--disable-features=IsolateOrigins,site-per-process,Translate," "InsecureDownloadWarnings,DownloadBubble,DownloadBubbleV2," "OptimizationTargetPrediction,OptimizationGuideModelDownloading," - "SidePanelPinning,UserAgentClientHint,PrivacySandboxSettings4", + "SidePanelPinning,UserAgentClientHint,PrivacySandboxSettings4," + "DisableLoadExtensionCommandLineSwitch", ] @property diff --git a/seleniumbase/undetected/cdp_driver/connection.py b/seleniumbase/undetected/cdp_driver/connection.py index 8bcdc82ff36..ecb7f13b345 100644 --- a/seleniumbase/undetected/cdp_driver/connection.py +++ b/seleniumbase/undetected/cdp_driver/connection.py @@ -378,6 +378,36 @@ async def set_geolocation(self, geolocation: Optional[tuple] = None): accuracy=100, )) + async def set_auth(self, username, password, tab): + async def auth_challenge_handler(event: cdp.fetch.AuthRequired): + await tab.send( + cdp.fetch.continue_with_auth( + request_id=event.request_id, + auth_challenge_response=cdp.fetch.AuthChallengeResponse( + response="ProvideCredentials", + username=username, + password=password, + ), + ) + ) + + async def req_paused(event: cdp.fetch.RequestPaused): + await tab.send( + cdp.fetch.continue_request(request_id=event.request_id) + ) + + tab.add_handler( + cdp.fetch.RequestPaused, + lambda event: asyncio.create_task(req_paused(event)), + ) + + tab.add_handler( + cdp.fetch.AuthRequired, + lambda event: asyncio.create_task(auth_challenge_handler(event)), + ) + + await tab.send(cdp.fetch.enable(handle_auth_requests=True)) + def __getattr__(self, item): """:meta private:""" try: diff --git a/setup.py b/setup.py index 76ea8d31143..899bdbaded7 100755 --- a/setup.py +++ b/setup.py @@ -211,7 +211,8 @@ "pytest-ordering==0.6", 'pytest-rerunfailures==14.0;python_version<"3.9"', 'pytest-rerunfailures==15.1;python_version>="3.9"', - 'pytest-xdist==3.6.1', + 'pytest-xdist==3.6.1;python_version<"3.9"', + 'pytest-xdist==3.7.0;python_version>="3.9"', 'parameterized==0.9.0', "behave==1.2.6", 'soupsieve==2.7', @@ -268,7 +269,7 @@ 'pdfminer.six==20250324;python_version<"3.9"', 'pdfminer.six==20250506;python_version>="3.9"', 'cryptography==39.0.2;python_version<"3.9"', - 'cryptography==45.0.2;python_version>="3.9"', + 'cryptography==45.0.3;python_version>="3.9"', 'cffi==1.17.1', "pycparser==2.22", ],