Merge pull request #1836 from seleniumbase/multithreading-improvements

Multithreading Improvements

Author: mdmintz

README.md

@@ -499,8 +499,6 @@ pytest my_first_test.py --pdb
 --edge  # (Shortcut for "--browser=edge".)
 --firefox  # (Shortcut for "--browser=firefox".)
 --safari  # (Shortcut for "--browser=safari".)
---cap-file=FILE  # (The web browser's desired capabilities to use.)
---cap-string=STRING  # (The web browser's desired capabilities to use.)
 --settings-file=FILE  # (Override default SeleniumBase settings.)
 --env=ENV  # (Set the test env. Access with "self.env" in tests.)
 --account=STR  # (Set account. Access with "self.account" in tests.)
@@ -513,11 +511,14 @@ pytest my_first_test.py --pdb
 --protocol=PROTOCOL  # (The Selenium Grid protocol: http|https.)
 --server=SERVER  # (The Selenium Grid server/IP used for tests.)
 --port=PORT  # (The Selenium Grid port used by the test server.)
---proxy=SERVER:PORT  # (Connect to a proxy server:port for tests.)
---proxy=USERNAME:PASSWORD@SERVER:PORT  # (Use authenticated proxy server.)
---proxy-bypass-list=STRING  # (";"-separated hosts to bypass, Eg "*.foo.com")
+--cap-file=FILE  # (The web browser's desired capabilities to use.)
+--cap-string=STRING  # (The web browser's desired capabilities to use.)
+--proxy=SERVER:PORT  # (Connect to a proxy server:port as tests are running)
+--proxy=USERNAME:PASSWORD@SERVER:PORT  # (Use an authenticated proxy server)
+--proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com")
 --proxy-pac-url=URL  # (Connect to a proxy server using a PAC_URL.pac file.)
 --proxy-pac-url=USERNAME:PASSWORD@URL  # (Authenticated proxy with PAC URL.)
+--multi-proxy  # (Allow multiple authenticated proxies when multi-threaded.)
 --agent=STRING  # (Modify the web browser's User-Agent string.)
 --mobile  # (Use the mobile device emulator while running tests.)
 --metrics=STRING  # (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)

examples/proxy_test.py

@@ -18,7 +18,7 @@ def test_proxy(self):
             print("Skipping test for using Safari.")
             self.skip("Skipping test for using Safari.")
         settings.SKIP_JS_WAITS = True
-        if not self.page_load_strategy == "none":
+        if not self.page_load_strategy == "none" and not self.undetectable:
             # This page takes too long to load otherwise
             self.get_new_driver(page_load_strategy="none")
         self.open("https://ipinfo.io/")

examples/raw_parameter_script.py

@@ -116,6 +116,7 @@
     sb.proxy_string = None
     sb.proxy_bypass_list = None
     sb.proxy_pac_url = None
+    sb.multi_proxy = False
     sb.swiftshader = False
     sb.ad_block_on = False
     sb.highlights = None

examples/test_hack_search.py

@@ -13,11 +13,11 @@ def test_hack_search(self):
             print("\n  This test is not for Headless Mode.")
             self.skip('Do not use "--headless" with this test.')
         self.open("https://google.com/ncr")
-        self.assert_element('input[title="Search"]')
+        self.assert_element('[title="Search"]')
         self.sleep(0.5)
         self.set_attribute('[action="/search"]', "action", "//bing.com/search")
         self.set_attributes('[value="Google Search"]', "value", "Bing Search")
-        self.type('input[title="Search"]', "GitHub SeleniumBase Docs Install")
+        self.type('[title="Search"]', "GitHub SeleniumBase Docs Install")
         self.sleep(0.5)
         self.js_click('[value="Bing Search"]')
         self.highlight("h1.b_logo", loops=8)

help_docs/customizing_test_runs.md

@@ -105,8 +105,6 @@ pytest my_first_test.py --settings-file=custom_settings.py
 --edge  # (Shortcut for "--browser=edge".)
 --firefox  # (Shortcut for "--browser=firefox".)
 --safari  # (Shortcut for "--browser=safari".)
---cap-file=FILE  # (The web browser's desired capabilities to use.)
---cap-string=STRING  # (The web browser's desired capabilities to use.)
 --settings-file=FILE  # (Override default SeleniumBase settings.)
 --env=ENV  # (Set the test env. Access with "self.env" in tests.)
 --account=STR  # (Set account. Access with "self.account" in tests.)
@@ -119,11 +117,14 @@ pytest my_first_test.py --settings-file=custom_settings.py
 --protocol=PROTOCOL  # (The Selenium Grid protocol: http|https.)
 --server=SERVER  # (The Selenium Grid server/IP used for tests.)
 --port=PORT  # (The Selenium Grid port used by the test server.)
---proxy=SERVER:PORT  # (Connect to a proxy server:port for tests.)
---proxy=USERNAME:PASSWORD@SERVER:PORT  # (Use authenticated proxy server.)
---proxy-bypass-list=STRING  # (";"-separated hosts to bypass, Eg "*.foo.com")
+--cap-file=FILE  # (The web browser's desired capabilities to use.)
+--cap-string=STRING  # (The web browser's desired capabilities to use.)
+--proxy=SERVER:PORT  # (Connect to a proxy server:port as tests are running)
+--proxy=USERNAME:PASSWORD@SERVER:PORT  # (Use an authenticated proxy server)
+--proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com")
 --proxy-pac-url=URL  # (Connect to a proxy server using a PAC_URL.pac file.)
 --proxy-pac-url=USERNAME:PASSWORD@URL  # (Authenticated proxy with PAC URL.)
+--multi-proxy  # (Allow multiple authenticated proxies when multi-threaded.)
 --agent=STRING  # (Modify the web browser's User-Agent string.)
 --mobile  # (Use the mobile device emulator while running tests.)
 --metrics=STRING  # (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)

mkdocs_build/requirements.txt

@@ -14,7 +14,7 @@ click==8.1.3
 ghp-import==2.1.0
 readme-renderer==37.3
 pymdown-extensions==9.11
-importlib-metadata==6.2.0
+importlib-metadata==6.3.0
 pipdeptree==2.7.0
 bleach==6.0.0
 lunr==0.6.2

requirements.txt

@@ -79,13 +79,15 @@ soupsieve==2.4;python_version>="3.7"
 beautifulsoup4==4.12.2
 cryptography==36.0.2;python_version<"3.7"
 cryptography==40.0.1;python_version>="3.7"
-pygments==2.14.0
+pygments==2.14.0;python_version<"3.7"
+pygments==2.15.0;python_version>="3.7"
 pyreadline3==3.4.1;platform_system=="Windows"
 tabcompleter==1.1.0
 pdbp==1.2.8
 colorama==0.4.5;python_version<"3.7"
 colorama==0.4.6;python_version>="3.7"
 exceptiongroup==1.1.1;python_version>="3.7"
+future-breakpoint==2.0.0;python_version<"3.7"
 importlib-metadata==4.2.0;python_version<"3.8"
 pycparser==2.21
 pyotp==2.7.0;python_version<"3.7"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.13.22"
+__version__ = "4.13.23"

seleniumbase/behave/behave_sb.py

@@ -25,11 +25,12 @@
 -D port=PORT  (The Selenium Grid port used by the test server.)
 -D cap-file=FILE  (The web browser's desired capabilities to use.)
 -D cap-string=STRING  (The web browser's desired capabilities to use.)
--D proxy=SERVER:PORT  (Connect to a proxy server:port for tests.)
--D proxy=USERNAME:PASSWORD@SERVER:PORT  (Use authenticated proxy server.)
+-D proxy=SERVER:PORT  (Connect to a proxy server:port as tests are running)
+-D proxy=USERNAME:PASSWORD@SERVER:PORT  (Use an authenticated proxy server)
 -D proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com")
 -D proxy-pac-url=URL  (Connect to a proxy server using a PAC_URL.pac file.)
 -D proxy-pac-url=USERNAME:PASSWORD@URL  (Authenticated proxy with PAC URL.)
+-D multi-proxy  (Allow multiple authenticated proxies when multi-threaded.)
 -D agent=STRING  (Modify the web browser's User-Agent string.)
 -D mobile  (Use the mobile device emulator while running tests.)
 -D metrics=STRING  (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)
@@ -223,6 +224,7 @@ def get_configured_sb(context):
     sb.proxy_string = None
     sb.proxy_bypass_list = None
     sb.proxy_pac_url = None
+    sb.multi_proxy = False
     sb.enable_3d_apis = False
     sb.swiftshader = False
     sb.ad_block_on = False
@@ -746,6 +748,10 @@ def get_configured_sb(context):
                 proxy_pac_url = sb.proxy_pac_url  # revert to default
             sb.proxy_pac_url = proxy_pac_url
             continue
+        # Handle: -D multi-proxy / multi_proxy
+        if low_key in ["multi-proxy", "multi_proxy"]:
+            sb.multi_proxy = True
+            continue
         # Handle: -D enable-3d-apis / enable_3d_apis
         if low_key in ["enable-3d-apis", "enable_3d_apis"]:
             sb.enable_3d_apis = True
@@ -1121,7 +1127,8 @@ def _perform_behave_unconfigure_():
     from seleniumbase.core import log_helper
     from seleniumbase.core import proxy_helper
 
-    proxy_helper.remove_proxy_zip_if_present()
+    if hasattr(sb_config, "multi_proxy") and not sb_config.multi_proxy:
+        proxy_helper.remove_proxy_zip_if_present()
     if hasattr(sb_config, "reuse_session") and sb_config.reuse_session:
         # Close the shared browser session
         if sb_config.shared_driver: