toxgen: Retry & fail if we fail to fetch PyPI data (#4251)

- try to refetch data if PyPI returns an error
- if we fail after 3 tries, fail the whole script (it doesn't make sense
to run it without access to up-to-date PyPI data)

Author: sentrivana

scripts/populate_tox/populate_tox.py

@@ -36,6 +36,8 @@
     lstrip_blocks=True,
 )
 
+PYPI_COOLDOWN = 0.15  # seconds to wait between requests to PyPI
+
 PYPI_PROJECT_URL = "https://pypi.python.org/pypi/{project}/json"
 PYPI_VERSION_URL = "https://pypi.python.org/pypi/{project}/{version}/json"
 CLASSIFIER_PREFIX = "Programming Language :: Python :: "
@@ -88,27 +90,34 @@
 }
 
 
-@functools.cache
-def fetch_package(package: str) -> dict:
-    """Fetch package metadata from PyPI."""
-    url = PYPI_PROJECT_URL.format(project=package)
-    pypi_data = requests.get(url)
+def fetch_url(url: str) -> Optional[dict]:
+    for attempt in range(3):
+        pypi_data = requests.get(url)
 
-    if pypi_data.status_code != 200:
-        print(f"{package} not found")
+        if pypi_data.status_code == 200:
+            return pypi_data.json()
 
-    return pypi_data.json()
+        backoff = PYPI_COOLDOWN * 2**attempt
+        print(
+            f"{url} returned an error: {pypi_data.status_code}. Attempt {attempt + 1}/3. Waiting {backoff}s"
+        )
+        time.sleep(backoff)
+
+    return None
 
 
 @functools.cache
-def fetch_release(package: str, version: Version) -> dict:
-    url = PYPI_VERSION_URL.format(project=package, version=version)
-    pypi_data = requests.get(url)
+def fetch_package(package: str) -> Optional[dict]:
+    """Fetch package metadata from PyPI."""
+    url = PYPI_PROJECT_URL.format(project=package)
+    return fetch_url(url)
 
-    if pypi_data.status_code != 200:
-        print(f"{package} not found")
 
-    return pypi_data.json()
+@functools.cache
+def fetch_release(package: str, version: Version) -> Optional[dict]:
+    """Fetch release metadata from PyPI."""
+    url = PYPI_VERSION_URL.format(project=package, version=version)
+    return fetch_url(url)
 
 
 def _prefilter_releases(
@@ -229,8 +238,14 @@ def get_supported_releases(
         expected_python_versions = SpecifierSet(f">={MIN_PYTHON_VERSION}")
 
     def _supports_lowest(release: Version) -> bool:
-        time.sleep(0.1)  # don't DoS PYPI
-        py_versions = determine_python_versions(fetch_release(package, release))
+        time.sleep(PYPI_COOLDOWN)  # don't DoS PYPI
+
+        pypi_data = fetch_release(package, release)
+        if pypi_data is None:
+            print("Failed to fetch necessary data from PyPI. Aborting.")
+            sys.exit(1)
+
+        py_versions = determine_python_versions(pypi_data)
         target_python_versions = TEST_SUITE_CONFIG[integration].get("python")
         if target_python_versions:
             target_python_versions = SpecifierSet(target_python_versions)
@@ -499,7 +514,11 @@ def _add_python_versions_to_release(
     integration: str, package: str, release: Version
 ) -> None:
     release_pypi_data = fetch_release(package, release)
-    time.sleep(0.1)  # give PYPI some breathing room
+    if release_pypi_data is None:
+        print("Failed to fetch necessary data from PyPI. Aborting.")
+        sys.exit(1)
+
+    time.sleep(PYPI_COOLDOWN)  # give PYPI some breathing room
 
     target_python_versions = TEST_SUITE_CONFIG[integration].get("python")
     if target_python_versions:
@@ -592,6 +611,9 @@ def main(fail_on_changes: bool = False) -> None:
 
             # Fetch data for the main package
             pypi_data = fetch_package(package)
+            if pypi_data is None:
+                print("Failed to fetch necessary data from PyPI. Aborting.")
+                sys.exit(1)
 
             # Get the list of all supported releases
 

tox.ini

@@ -10,7 +10,7 @@
 # The file (and all resulting CI YAMLs) then need to be regenerated via
 # "scripts/generate-test-files.sh".
 #
-# Last generated: 2025-04-03T11:46:44.595900+00:00
+# Last generated: 2025-04-08T10:33:11.499210+00:00
 
 [tox]
 requires =
@@ -179,7 +179,7 @@ envlist =
 
     {py3.7,py3.12,py3.13}-statsig-v0.55.3
     {py3.7,py3.12,py3.13}-statsig-v0.56.0
-    {py3.7,py3.12,py3.13}-statsig-v0.57.1
+    {py3.7,py3.12,py3.13}-statsig-v0.57.2
 
     {py3.8,py3.12,py3.13}-unleash-v6.0.1
     {py3.8,py3.12,py3.13}-unleash-v6.1.0
@@ -202,7 +202,7 @@ envlist =
     {py3.8,py3.10,py3.11}-strawberry-v0.209.8
     {py3.8,py3.11,py3.12}-strawberry-v0.227.7
     {py3.8,py3.11,py3.12}-strawberry-v0.245.0
-    {py3.9,py3.12,py3.13}-strawberry-v0.263.0
+    {py3.9,py3.12,py3.13}-strawberry-v0.263.2
 
 
     # ~~~ Network ~~~
@@ -215,7 +215,7 @@ envlist =
     # ~~~ Tasks ~~~
     {py3.6,py3.7,py3.8}-celery-v4.4.7
     {py3.6,py3.7,py3.8}-celery-v5.0.5
-    {py3.8,py3.12,py3.13}-celery-v5.5.0
+    {py3.8,py3.12,py3.13}-celery-v5.5.1
 
     {py3.6,py3.7}-dramatiq-v1.9.0
     {py3.6,py3.8,py3.9}-dramatiq-v1.12.3
@@ -260,7 +260,7 @@ envlist =
     {py3.8,py3.10,py3.11}-litestar-v2.0.1
     {py3.8,py3.11,py3.12}-litestar-v2.5.5
     {py3.8,py3.11,py3.12}-litestar-v2.10.0
-    {py3.8,py3.12,py3.13}-litestar-v2.15.1
+    {py3.8,py3.12,py3.13}-litestar-v2.15.2
 
     {py3.6}-pyramid-v1.8.6
     {py3.6,py3.8,py3.9}-pyramid-v1.10.8
@@ -542,7 +542,7 @@ deps =
 
     statsig-v0.55.3: statsig==0.55.3
     statsig-v0.56.0: statsig==0.56.0
-    statsig-v0.57.1: statsig==0.57.1
+    statsig-v0.57.2: statsig==0.57.2
     statsig: typing_extensions
 
     unleash-v6.0.1: UnleashClient==6.0.1
@@ -574,7 +574,7 @@ deps =
     strawberry-v0.209.8: strawberry-graphql[fastapi,flask]==0.209.8
     strawberry-v0.227.7: strawberry-graphql[fastapi,flask]==0.227.7
     strawberry-v0.245.0: strawberry-graphql[fastapi,flask]==0.245.0
-    strawberry-v0.263.0: strawberry-graphql[fastapi,flask]==0.263.0
+    strawberry-v0.263.2: strawberry-graphql[fastapi,flask]==0.263.2
     strawberry: httpx
     strawberry-v0.209.8: pydantic<2.11
     strawberry-v0.227.7: pydantic<2.11
@@ -595,7 +595,7 @@ deps =
     # ~~~ Tasks ~~~
     celery-v4.4.7: celery==4.4.7
     celery-v5.0.5: celery==5.0.5
-    celery-v5.5.0: celery==5.5.0
+    celery-v5.5.1: celery==5.5.1
     celery: newrelic
     celery: redis
     py3.7-celery: importlib-metadata<5.0
@@ -683,7 +683,7 @@ deps =
     litestar-v2.0.1: litestar==2.0.1
     litestar-v2.5.5: litestar==2.5.5
     litestar-v2.10.0: litestar==2.10.0
-    litestar-v2.15.1: litestar==2.15.1
+    litestar-v2.15.2: litestar==2.15.2
     litestar: pytest-asyncio
     litestar: python-multipart
     litestar: requests