Skip to content

Commit 8f22def

Browse files
sentrivanaszokeasaurusrex
sentrivana
and
szokeasaurusrex
authored
tests: Generate part of the Tasks tox entries by a script (#3976)
- remove hardcoded entries for `celery`, `spark`, `huey`, `dramatiq` from the tox template - remove them from the ignore list in `populate_tox.py` - run `populate_tox.py` to fill in entries for them - run `split_gh_tox_actions.py` to generate the CI yaml files so that they correspond to the new tox.ini The remaining integrations in this group are not trivial to switch over to the script, I'll do this step by step in follow-up PRs. --------- Co-authored-by: Daniel Szoke <7881302+szokeasaurusrex@users.noreply.github.com>
1 parent 25ddbca commit 8f22def

File tree

6 files changed

+56
-128
lines changed

6 files changed

+56
-128
lines changed

‎.github/workflows/test-integrations-ai.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ jobs:
101101
strategy:
102102
fail-fast: false
103103
matrix:
104-
python-version: ["3.8","3.9","3.10","3.11","3.12","3.13"]
104+
python-version: ["3.8","3.9","3.11","3.12"]
105105
# python3.6 reached EOL and is no longer being supported on
106106
# new versions of hosted runners on Github Actions
107107
# ubuntu-20.04 is the last version that supported python3.6

‎.github/workflows/test-integrations-tasks.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ jobs:
2929
strategy:
3030
fail-fast: false
3131
matrix:
32-
python-version: ["3.6","3.7","3.8","3.10","3.11","3.12","3.13"]
32+
python-version: ["3.7","3.8","3.10","3.11","3.12","3.13"]
3333
# python3.6 reached EOL and is no longer being supported on
3434
# new versions of hosted runners on Github Actions
3535
# ubuntu-20.04 is the last version that supported python3.6
@@ -115,7 +115,7 @@ jobs:
115115
strategy:
116116
fail-fast: false
117117
matrix:
118-
python-version: ["3.6","3.7","3.8","3.9","3.10","3.11","3.12"]
118+
python-version: ["3.6","3.7","3.8","3.9","3.10","3.11","3.12","3.13"]
119119
# python3.6 reached EOL and is no longer being supported on
120120
# new versions of hosted runners on Github Actions
121121
# ubuntu-20.04 is the last version that supported python3.6

‎scripts/populate_tox/config.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,19 @@
1313
},
1414
"python": ">=3.8",
1515
},
16+
"celery": {
17+
"package": "celery",
18+
"deps": {
19+
"*": ["newrelic", "redis"],
20+
"py3.7": ["importlib-metadata<5.0"],
21+
},
22+
},
1623
"clickhouse_driver": {
1724
"package": "clickhouse-driver",
1825
},
26+
"dramatiq": {
27+
"package": "dramatiq",
28+
},
1929
"flask": {
2030
"package": "flask",
2131
"deps": {
@@ -33,6 +43,9 @@
3343
"py3.6": ["aiocontextvars"],
3444
},
3545
},
46+
"huey": {
47+
"package": "huey",
48+
},
3649
"huggingface_hub": {
3750
"package": "huggingface_hub",
3851
},
@@ -54,6 +67,10 @@
5467
"redis_py_cluster_legacy": {
5568
"package": "redis-py-cluster",
5669
},
70+
"spark": {
71+
"package": "pyspark",
72+
"python": ">=3.8",
73+
},
5774
"sqlalchemy": {
5875
"package": "sqlalchemy",
5976
},

‎scripts/populate_tox/populate_tox.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,19 +59,18 @@
5959
"beam",
6060
"boto3",
6161
"bottle",
62-
"celery",
6362
"chalice",
6463
"cohere",
6564
"cloud_resource_context",
6665
"cohere",
6766
"django",
68-
"dramatiq",
6967
"falcon",
7068
"fastapi",
7169
"gcp",
7270
"grpc",
7371
"httpx",
7472
"huey",
73+
"huggingface_hub",
7574
"langchain",
7675
"langchain_notiktoken",
7776
"litestar",
@@ -85,7 +84,6 @@
8584
"requests",
8685
"rq",
8786
"sanic",
88-
"spark",
8987
"starlite",
9088
"tornado",
9189
}

‎scripts/populate_tox/tox.jinja

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,6 @@ envlist =
7474
{py3.6,py3.9}-bottle-v{0.12}
7575
{py3.6,py3.12,py3.13}-bottle-latest
7676

77-
# Celery
78-
{py3.6,py3.8}-celery-v{4}
79-
{py3.6,py3.8}-celery-v{5.0}
80-
{py3.7,py3.10}-celery-v{5.1,5.2}
81-
{py3.8,py3.11,py3.12}-celery-v{5.3,5.4,5.5}
82-
{py3.8,py3.12,py3.13}-celery-latest
83-
8477
# Chalice
8578
{py3.6,py3.9}-chalice-v{1.16}
8679
{py3.8,py3.12,py3.13}-chalice-latest
@@ -107,12 +100,6 @@ envlist =
107100
{py3.10,py3.11,py3.12}-django-v{5.0,5.1}
108101
{py3.10,py3.12,py3.13}-django-latest
109102

110-
# dramatiq
111-
{py3.6,py3.9}-dramatiq-v{1.13}
112-
{py3.7,py3.10,py3.11}-dramatiq-v{1.15}
113-
{py3.8,py3.11,py3.12}-dramatiq-v{1.17}
114-
{py3.8,py3.11,py3.12}-dramatiq-latest
115-
116103
# Falcon
117104
{py3.6,py3.7}-falcon-v{1,1.4,2}
118105
{py3.6,py3.11,py3.12}-falcon-v{3}
@@ -139,10 +126,6 @@ envlist =
139126
{py3.9,py3.11,py3.12}-httpx-v{0.25,0.27}
140127
{py3.9,py3.12,py3.13}-httpx-latest
141128

142-
# Huey
143-
{py3.6,py3.11,py3.12}-huey-v{2.0}
144-
{py3.6,py3.12,py3.13}-huey-latest
145-
146129
# Langchain
147130
{py3.9,py3.11,py3.12}-langchain-v0.1
148131
{py3.9,py3.11,py3.12}-langchain-v0.3
@@ -208,10 +191,6 @@ envlist =
208191
{py3.8,py3.11,py3.12}-sanic-v{24.6}
209192
{py3.9,py3.12,py3.13}-sanic-latest
210193

211-
# Spark
212-
{py3.8,py3.10,py3.11}-spark-v{3.1,3.3,3.5}
213-
{py3.8,py3.10,py3.11,py3.12}-spark-latest
214-
215194
# Starlite
216195
{py3.8,py3.11}-starlite-v{1.48,1.51}
217196
# 1.51.14 is the last starlite version; the project continues as litestar
@@ -317,21 +296,6 @@ deps =
317296
bottle-v0.12: bottle~=0.12.0
318297
bottle-latest: bottle
319298
320-
# Celery
321-
celery: redis
322-
celery-v4: Celery~=4.0
323-
celery-v5.0: Celery~=5.0.0
324-
celery-v5.1: Celery~=5.1.0
325-
celery-v5.2: Celery~=5.2.0
326-
celery-v5.3: Celery~=5.3.0
327-
celery-v5.4: Celery~=5.4.0
328-
# TODO: update when stable is out
329-
celery-v5.5: Celery==5.5.0rc4
330-
celery-latest: Celery
331-
332-
celery: newrelic
333-
{py3.7}-celery: importlib-metadata<5.0
334-
335299
# Chalice
336300
chalice: pytest-chalice==0.0.5
337301
chalice-v1.16: chalice~=1.16.0
@@ -370,12 +334,6 @@ deps =
370334
django-v5.1: Django==5.1rc1
371335
django-latest: Django
372336
373-
# dramatiq
374-
dramatiq-v1.13: dramatiq>=1.13,<1.14
375-
dramatiq-v1.15: dramatiq>=1.15,<1.16
376-
dramatiq-v1.17: dramatiq>=1.17,<1.18
377-
dramatiq-latest: dramatiq
378-
379337
# Falcon
380338
falcon-v1.4: falcon~=1.4.0
381339
falcon-v1: falcon~=1.0
@@ -425,10 +383,6 @@ deps =
425383
httpx-v0.27: httpx~=0.27.0
426384
httpx-latest: httpx
427385
428-
# Huey
429-
huey-v2.0: huey~=2.0.0
430-
huey-latest: huey
431-
432386
# Langchain
433387
langchain-v0.1: openai~=1.0.0
434388
langchain-v0.1: langchain~=0.1.11
@@ -543,14 +497,6 @@ deps =
543497
sanic-v24.6: sanic~=24.6.0
544498
sanic-latest: sanic
545499
546-
# Spark
547-
spark-v3.1: pyspark~=3.1.0
548-
spark-v3.3: pyspark~=3.3.0
549-
spark-v3.5: pyspark~=3.5.0
550-
# TODO: update to ~=4.0.0 once stable is out
551-
spark-v4.0: pyspark==4.0.0.dev2
552-
spark-latest: pyspark
553-
554500
# Starlite
555501
starlite: pytest-asyncio
556502
starlite: python-multipart

‎tox.ini

Lines changed: 35 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,6 @@ envlist =
7474
{py3.6,py3.9}-bottle-v{0.12}
7575
{py3.6,py3.12,py3.13}-bottle-latest
7676

77-
# Celery
78-
{py3.6,py3.8}-celery-v{4}
79-
{py3.6,py3.8}-celery-v{5.0}
80-
{py3.7,py3.10}-celery-v{5.1,5.2}
81-
{py3.8,py3.11,py3.12}-celery-v{5.3,5.4,5.5}
82-
{py3.8,py3.12,py3.13}-celery-latest
83-
8477
# Chalice
8578
{py3.6,py3.9}-chalice-v{1.16}
8679
{py3.8,py3.12,py3.13}-chalice-latest
@@ -107,12 +100,6 @@ envlist =
107100
{py3.10,py3.11,py3.12}-django-v{5.0,5.1}
108101
{py3.10,py3.12,py3.13}-django-latest
109102

110-
# dramatiq
111-
{py3.6,py3.9}-dramatiq-v{1.13}
112-
{py3.7,py3.10,py3.11}-dramatiq-v{1.15}
113-
{py3.8,py3.11,py3.12}-dramatiq-v{1.17}
114-
{py3.8,py3.11,py3.12}-dramatiq-latest
115-
116103
# Falcon
117104
{py3.6,py3.7}-falcon-v{1,1.4,2}
118105
{py3.6,py3.11,py3.12}-falcon-v{3}
@@ -139,10 +126,6 @@ envlist =
139126
{py3.9,py3.11,py3.12}-httpx-v{0.25,0.27}
140127
{py3.9,py3.12,py3.13}-httpx-latest
141128

142-
# Huey
143-
{py3.6,py3.11,py3.12}-huey-v{2.0}
144-
{py3.6,py3.12,py3.13}-huey-latest
145-
146129
# Langchain
147130
{py3.9,py3.11,py3.12}-langchain-v0.1
148131
{py3.9,py3.11,py3.12}-langchain-v0.3
@@ -208,10 +191,6 @@ envlist =
208191
{py3.8,py3.11,py3.12}-sanic-v{24.6}
209192
{py3.9,py3.12,py3.13}-sanic-latest
210193

211-
# Spark
212-
{py3.8,py3.10,py3.11}-spark-v{3.1,3.3,3.5}
213-
{py3.8,py3.10,py3.11,py3.12}-spark-latest
214-
215194
# Starlite
216195
{py3.8,py3.11}-starlite-v{1.48,1.51}
217196
# 1.51.14 is the last starlite version; the project continues as litestar
@@ -225,13 +204,6 @@ envlist =
225204
# These come from the populate_tox.py script. Eventually we should move all
226205
# integration tests there.
227206

228-
# ~~~ AI ~~~
229-
{py3.8,py3.10,py3.11}-huggingface_hub-v0.22.2
230-
{py3.8,py3.10,py3.11}-huggingface_hub-v0.24.7
231-
{py3.8,py3.11,py3.12}-huggingface_hub-v0.26.5
232-
{py3.8,py3.12,py3.13}-huggingface_hub-v0.28.1
233-
234-
235207
# ~~~ DBs ~~~
236208
{py3.7,py3.11,py3.12}-clickhouse_driver-v0.2.9
237209

@@ -282,6 +254,22 @@ envlist =
282254
{py3.9,py3.12,py3.13}-strawberry-v0.260.2
283255

284256

257+
# ~~~ Tasks ~~~
258+
{py3.6,py3.7,py3.8}-celery-v4.4.7
259+
{py3.6,py3.7,py3.8}-celery-v5.0.5
260+
{py3.8,py3.11,py3.12}-celery-v5.4.0
261+
262+
{py3.6,py3.7}-dramatiq-v1.9.0
263+
{py3.6,py3.8,py3.9}-dramatiq-v1.12.3
264+
{py3.7,py3.10,py3.11}-dramatiq-v1.15.0
265+
{py3.8,py3.12,py3.13}-dramatiq-v1.17.1
266+
267+
{py3.8,py3.9}-spark-v3.0.3
268+
{py3.8,py3.9}-spark-v3.2.4
269+
{py3.8,py3.10,py3.11}-spark-v3.4.4
270+
{py3.8,py3.10,py3.11}-spark-v3.5.4
271+
272+
285273
# ~~~ Web 1 ~~~
286274
{py3.6,py3.7,py3.8}-flask-v1.1.4
287275
{py3.8,py3.12,py3.13}-flask-v2.3.3
@@ -389,21 +377,6 @@ deps =
389377
bottle-v0.12: bottle~=0.12.0
390378
bottle-latest: bottle
391379

392-
# Celery
393-
celery: redis
394-
celery-v4: Celery~=4.0
395-
celery-v5.0: Celery~=5.0.0
396-
celery-v5.1: Celery~=5.1.0
397-
celery-v5.2: Celery~=5.2.0
398-
celery-v5.3: Celery~=5.3.0
399-
celery-v5.4: Celery~=5.4.0
400-
# TODO: update when stable is out
401-
celery-v5.5: Celery==5.5.0rc4
402-
celery-latest: Celery
403-
404-
celery: newrelic
405-
{py3.7}-celery: importlib-metadata<5.0
406-
407380
# Chalice
408381
chalice: pytest-chalice==0.0.5
409382
chalice-v1.16: chalice~=1.16.0
@@ -442,12 +415,6 @@ deps =
442415
django-v5.1: Django==5.1rc1
443416
django-latest: Django
444417

445-
# dramatiq
446-
dramatiq-v1.13: dramatiq>=1.13,<1.14
447-
dramatiq-v1.15: dramatiq>=1.15,<1.16
448-
dramatiq-v1.17: dramatiq>=1.17,<1.18
449-
dramatiq-latest: dramatiq
450-
451418
# Falcon
452419
falcon-v1.4: falcon~=1.4.0
453420
falcon-v1: falcon~=1.0
@@ -497,10 +464,6 @@ deps =
497464
httpx-v0.27: httpx~=0.27.0
498465
httpx-latest: httpx
499466

500-
# Huey
501-
huey-v2.0: huey~=2.0.0
502-
huey-latest: huey
503-
504467
# Langchain
505468
langchain-v0.1: openai~=1.0.0
506469
langchain-v0.1: langchain~=0.1.11
@@ -615,14 +578,6 @@ deps =
615578
sanic-v24.6: sanic~=24.6.0
616579
sanic-latest: sanic
617580

618-
# Spark
619-
spark-v3.1: pyspark~=3.1.0
620-
spark-v3.3: pyspark~=3.3.0
621-
spark-v3.5: pyspark~=3.5.0
622-
# TODO: update to ~=4.0.0 once stable is out
623-
spark-v4.0: pyspark==4.0.0.dev2
624-
spark-latest: pyspark
625-
626581
# Starlite
627582
starlite: pytest-asyncio
628583
starlite: python-multipart
@@ -645,13 +600,6 @@ deps =
645600
# These come from the populate_tox.py script. Eventually we should move all
646601
# integration tests there.
647602

648-
# ~~~ AI ~~~
649-
huggingface_hub-v0.22.2: huggingface_hub==0.22.2
650-
huggingface_hub-v0.24.7: huggingface_hub==0.24.7
651-
huggingface_hub-v0.26.5: huggingface_hub==0.26.5
652-
huggingface_hub-v0.28.1: huggingface_hub==0.28.1
653-
654-
655603
# ~~~ DBs ~~~
656604
clickhouse_driver-v0.2.9: clickhouse-driver==0.2.9
657605

@@ -713,6 +661,25 @@ deps =
713661
strawberry: httpx
714662

715663

664+
# ~~~ Tasks ~~~
665+
celery-v4.4.7: celery==4.4.7
666+
celery-v5.0.5: celery==5.0.5
667+
celery-v5.4.0: celery==5.4.0
668+
celery: newrelic
669+
celery: redis
670+
py3.7-celery: importlib-metadata<5.0
671+
672+
dramatiq-v1.9.0: dramatiq==1.9.0
673+
dramatiq-v1.12.3: dramatiq==1.12.3
674+
dramatiq-v1.15.0: dramatiq==1.15.0
675+
dramatiq-v1.17.1: dramatiq==1.17.1
676+
677+
spark-v3.0.3: pyspark==3.0.3
678+
spark-v3.2.4: pyspark==3.2.4
679+
spark-v3.4.4: pyspark==3.4.4
680+
spark-v3.5.4: pyspark==3.5.4
681+
682+
716683
# ~~~ Web 1 ~~~
717684
flask-v1.1.4: flask==1.1.4
718685
flask-v2.3.3: flask==2.3.3

0 commit comments

Comments
 (0)