Skip to content

Commit 763fdce

Browse files
timfeltimfel
committed
[GR-46867] Adds tensorflow wheel recipe and some windows recipes
PullRequest: graalpython/3106
2 parents b8687d9 + 9d9f656 commit 763fdce

12 files changed

+1007
-3249
lines changed

‎.github/workflows/build-act-wheels.yml

Lines changed: 0 additions & 3101 deletions
This file was deleted.

‎.github/workflows/build-linux-aarch64-wheels.yml

Lines changed: 128 additions & 19 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-linux-amd64-wheels.yml

Lines changed: 298 additions & 19 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-macos-aarch64-wheels.yml

Lines changed: 120 additions & 19 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-macos-amd64-wheels.yml

Lines changed: 120 additions & 19 deletions
Large diffs are not rendered by default.

‎.github/workflows/build-repository.yml

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,54 @@ jobs:
44
build-repo:
55
runs-on: ubuntu-latest
66
steps:
7+
- name: Checkout
8+
uses: actions/checkout@v4
79
- name: Download artifacts for linux-amd64
8-
uses: dawidd6/action-download-artifact@v2
10+
uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
11+
continue-on-error: true
912
with:
10-
workflow: build-linux-amd64-wheels
13+
workflow: build-linux-amd64-wheels.yml
14+
workflow_conclusion: ''
15+
if_no_artifact_found: warn
16+
allow_forks: 'false'
1117
- name: Download artifacts for linux-aarch64
12-
uses: dawidd6/action-download-artifact@v2
18+
uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
19+
continue-on-error: true
1320
with:
14-
workflow: build-linux-aarch64-wheels
21+
workflow: build-linux-aarch64-wheels.yml
22+
workflow_conclusion: ''
23+
if_no_artifact_found: warn
24+
allow_forks: 'false'
1525
- name: Download artifacts for macos-amd64
16-
uses: dawidd6/action-download-artifact@v2
26+
uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
27+
continue-on-error: true
1728
with:
18-
workflow: build-macos-amd64-wheels
29+
workflow: build-macos-amd64-wheels.yml
30+
workflow_conclusion: ''
31+
if_no_artifact_found: warn
32+
allow_forks: 'false'
1933
- name: Download artifacts for macos-aarch64
20-
uses: dawidd6/action-download-artifact@v2
34+
uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
35+
continue-on-error: true
2136
with:
22-
workflow: build-macos-aarch64-wheels
37+
workflow: build-macos-aarch64-wheels.yml
38+
workflow_conclusion: ''
39+
if_no_artifact_found: warn
40+
allow_forks: 'false'
2341
- name: Download artifacts for windows-amd64
24-
uses: dawidd6/action-download-artifact@v2
42+
uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
43+
continue-on-error: true
2544
with:
26-
workflow: build-windows-amd64-wheels
45+
workflow: build-windows-amd64-wheels.yml
46+
workflow_conclusion: ''
47+
if_no_artifact_found: warn
48+
allow_forks: 'false'
2749
- name: Set up Python
2850
uses: actions/setup-python@v4
2951
with:
3052
python-version: '3.10'
3153
- name: Create repository
32-
run: python ${GITHUB_WORKSPACE}/generate_repository.py
54+
run: python ${GITHUB_WORKSPACE}/scripts/wheelbuilder/generate_repository.py
3355
- name: Store repository
3456
uses: umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db
3557
with:

‎.github/workflows/build-windows-amd64-wheels.yml

Lines changed: 143 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,146 @@ name: build-windows-amd64-wheels
1010
type: string
1111
description: GraalPy download url prefix (empty for default)
1212
required: false
13-
jobs: {}
13+
jobs:
14+
numpy-windows-amd64:
15+
runs-on:
16+
- self-hosted
17+
- windows
18+
- X64
19+
if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'numpy') }}
20+
steps:
21+
- uses: ilammy/msvc-dev-cmd@v1
22+
- name: Install dependencies
23+
run: |
24+
Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
25+
Expand-Archive patch.zip -DestinationPath ../patch -Force
26+
- name: Checkout
27+
uses: actions/checkout@v3
28+
- name: Setup custom GraalPy
29+
if: inputs.graalpy != ''
30+
run: |-
31+
$ProgressPreference = 'SilentlyContinue'
32+
Invoke-WebRequest ${{ inputs.graalpy }}-windows-amd64.zip -OutFile graalpy-windows-amd64.zip
33+
Expand-Archive graalpy-windows-amd64.zip
34+
mv graalpy-windows-amd64/* graalpy
35+
graalpy/bin/graalpy.exe -s -m ensurepip
36+
graalpy/bin/graalpy.exe -m pip install wheel
37+
- name: Setup GraalPy
38+
uses: actions/setup-python@main
39+
if: inputs.graalpy == ''
40+
with:
41+
python-version: graalpy23.1
42+
- name: Setup local GraalPy venv
43+
if: inputs.graalpy == ''
44+
run: python -m venv graalpy
45+
- name: Build wheel
46+
run: |-
47+
$env:PIP_FIND_LINKS=$PWD
48+
$env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
49+
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy
50+
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy==1.21.6
51+
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy==1.22.4
52+
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy==1.23.1
53+
graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy==1.23.4
54+
- name: Store wheels
55+
uses: umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db
56+
with:
57+
name: numpy-windows-amd64
58+
path: numpy*.whl
59+
if-no-files-found: error
60+
PyYAML-windows-amd64:
61+
runs-on:
62+
- self-hosted
63+
- windows
64+
- X64
65+
needs: numpy-windows-amd64
66+
if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'PyYAML') }}
67+
steps:
68+
- uses: ilammy/msvc-dev-cmd@v1
69+
- name: Install dependencies
70+
run: |
71+
Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
72+
Expand-Archive patch.zip -DestinationPath ../patch -Force
73+
- name: Checkout
74+
uses: actions/checkout@v3
75+
- name: Setup custom GraalPy
76+
if: inputs.graalpy != ''
77+
run: |-
78+
$ProgressPreference = 'SilentlyContinue'
79+
Invoke-WebRequest ${{ inputs.graalpy }}-windows-amd64.zip -OutFile graalpy-windows-amd64.zip
80+
Expand-Archive graalpy-windows-amd64.zip
81+
mv graalpy-windows-amd64/* graalpy
82+
graalpy/bin/graalpy.exe -s -m ensurepip
83+
graalpy/bin/graalpy.exe -m pip install wheel
84+
- name: Setup GraalPy
85+
uses: actions/setup-python@main
86+
if: inputs.graalpy == ''
87+
with:
88+
python-version: graalpy23.1
89+
- name: Setup local GraalPy venv
90+
if: inputs.graalpy == ''
91+
run: python -m venv graalpy
92+
- name: Download artifacts from numpy
93+
uses: actions/download-artifact@v3
94+
continue-on-error: true
95+
with:
96+
name: numpy-windows-amd64
97+
- name: Build wheel
98+
run: |-
99+
$env:PIP_FIND_LINKS=$PWD
100+
$env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
101+
graalpy/bin/graalpy -m pip wheel --find-links $PWD PyYAML
102+
- name: Store wheels
103+
uses: umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db
104+
with:
105+
name: PyYAML-windows-amd64
106+
path: PyYAML*.whl
107+
if-no-files-found: error
108+
ujson-windows-amd64:
109+
runs-on:
110+
- self-hosted
111+
- windows
112+
- X64
113+
needs: PyYAML-windows-amd64
114+
if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'ujson') }}
115+
steps:
116+
- uses: ilammy/msvc-dev-cmd@v1
117+
- name: Install dependencies
118+
run: |
119+
Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
120+
Expand-Archive patch.zip -DestinationPath ../patch -Force
121+
- name: Checkout
122+
uses: actions/checkout@v3
123+
- name: Setup custom GraalPy
124+
if: inputs.graalpy != ''
125+
run: |-
126+
$ProgressPreference = 'SilentlyContinue'
127+
Invoke-WebRequest ${{ inputs.graalpy }}-windows-amd64.zip -OutFile graalpy-windows-amd64.zip
128+
Expand-Archive graalpy-windows-amd64.zip
129+
mv graalpy-windows-amd64/* graalpy
130+
graalpy/bin/graalpy.exe -s -m ensurepip
131+
graalpy/bin/graalpy.exe -m pip install wheel
132+
- name: Setup GraalPy
133+
uses: actions/setup-python@main
134+
if: inputs.graalpy == ''
135+
with:
136+
python-version: graalpy23.1
137+
- name: Setup local GraalPy venv
138+
if: inputs.graalpy == ''
139+
run: python -m venv graalpy
140+
- name: Download artifacts from PyYAML
141+
uses: actions/download-artifact@v3
142+
continue-on-error: true
143+
with:
144+
name: PyYAML-windows-amd64
145+
- name: Build wheel
146+
run: |-
147+
$env:PIP_FIND_LINKS=$PWD
148+
$env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
149+
graalpy/bin/graalpy -m pip wheel --find-links $PWD ujson
150+
- name: Store wheels
151+
uses: umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db
152+
with:
153+
name: ujson-windows-amd64
154+
path: ujson*.whl
155+
if-no-files-found: error

‎scripts/wheelbuilder/README.md

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,18 +42,19 @@ to use the GitHub Action runner script.
4242
### Running actions locally with nektos/act
4343

4444
[Act](https://github.com/nektos/act) allows running GitHub actions locally. We
45-
can use that to just build all the packages on a local machine:
45+
can use that to just build packages on a local machine:
4646

4747
```
4848
./act --artifact-server-path /tmp/artifacts \
49-
-W .github/workflows/build-act-wheels.yml \
49+
-W .github/workflows/build-linux-amd64-wheels.yml \
5050
-P self-hosted=-self-hosted \
5151
-P macOS=-self-hosted -P Linux=-self-hosted \
5252
-P X64=-self-hosted -P ARM64=-self-hosted \
53-
--input name=all
53+
--input name=psutil
5454
```
5555

56-
You can vary the `--input name=` argument to build only a single package.
56+
You can vary the `--input name=` argument to select which package to build or to
57+
build all.
5758

5859
On Linux you will need Docker or Podman. If you're using Podman, make sure you
5960
are running the system service (e.g. `podman system service -t 0` to run it in

‎scripts/wheelbuilder/generate_repository.py

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import re
4949
import shutil
5050
import sys
51+
import html
5152

5253

5354
def normalize(name):
@@ -67,28 +68,49 @@ def normalize(name):
6768
wheels = glob.glob(os.path.join(workspace, "**/*.whl")) or glob.glob(
6869
os.path.join(workspace, "**/**/*.whl.gz__")
6970
)
70-
repo = os.path.join(workspace, "repository")
71+
repo = os.path.join(workspace, "repository", "simple")
7172
os.makedirs(repo, exist_ok=True)
73+
pkg_index = [
74+
"<html><head><title>Simple Index</title>",
75+
"<meta name='api-version' value='2' /></head><body>",
76+
]
77+
processed_wheels = set()
7278
for wheel in wheels:
7379
print("Processing", wheel)
74-
parts = os.path.basename(wheel).split("-")
80+
basename = os.path.basename(wheel)
81+
parts = basename.split("-")
7582
for idx, part in enumerate(parts):
76-
if part.startswith("graalpy3") or part.startswith("py3"):
83+
if part.startswith("graalpy3") or part.startswith("py2") or part.startswith("py3"):
7784
version_idx = idx - 1
7885
break
86+
else:
87+
continue
7988
wheel_name = normalize("-".join(parts[:version_idx]))
8089
target_dir = os.path.join(repo, wheel_name)
8190
os.makedirs(target_dir, exist_ok=True)
8291
if wheel.endswith(".gz__"):
8392
with gzip.open(wheel, "rb") as f_in:
8493
with open(
8594
os.path.join(
86-
target_dir, os.path.basename(wheel).replace(".gz__", "")
95+
target_dir, basename.replace(".gz__", "")
8796
),
8897
"wb",
8998
) as f_out:
9099
shutil.copyfileobj(f_in, f_out)
91100
else:
92101
shutil.copy(wheel, target_dir)
93102

103+
if wheel_name not in processed_wheels:
104+
processed_wheels.add(wheel_name)
105+
wheel_name = html.escape(wheel_name)
106+
pkg_index.append(f"<a href='{wheel_name}/'>{wheel_name}</a><br />")
107+
108+
with open(os.path.join(target_dir, "index.html"), "a") as f:
109+
basename = html.escape(basename)
110+
f.write(f"<a href='{basename}'>{basename}</a><br />\n")
111+
112+
pkg_index.append("</body></html>")
113+
with open(os.path.join(repo, "index.html"), "w") as f:
114+
f.write("\n".join(pkg_index))
115+
94116
shutil.make_archive(f"{workspace}/repository", "zip", repo)

‎scripts/wheelbuilder/generate_workflow.py

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
numpy = BuildSpec(
5454
name="numpy",
5555
extra_versions=["1.21.6", "1.22.4", "1.23.1", "1.23.4"],
56+
platforms=[Linux, Mac, Windows],
5657
system_dependencies={
5758
Linux: ["gcc-toolset-12-gcc-gfortran", "openblas-devel"],
5859
Mac: ["gcc", "openblas"],
@@ -126,8 +127,15 @@
126127
Mac: ["libffi"],
127128
},
128129
)
129-
pyyaml = BuildSpec(name="PyYAML")
130+
pyyaml = BuildSpec(
131+
name="PyYAML",
132+
platforms=[Linux, Mac, Windows],
133+
)
130134
cmake = BuildSpec(name="cmake")
135+
BuildSpec(
136+
name="ujson",
137+
platforms=[Linux, Mac, Windows],
138+
)
131139
BuildSpec(
132140
name="torch",
133141
spec_dependencies=[numpy, ninja, cmake, pybind11, cffi, pyyaml],
@@ -156,6 +164,31 @@
156164
"BUILD_TEST": 0,
157165
},
158166
)
167+
opt_einsum = BuildSpec(
168+
name="opt_einsum",
169+
platforms=[LinuxX86],
170+
)
171+
keras_preprocessing = BuildSpec(
172+
name="keras_preprocessing",
173+
platforms=[LinuxX86],
174+
)
175+
BuildSpec(
176+
name="tensorflow",
177+
platforms=[LinuxX86],
178+
spec_dependencies=[opt_einsum, keras_preprocessing],
179+
before_build=[
180+
"export PIP_FIND_LINKS=$(pwd)",
181+
"pip install pip numpy wheel packaging requests opt_einsum",
182+
"pip install keras_preprocessing --no-deps",
183+
"curl -L https://github.com/bazelbuild/bazel/releases/download/6.4.0/bazel-6.4.0-linux-x86_64 -o $(pwd)/graalpy/bin/bazel",
184+
"chmod +x graalpy/bin/bazel",
185+
"export PATH=$(pwd)/graalpy/bin/:$PATH",
186+
"bazel --version",
187+
],
188+
system_dependencies=[
189+
"openblas-devel", "/usr/bin/cmake", "/usr/bin/sudo", "/usr/bin/curl", "java-11-openjdk-devel"
190+
],
191+
)
159192

160193

161194
if __name__ == "__main__":

0 commit comments

Comments
 (0)