Skip to content

Commit 536fefe

Browse files
atalmanatalman
authored
[Validations] Add option to use meta cdn for pypi index (#1999)
* [Validations] Add option to use meta cdn for pypi index * fix * test
1 parent 89d9a8c commit 536fefe

6 files changed

+70
-60
lines changed

‎.github/scripts/validate_binaries.sh

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ else
1717
# Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
1818
conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
1919
conda activate ${ENV_NAME}
20-
20+
2121
# Remove when https://github.com/pytorch/builder/issues/1985 is fixed
22-
if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda-aarch64' ]]; then
22+
if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda-aarch64' ]]; then
2323
pip3 install numpy --force-reinstall
2424
fi
2525

@@ -35,6 +35,12 @@ else
3535
INSTALLATION=${INSTALLATION/"--index-url"/"--extra-index-url"}
3636
fi
3737

38+
# use-meta-cdn: use meta cdn for pypi download
39+
if [[ ${USE_META_CDN} == 'true' ]]; then
40+
INSTALLATION=${INSTALLATION/"download.pytorch.org"/"d3kup0pazkvub8.cloudfront.net"}
41+
fi
42+
43+
3844
if [[ ${TORCH_ONLY} == 'true' ]]; then
3945
INSTALLATION=${INSTALLATION/"torchvision torchaudio"/""}
4046
TEST_SUFFIX=" --package torchonly"

‎.github/workflows/validate-aarch64-linux-binaries.yml

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ on:
77
description: "Channel to use (nightly, test, release, all)"
88
required: true
99
type: string
10-
ref:
11-
description: 'Reference to checkout, defaults to empty'
12-
default: ""
13-
required: false
14-
type: string
1510
torchonly:
1611
description: 'Validate torchonly'
1712
default: false
@@ -42,6 +37,11 @@ on:
4237
default: false
4338
required: false
4439
type: boolean
40+
use-meta-cdn:
41+
description: 'Use meta cdn for installing pip binaries'
42+
default: false
43+
required: false
44+
type: boolean
4545
use-extra-index-url:
4646
description: 'Use extra-index url for pip tests'
4747
default: false
@@ -58,11 +58,6 @@ on:
5858
- nightly
5959
- test
6060
- all
61-
ref:
62-
description: 'Reference to checkout, defaults to empty'
63-
default: ""
64-
required: false
65-
type: string
6661
torchonly:
6762
description: 'Validate torchonly'
6863
default: false
@@ -88,6 +83,11 @@ on:
8883
default: false
8984
required: false
9085
type: boolean
86+
use-meta-cdn:
87+
description: 'Use meta cdn for installing pip binaries'
88+
default: false
89+
required: false
90+
type: boolean
9191
use-extra-index-url:
9292
description: 'Use extra-index url for pip tests'
9393
default: false
@@ -114,7 +114,7 @@ jobs:
114114
with:
115115
runner: ${{ matrix.validation_runner }}
116116
repository: "pytorch/builder"
117-
ref: ${{ inputs.ref || github.ref }}
117+
ref: ${{ github.ref }}
118118
job-name: ${{ matrix.build_name }}
119119
docker-image: ${{ matrix.container_image }}
120120
binary-matrix: ${{ toJSON(matrix) }}
@@ -131,6 +131,7 @@ jobs:
131131
export RELEASE_VERSION=${{ inputs.version }}
132132
export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
133133
export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
134+
export USE_META_CDN=${{ inputs.use-meta-cdn }}
134135
export USE_VERSION_SET=${{ inputs.use-version-set }}
135136
if [[ ${USE_VERSION_SET} == 'true' ]]; then
136137
export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }}

‎.github/workflows/validate-binaries.yml

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,6 @@ on:
1717
description: "Channel to use (nightly, test, release, all)"
1818
required: true
1919
type: string
20-
ref:
21-
description: 'Reference to checkout, defaults to empty'
22-
default: ""
23-
required: false
24-
type: string
2520
torchonly:
2621
description: 'Validate torchonly'
2722
default: false
@@ -47,6 +42,11 @@ on:
4742
default: false
4843
required: false
4944
type: boolean
45+
use-meta-cdn:
46+
description: 'Use meta cdn for installing pip binaries'
47+
default: false
48+
required: false
49+
type: boolean
5050
use_split_build:
5151
description: |
5252
[Experimental] Use Split Build
@@ -81,11 +81,6 @@ on:
8181
- nightly
8282
- test
8383
- all
84-
ref:
85-
description: 'Reference to checkout, defaults to empty'
86-
default: ""
87-
required: false
88-
type: string
8984
torchonly:
9085
description: 'Validate torchonly'
9186
default: false
@@ -111,6 +106,11 @@ on:
111106
default: false
112107
required: false
113108
type: boolean
109+
use-meta-cdn:
110+
description: 'Use meta cdn for installing pip binaries'
111+
default: false
112+
required: false
113+
type: boolean
114114
use_split_build:
115115
description: |
116116
[Experimental] Use Split Build
@@ -136,21 +136,20 @@ jobs:
136136
uses: ./.github/workflows/validate-windows-binaries.yml
137137
with:
138138
channel: ${{ inputs.channel }}
139-
ref: ${{ inputs.ref || github.ref }}
140139
torchonly: ${{ inputs.torchonly }}
141140
version: ${{ inputs.version }}
142141
release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
143142
use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
144143
use-force-reinstall: ${{ inputs.use-force-reinstall }}
145144
use-extra-index-url: ${{ inputs.use-extra-index-url }}
145+
use-meta-cdn: ${{ inputs.use-meta-cdn }}
146146

147147
linux:
148148
if: inputs.os == 'linux' || inputs.os == 'all'
149149
needs: generate-release-matrix
150150
uses: ./.github/workflows/validate-linux-binaries.yml
151151
with:
152152
channel: ${{ inputs.channel }}
153-
ref: ${{ inputs.ref || github.ref }}
154153
torchonly: ${{ inputs.torchonly }}
155154
version: ${{ inputs.version }}
156155
release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
@@ -159,31 +158,32 @@ jobs:
159158
use-force-reinstall: ${{ inputs.use-force-reinstall }}
160159
use_split_build: ${{ inputs.use_split_build }}
161160
use-extra-index-url: ${{ inputs.use-extra-index-url }}
161+
use-meta-cdn: ${{ inputs.use-meta-cdn }}
162162

163163
linux-aarch64:
164164
if: inputs.os == 'linux-aarch64' || inputs.os == 'all'
165165
needs: generate-release-matrix
166166
uses: ./.github/workflows/validate-aarch64-linux-binaries.yml
167167
with:
168168
channel: ${{ inputs.channel }}
169-
ref: ${{ inputs.ref || github.ref }}
170169
torchonly: ${{ inputs.torchonly }}
171170
version: ${{ inputs.version }}
172171
release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
173172
use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
174173
use-force-reinstall: ${{ inputs.use-force-reinstall }}
175174
use-extra-index-url: ${{ inputs.use-extra-index-url }}
175+
use-meta-cdn: ${{ inputs.use-meta-cdn }}
176176

177177
mac-arm64:
178178
if: inputs.os == 'macos' || inputs.os == 'all'
179179
needs: generate-release-matrix
180180
uses: ./.github/workflows/validate-macos-arm64-binaries.yml
181181
with:
182182
channel: ${{ inputs.channel }}
183-
ref: ${{ inputs.ref || github.ref }}
184183
torchonly: ${{ inputs.torchonly }}
185184
version: ${{ inputs.version }}
186185
release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
187186
use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
188187
use-force-reinstall: ${{ inputs.use-force-reinstall }}
189188
use-extra-index-url: ${{ inputs.use-extra-index-url }}
189+
use-meta-cdn: ${{ inputs.use-meta-cdn }}

‎.github/workflows/validate-linux-binaries.yml

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ on:
77
description: "Channel to use (nightly, test, release, all)"
88
required: true
99
type: string
10-
ref:
11-
description: 'Reference to checkout, defaults to empty'
12-
default: ""
13-
required: false
14-
type: string
1510
torchonly:
1611
description: 'Validate torchonly'
1712
default: false
@@ -47,6 +42,11 @@ on:
4742
default: false
4843
required: false
4944
type: boolean
45+
use-meta-cdn:
46+
description: 'Use meta cdn for installing pip binaries'
47+
default: false
48+
required: false
49+
type: boolean
5050
use_split_build:
5151
description: |
5252
[Experimental] Use split build
@@ -69,11 +69,6 @@ on:
6969
- nightly
7070
- test
7171
- all
72-
ref:
73-
description: 'Reference to checkout, defaults to empty'
74-
default: ""
75-
required: false
76-
type: string
7772
torchonly:
7873
description: 'Validate torchonly'
7974
default: false
@@ -104,6 +99,11 @@ on:
10499
default: false
105100
required: false
106101
type: boolean
102+
use-meta-cdn:
103+
description: 'Use meta cdn for installing pip binaries'
104+
default: false
105+
required: false
106+
type: boolean
107107
use_split_build:
108108
description: |
109109
[Experimental] Use split build
@@ -137,7 +137,7 @@ jobs:
137137
with:
138138
runner: ${{ matrix.validation_runner }}
139139
repository: "pytorch/builder"
140-
ref: ${{ inputs.ref || github.ref }}
140+
ref: ${{ github.ref }}
141141
job-name: ${{ matrix.build_name }}
142142
docker-image: ${{ matrix.gpu_arch_type == 'xpu' && matrix.container_image || 'pytorch/conda-builder' }}
143143
binary-matrix: ${{ toJSON(matrix) }}
@@ -149,6 +149,7 @@ jobs:
149149
export INCLUDE_TEST_OPS=${{ inputs.include-test-ops }}
150150
export USE_ONLY_DL_PYTORCH_ORG=${{ inputs.use-only-dl-pytorch-org }}
151151
export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
152+
export USE_META_CDN=${{ inputs.use-meta-cdn }}
152153
export RELEASE_VERSION=${{ inputs.version }}
153154
export USE_VERSION_SET=${{ inputs.use-version-set }}
154155
if [[ ${USE_VERSION_SET} == 'true' ]]; then

‎.github/workflows/validate-macos-arm64-binaries.yml

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ on:
77
description: "Channel to use (nightly, test, release, all)"
88
required: true
99
type: string
10-
ref:
11-
description: 'Reference to checkout, defaults to empty'
12-
default: ""
13-
required: false
14-
type: string
1510
torchonly:
1611
description: 'Validate torchonly'
1712
default: false
@@ -42,6 +37,11 @@ on:
4237
default: false
4338
required: false
4439
type: boolean
40+
use-meta-cdn:
41+
description: 'Use meta cdn for installing pip binaries'
42+
default: false
43+
required: false
44+
type: boolean
4545
use-extra-index-url:
4646
description: 'Use extra-index url for pip tests'
4747
default: false
@@ -58,11 +58,6 @@ on:
5858
- nightly
5959
- test
6060
- all
61-
ref:
62-
description: 'Reference to checkout, defaults to empty'
63-
default: ""
64-
required: false
65-
type: string
6661
torchonly:
6762
description: 'Validate torchonly'
6863
default: false
@@ -88,6 +83,11 @@ on:
8883
default: false
8984
required: false
9085
type: boolean
86+
use-meta-cdn:
87+
description: 'Use meta cdn for installing pip binaries'
88+
default: false
89+
required: false
90+
type: boolean
9191
use-extra-index-url:
9292
description: 'Use extra-index url for pip tests'
9393
default: false
@@ -113,7 +113,7 @@ jobs:
113113
with:
114114
runner: ${{ matrix.validation_runner }}
115115
repository: "pytorch/builder"
116-
ref: ${{ inputs.ref || github.ref }}
116+
ref: ${{ github.ref }}
117117
job-name: ${{ matrix.build_name }}
118118
binary-matrix: ${{ toJSON(matrix) }}
119119
script: |
@@ -124,6 +124,7 @@ jobs:
124124
export RELEASE_VERSION=${{ inputs.version }}
125125
export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
126126
export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
127+
export USE_META_CDN=${{ inputs.use-meta-cdn }}
127128
export USE_VERSION_SET=${{ inputs.use-version-set }}
128129
129130
if [[ ${{ matrix.package_type }} == "conda" ]]; then

‎.github/workflows/validate-windows-binaries.yml

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ on:
77
description: "Channel to use (nightly, test, release, all)"
88
required: true
99
type: string
10-
ref:
11-
description: 'Reference to checkout, defaults to empty'
12-
default: ""
13-
required: false
14-
type: string
1510
torchonly:
1611
description: 'Validate torchonly'
1712
default: false
@@ -42,6 +37,11 @@ on:
4237
default: false
4338
required: false
4439
type: boolean
40+
use-meta-cdn:
41+
description: 'Use meta cdn for installing pip binaries'
42+
default: false
43+
required: false
44+
type: boolean
4545
use-extra-index-url:
4646
description: 'Use extra-index url for pip tests'
4747
default: false
@@ -58,11 +58,6 @@ on:
5858
- nightly
5959
- test
6060
- all
61-
ref:
62-
description: 'Reference to checkout, defaults to empty'
63-
default: ""
64-
required: false
65-
type: string
6661
torchonly:
6762
description: 'Validate torchonly'
6863
default: false
@@ -88,6 +83,11 @@ on:
8883
default: false
8984
required: false
9085
type: boolean
86+
use-meta-cdn:
87+
description: 'Use meta cdn for installing pip binaries'
88+
default: false
89+
required: false
90+
type: boolean
9191
use-extra-index-url:
9292
description: 'Use extra-index url for pip tests'
9393
default: false
@@ -113,7 +113,7 @@ jobs:
113113
with:
114114
runner: ${{ matrix.package_type == 'libtorch' && 'windows.4xlarge' || matrix.validation_runner }}
115115
repository: "pytorch/builder"
116-
ref: ${{ inputs.ref || github.ref }}
116+
ref: ${{ github.ref }}
117117
job-name: ${{ matrix.build_name }}
118118
binary-matrix: ${{ toJSON(matrix) }}
119119
timeout: 60
@@ -125,6 +125,7 @@ jobs:
125125
export RELEASE_VERSION=${{ inputs.version }}
126126
export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
127127
export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
128+
export USE_META_CDN=${{ inputs.use-meta-cdn }}
128129
export USE_VERSION_SET=${{ inputs.use-version-set }}
129130
if [[ ${USE_VERSION_SET} == 'true' ]]; then
130131
export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }}

0 commit comments

Comments
 (0)