Skip to content

Commit e2880c8

Browse files
clee2000clee2000
committed
update
1 parent 4cd44ae commit e2880c8

File tree

8 files changed

+300
-30
lines changed

8 files changed

+300
-30
lines changed

‎.circleci/config.yml

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -311,127 +311,127 @@ workflows:
311311
- pytorch_tutorial_pr_build_worker_0:
312312
filters:
313313
branches:
314-
ignore:
314+
only:
315315
- master
316316
- main
317317
- pytorch_tutorial_pr_build_worker_1:
318318
filters:
319319
branches:
320-
ignore:
320+
only:
321321
- master
322322
- main
323323
- pytorch_tutorial_pr_build_worker_2:
324324
filters:
325325
branches:
326-
ignore:
326+
only:
327327
- master
328328
- main
329329
- pytorch_tutorial_pr_build_worker_3:
330330
filters:
331331
branches:
332-
ignore:
332+
only:
333333
- master
334334
- main
335335
- pytorch_tutorial_pr_build_worker_4:
336336
filters:
337337
branches:
338-
ignore:
338+
only:
339339
- master
340340
- main
341341
- pytorch_tutorial_pr_build_worker_5:
342342
filters:
343343
branches:
344-
ignore:
344+
only:
345345
- master
346346
- main
347347
- pytorch_tutorial_pr_build_worker_6:
348348
filters:
349349
branches:
350-
ignore:
350+
only:
351351
- master
352352
- main
353353
- pytorch_tutorial_pr_build_worker_7:
354354
filters:
355355
branches:
356-
ignore:
356+
only:
357357
- master
358358
- main
359359
- pytorch_tutorial_pr_build_worker_8:
360360
filters:
361361
branches:
362-
ignore:
362+
only:
363363
- master
364364
- main
365365
- pytorch_tutorial_pr_build_worker_9:
366366
filters:
367367
branches:
368-
ignore:
368+
only:
369369
- master
370370
- main
371371
- pytorch_tutorial_pr_build_worker_10:
372372
filters:
373373
branches:
374-
ignore:
374+
only:
375375
- master
376376
- main
377377
- pytorch_tutorial_pr_build_worker_11:
378378
filters:
379379
branches:
380-
ignore:
380+
only:
381381
- master
382382
- main
383383
- pytorch_tutorial_pr_build_worker_12:
384384
filters:
385385
branches:
386-
ignore:
386+
only:
387387
- master
388388
- main
389389
- pytorch_tutorial_pr_build_worker_13:
390390
filters:
391391
branches:
392-
ignore:
392+
only:
393393
- master
394394
- main
395395
- pytorch_tutorial_pr_build_worker_14:
396396
filters:
397397
branches:
398-
ignore:
398+
only:
399399
- master
400400
- main
401401
- pytorch_tutorial_pr_build_worker_15:
402402
filters:
403403
branches:
404-
ignore:
404+
only:
405405
- master
406406
- main
407407
- pytorch_tutorial_pr_build_worker_16:
408408
filters:
409409
branches:
410-
ignore:
410+
only:
411411
- master
412412
- main
413413
- pytorch_tutorial_pr_build_worker_17:
414414
filters:
415415
branches:
416-
ignore:
416+
only:
417417
- master
418418
- main
419419
- pytorch_tutorial_pr_build_worker_18:
420420
filters:
421421
branches:
422-
ignore:
422+
only:
423423
- master
424424
- main
425425
- pytorch_tutorial_pr_build_worker_19:
426426
filters:
427427
branches:
428-
ignore:
428+
only:
429429
- master
430430
- main
431431
- pytorch_tutorial_pr_build_manager:
432432
filters:
433433
branches:
434-
ignore:
434+
only:
435435
- master
436436
- main
437437
requires:
@@ -627,25 +627,25 @@ workflows:
627627
# - pytorch_tutorial_windows_pr_build_worker_0:
628628
# filters:
629629
# branches:
630-
# ignore:
630+
# only:
631631
# - master
632632
# - main
633633
# - pytorch_tutorial_windows_pr_build_worker_1:
634634
# filters:
635635
# branches:
636-
# ignore:
636+
# only:
637637
# - master
638638
# - main
639639
# - pytorch_tutorial_windows_pr_build_worker_2:
640640
# filters:
641641
# branches:
642-
# ignore:
642+
# only:
643643
# - master
644644
# - main
645645
# - pytorch_tutorial_windows_pr_build_worker_3:
646646
# filters:
647647
# branches:
648-
# ignore:
648+
# only:
649649
# - master
650650
# - main
651651
# - pytorch_tutorial_windows_trunk_build_worker_0:

‎.circleci/regenerate.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import yaml
1010
from jinja2 import select_autoescape
1111

12-
WORKFLOWS_JOBS_PR = {"filters": {"branches": {"ignore": ["master", "main"]}}}
12+
WORKFLOWS_JOBS_PR = {"filters": {"branches": {"only": ["master", "main"]}}}
1313

1414
WORKFLOWS_JOBS_TRUNK = {
1515
"context": "org-member",

‎.github/workflows/build-tutorials.yml

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: Build tutorials
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
env:
9+
AWS_ACCESS_KEY_ID: ${{ secrets.CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET }}
10+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET }}
11+
jobs:
12+
worker:
13+
name: pytorch_tutorial_build_worker
14+
strategy:
15+
matrix:
16+
include:
17+
- { shard: 1, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" }
18+
runs-on: ${{ matrix.runner }}
19+
env:
20+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7"
21+
CUDA_VERSION: "9"
22+
steps:
23+
- name: Setup SSH (Click me for login details)
24+
uses: pytorch/test-infra/.github/actions/setup-ssh@main
25+
with:
26+
github-secret: ${{ secrets.GITHUB_TOKEN }}
27+
instructions: |
28+
All testing is done inside the container, to start an interactive session run:
29+
docker exec -it $(docker container ps --format '{{.ID}}') bash
30+
31+
- name: Checkout Tutorials
32+
uses: actions/checkout@v3
33+
34+
- name: Setup Linux
35+
uses: pytorch/pytorch/.github/actions/setup-linux@master
36+
37+
- name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
38+
uses: pytorch/test-infra/.github/actions/setup-nvidia@main
39+
40+
- name: Calculate docker image
41+
shell: bash
42+
id: docker-image
43+
run: |
44+
set -ex
45+
46+
# for some reason, pip installs it in a different place than what is looked at in the py file
47+
pip3 install requests
48+
pyTorchDockerImageTag=$(python3 .jenkins/get_docker_tag.py)
49+
50+
echo "docker-image=${DOCKER_IMAGE}:${pyTorchDockerImageTag}" >> "${GITHUB_OUTPUT}"
51+
52+
- name: Pull docker image
53+
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
54+
with:
55+
docker-image: ${{ steps.docker-image.outputs.docker-image }}
56+
57+
- name: Build
58+
shell: bash
59+
env:
60+
DOCKER_IMAGE: ${{ steps.docker-image.outputs.docker-image }}
61+
NUM_WORKERS: ${{ matrix.num_shards }}
62+
WORKER_ID: ${{ matrix.shard }}
63+
run: |
64+
set -ex
65+
66+
chmod +x ".jenkins/worker.sh"
67+
68+
container_name=$(docker run \
69+
${GPU_FLAG:-} \
70+
-e WORKER_ID \
71+
-e NUM_WORKERS \
72+
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
73+
--tty \
74+
--detach \
75+
--user jenkins \
76+
--name="${container_name}" \
77+
-v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
78+
-w /var/lib/jenkins/workspace \
79+
"${DOCKER_IMAGE}"
80+
)
81+
82+
echo "rm /opt/cache/bin/*" | docker exec -u root -i "${container_name}" bash
83+
84+
docker exec -t "${container_name}" sh -c ".jenkins/worker.sh"
85+
86+
87+
- name: Upload Python Docs Preview
88+
uses: seemethere/upload-artifact-s3@v5
89+
with:
90+
retention-days: 14
91+
s3-bucket: pytorch-tutorial-build-pull-request
92+
if-no-files-found: error
93+
path: worker_${{ matrix.shard }}.7z
94+
s3-prefix: temp/${{ github.event.pull_request.number }}
95+
96+
- name: Teardown Linux
97+
uses: pytorch/test-infra/.github/actions/teardown-linux@main
98+
if: always()
99+
100+
# manager:
101+
# name: pytorch_tutorial_build_manager
102+
# needs: pytorch_tutorial_build_worker
103+
# runs-on: [self-hosted, linux.large]
104+
# steps:
105+
# - name: Checkout Tutorials
106+
# uses: actions/checkout@v3

‎.jenkins/build.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ pip install -r $DIR/../requirements.txt
2525
# Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
2626
# RC Link
2727
# pip uninstall -y torch torchvision torchaudio torchtext
28-
# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext
28+
# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext
2929
# pip uninstall -y torch torchvision torchaudio torchtext
3030
# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext
3131

@@ -47,7 +47,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
4747
# python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true
4848
# Temp remove for 1.10 release.
4949
# python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true
50-
50+
5151
# TODO: Fix bugs in these tutorials to make them runnable again
5252
# python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true
5353

‎.jenkins/common.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
# set locale for click dependency in spacy
6+
export LC_ALL=C.UTF-8
7+
export LANG=C.UTF-8
8+
9+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
10+
11+
# Update root certificates by installing new libgnutls30
12+
sudo apt-get update || sudo apt-get install libgnutls30
13+
sudo apt-get update
14+
sudo apt-get install -y --no-install-recommends unzip p7zip-full sox libsox-dev libsox-fmt-all rsync
15+
16+
export PATH=/opt/conda/bin:$PATH
17+
rm -rf src
18+
pip install -r $DIR/../requirements.txt
19+
20+
#Install PyTorch Nightly for test.
21+
# Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
22+
# RC Link
23+
# pip uninstall -y torch torchvision torchaudio torchtext
24+
# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext
25+
# pip uninstall -y torch torchvision torchaudio torchtext
26+
# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext
27+
28+
# Install two language tokenizers for Translation with TorchText tutorial
29+
python -m spacy download en_core_web_sm
30+
python -m spacy download de_core_news_sm

‎.jenkins/get_files_to_run.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ def parse_args() -> Any:
8787
from argparse import ArgumentParser
8888
parser = ArgumentParser("Select files to run")
8989
parser.add_argument("--dry-run", action="store_true")
90-
parser.add_argument("--num-shards", type=int, default=int(os.environ.get("NUM_WORKERS", 20)))
91-
parser.add_argument("--shard-num", type=int, default=int(os.environ.get("WORKER_ID", 0)))
90+
parser.add_argument("--num-shards", type=int, default=int(os.environ.get("NUM_WORKERS", "20")))
91+
parser.add_argument("--shard-num", type=int, default=int(os.environ.get("WORKER_ID", "0")))
9292
return parser.parse_args()
9393

9494

0 commit comments

Comments
 (0)