Skip to content

Commit 72a6ba3

Browse files
clee2000clee2000
committed
uncommit_later
1 parent 6861956 commit 72a6ba3

File tree

2 files changed

+101
-25
lines changed

2 files changed

+101
-25
lines changed

‎.github/workflows/build-tutorials.yml

Lines changed: 83 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,86 @@ jobs:
9898
uses: pytorch/test-infra/.github/actions/teardown-linux@main
9999
if: always()
100100

101-
# manager:
102-
# name: pytorch_tutorial_build_manager
103-
# needs: pytorch_tutorial_build_worker
104-
# runs-on: [self-hosted, linux.large]
105-
# steps:
106-
# - name: Checkout Tutorials
107-
# uses: actions/checkout@v3
101+
manager:
102+
name: pytorch_tutorial_build_manager
103+
needs: pytorch_tutorial_build_worker
104+
runs-on: [self-hosted, linux.2xlarge]
105+
env:
106+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7"
107+
CUDA_VERSION: "9"
108+
steps:
109+
- name: Setup SSH (Click me for login details)
110+
uses: pytorch/test-infra/.github/actions/setup-ssh@main
111+
with:
112+
github-secret: ${{ secrets.GITHUB_TOKEN }}
113+
instructions: |
114+
All testing is done inside the container, to start an interactive session run:
115+
docker exec -it $(docker container ps --format '{{.ID}}') bash
116+
117+
- name: Checkout Tutorials
118+
uses: actions/checkout@v3
119+
120+
- name: Setup Linux
121+
uses: pytorch/pytorch/.github/actions/setup-linux@master
122+
123+
- name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
124+
uses: pytorch/test-infra/.github/actions/setup-nvidia@main
125+
126+
- name: Calculate docker image
127+
shell: bash
128+
id: docker-image
129+
run: |
130+
set -ex
131+
132+
# for some reason, pip installs it in a different place than what is looked at in the py file
133+
pip3 install requests
134+
pyTorchDockerImageTag=$(python3 .jenkins/get_docker_tag.py)
135+
136+
echo "docker-image=${DOCKER_IMAGE}:${pyTorchDockerImageTag}" >> "${GITHUB_OUTPUT}"
137+
138+
- name: Pull docker image
139+
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
140+
with:
141+
docker-image: ${{ steps.docker-image.outputs.docker-image }}
142+
143+
- name: Build
144+
shell: bash
145+
env:
146+
DOCKER_IMAGE: ${{ steps.docker-image.outputs.docker-image }}
147+
run: |
148+
set -ex
149+
150+
chmod +x ".jenkins/manager.sh"
151+
152+
container_name=$(docker run \
153+
${GPU_FLAG:-} \
154+
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
155+
--tty \
156+
--detach \
157+
--user jenkins \
158+
--name="${container_name}" \
159+
-v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
160+
-w /var/lib/jenkins/workspace \
161+
"${DOCKER_IMAGE}"
162+
)
163+
164+
echo "rm /opt/cache/bin/*" | docker exec -u root -i "${container_name}" bash
165+
166+
docker exec -t "${container_name}" sh -c ".jenkins/worker.sh"
167+
168+
169+
- name: Upload Python Docs Preview
170+
uses: seemethere/upload-artifact-s3@v5
171+
with:
172+
retention-days: 14
173+
s3-bucket: pytorch-tutorial-build-pull-request
174+
if-no-files-found: error
175+
path: manager.7z
176+
s3-prefix: temp/${{ github.event.pull_request.number }}
177+
env:
178+
AWS_ACCESS_KEY_ID: ${{ secrets.CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET }}
179+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET }}
180+
181+
- name: Teardown Linux
182+
uses: pytorch/test-infra/.github/actions/teardown-linux@main
183+
if: always()

‎.jenkins/worker.sh

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -34,46 +34,46 @@ make docs
3434
# then we remove them
3535
set +x
3636
for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.html'); do
37-
file_basename=$(basename $filename .html)
38-
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
37+
file_basename=$(basename $filename .html)
38+
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
3939
echo "removing $filename"
4040
rm $filename
41-
fi
41+
fi
4242
done
4343
for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.rst'); do
44-
file_basename=$(basename $filename .rst)
45-
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
44+
file_basename=$(basename $filename .rst)
45+
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
4646
echo "removing $filename"
4747
rm $filename
48-
fi
48+
fi
4949
done
5050
for filename in $(find docs/_downloads -name '*.py'); do
51-
file_basename=$(basename $filename .py)
52-
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
51+
file_basename=$(basename $filename .py)
52+
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
5353
echo "removing $filename"
5454
rm $filename
55-
fi
55+
fi
5656
done
5757
for filename in $(find docs/_downloads -name '*.ipynb'); do
58-
file_basename=$(basename $filename .ipynb)
59-
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
58+
file_basename=$(basename $filename .ipynb)
59+
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
6060
echo "removing $filename"
6161
rm $filename
62-
fi
62+
fi
6363
done
6464
for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced docs/_sources/recipes -name '*.rst.txt'); do
65-
file_basename=$(basename $filename .rst.txt)
66-
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
65+
file_basename=$(basename $filename .rst.txt)
66+
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
6767
echo "removing $filename"
6868
rm $filename
69-
fi
69+
fi
7070
done
7171
for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced docs/.doctrees/recipes docs/.doctrees/prototype -name '*.doctree'); do
72-
file_basename=$(basename $filename .doctree)
73-
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
72+
file_basename=$(basename $filename .doctree)
73+
if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
7474
echo "removing $filename"
7575
rm $filename
76-
fi
76+
fi
7777
done
7878
set -x
7979

0 commit comments

Comments
 (0)