Skip to content

Commit e31bd3f

Browse files
holly1238holly1238
authored
Merge branch 'master' into patch-1
2 parents 0b06ec1 + bdbaa7a commit e31bd3f

File tree

220 files changed

+22996
-4624
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

220 files changed

+22996
-4624
lines changed

‎.circleci/config.yml

Lines changed: 135 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
version: 2.1
22

3+
executors:
4+
windows-with-nvidia-gpu:
5+
machine:
6+
resource_class: windows.gpu.nvidia.medium
7+
image: windows-server-2019-nvidia:stable
8+
shell: bash.exe
9+
310
install_official_git_client: &install_official_git_client
411
name: Install Official Git Client
512
no_output_timeout: "1h"
@@ -99,7 +106,7 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
99106
sudo pip -q install awscli==1.16.35
100107
101108
if [ -n "${CUDA_VERSION}" ]; then
102-
DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
109+
DRIVER_FN="NVIDIA-Linux-x86_64-460.39.run"
103110
wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
104111
sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
105112
nvidia-smi
@@ -109,13 +116,15 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
109116
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_ONLY}
110117
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY}
111118
eval $(aws ecr get-login --region us-east-1 --no-include-email)
119+
- restore_cache:
120+
key: v1.0-tutorial-{{ .Environment.CIRCLE_JOB }}
112121
- run:
113122
name: Build
114123
no_output_timeout: "20h"
115124
command: |
116125
set -e
117126
118-
export pyTorchDockerImageTag=291
127+
export pyTorchDockerImageTag=9de29bef4a5dc0dd1dd19428d83e5a66a44a1ed2
119128
echo "PyTorchDockerImageTag: "${pyTorchDockerImageTag}
120129
121130
cat >/home/circleci/project/ci_build_script.sh \<<EOL
@@ -157,24 +166,77 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
157166
fi
158167
set -x
159168
169+
# This also copies the cached build to docker.
160170
docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
161171
162172
export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
163173
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
164174
175+
# Copy the last build from docker
176+
docker cp "$id:/var/lib/jenkins/workspace/_build" /home/circleci/project
177+
docker cp "$id:/var/lib/jenkins/workspace/docs" /home/circleci/project
178+
docker cp "$id:/var/lib/jenkins/workspace/advanced" /home/circleci/project
179+
docker cp "$id:/var/lib/jenkins/workspace/beginner" /home/circleci/project
180+
docker cp "$id:/var/lib/jenkins/workspace/intermediate" /home/circleci/project
181+
docker cp "$id:/var/lib/jenkins/workspace/prototype" /home/circleci/project
182+
docker cp "$id:/var/lib/jenkins/workspace/recipes" /home/circleci/project
183+
docker cp "$id:/var/lib/jenkins/workspace/src" /home/circleci/project
184+
185+
- save_cache:
186+
# Save to cache for incremental build
187+
key: v1.0-tutorial-{{ .Environment.CIRCLE_JOB }}
188+
paths:
189+
- /home/circleci/project/_build
190+
- /home/circleci/project/docs
191+
- /home/circleci/project/advanced
192+
- /home/circleci/project/beginner
193+
- /home/circleci/project/intermediate
194+
- /home/circleci/project/prototype
195+
- /home/circleci/project/recipes
196+
- /home/circleci/project/src
197+
165198
pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults
166199
environment:
167-
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3"
200+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
168201
CUDA_VERSION: "9"
169202
resource_class: gpu.medium
170203
<<: *pytorch_tutorial_build_defaults
171204

172205
pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_defaults
173206
environment:
174-
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3"
207+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
175208
resource_class: medium
176209
<<: *pytorch_tutorial_build_defaults
177210

211+
pytorch_windows_build_worker: &pytorch_windows_build_worker
212+
executor: windows-with-nvidia-gpu
213+
steps:
214+
- checkout
215+
- run:
216+
name: Install Cuda
217+
no_output_timeout: 30m
218+
command: |
219+
.circleci/scripts/windows_cuda_install.sh
220+
- run:
221+
name: Generate cache key
222+
# This will refresh cache on Sundays, build should generate new cache.
223+
command: echo "$(date +"%Y-%U")" > .circleci-weekly
224+
- restore_cache:
225+
keys:
226+
- data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
227+
- run:
228+
name: test
229+
no_output_timeout: "1h"
230+
command: |
231+
.circleci/scripts/build_for_windows.sh
232+
- save_cache:
233+
key: data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
234+
paths:
235+
- advanced_source/data
236+
- beginner_source/data
237+
- intermediate_source/data
238+
- prototype_source/data
239+
178240
jobs:
179241
pytorch_tutorial_pr_build_worker_0:
180242
<<: *pytorch_tutorial_build_worker_defaults
@@ -302,6 +364,30 @@ jobs:
302364
pytorch_tutorial_master_build_manager:
303365
<<: *pytorch_tutorial_build_manager_defaults
304366

367+
pytorch_tutorial_windows_pr_build_worker_0:
368+
<<: *pytorch_windows_build_worker
369+
370+
pytorch_tutorial_windows_pr_build_worker_1:
371+
<<: *pytorch_windows_build_worker
372+
373+
pytorch_tutorial_windows_pr_build_worker_2:
374+
<<: *pytorch_windows_build_worker
375+
376+
pytorch_tutorial_windows_pr_build_worker_3:
377+
<<: *pytorch_windows_build_worker
378+
379+
pytorch_tutorial_windows_master_build_worker_0:
380+
<<: *pytorch_windows_build_worker
381+
382+
pytorch_tutorial_windows_master_build_worker_1:
383+
<<: *pytorch_windows_build_worker
384+
385+
pytorch_tutorial_windows_master_build_worker_2:
386+
<<: *pytorch_windows_build_worker
387+
388+
pytorch_tutorial_windows_master_build_worker_3:
389+
<<: *pytorch_windows_build_worker
390+
305391
workflows:
306392
build:
307393
jobs:
@@ -538,3 +624,48 @@ workflows:
538624
branches:
539625
only:
540626
- master
627+
- pytorch_tutorial_windows_pr_build_worker_0:
628+
filters:
629+
branches:
630+
ignore:
631+
- master
632+
- pytorch_tutorial_windows_pr_build_worker_1:
633+
filters:
634+
branches:
635+
ignore:
636+
- master
637+
- pytorch_tutorial_windows_pr_build_worker_2:
638+
filters:
639+
branches:
640+
ignore:
641+
- master
642+
- pytorch_tutorial_windows_pr_build_worker_3:
643+
filters:
644+
branches:
645+
ignore:
646+
- master
647+
- pytorch_tutorial_windows_master_build_worker_0:
648+
context: org-member
649+
filters:
650+
branches:
651+
only:
652+
- master
653+
- pytorch_tutorial_windows_master_build_worker_1:
654+
context: org-member
655+
filters:
656+
branches:
657+
only:
658+
- master
659+
- pytorch_tutorial_windows_master_build_worker_2:
660+
context: org-member
661+
filters:
662+
branches:
663+
only:
664+
- master
665+
- pytorch_tutorial_windows_master_build_worker_3:
666+
context: org-member
667+
filters:
668+
branches:
669+
only:
670+
- master
671+

‎.circleci/scripts/build_for_windows.sh

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
retry () {
5+
$* || (sleep 1 && $*) || (sleep 2 && $*)
6+
}
7+
8+
SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
9+
PROJECT_DIR="${SOURCE_DIR}/../.."
10+
pushd $SOURCE_DIR
11+
12+
#install wget and make
13+
curl -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
14+
unzip mingw32.zip -d mingw32
15+
mingw32/bin/mingw-get.exe install mingw32-make
16+
mingw32/bin/mingw-get.exe install msys-findutils
17+
mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
18+
curl -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
19+
export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
20+
21+
#install anaconda3
22+
export CONDA_HOME="${SOURCE_DIR}/conda"
23+
export tmp_conda="${SOURCE_DIR}/conda"
24+
export miniconda_exe="${SOURCE_DIR}/miniconda.exe"
25+
rm -rf conda miniconda.exe
26+
curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
27+
./install_conda.bat
28+
export PATH="${tmp_conda}:${tmp_conda}/Library/usr/bin:${tmp_conda}/Library/bin:${tmp_conda}/Scripts:${tmp_conda}/bin:$PATH"
29+
30+
eval "$(conda shell.bash hook)"
31+
conda create -qyn testenv python=3.7
32+
conda activate testenv
33+
34+
REQUIREMENTS="$(grep -v '^ *#\|^torch\|^torchaudio\|^torchvision|^torchtext' $PROJECT_DIR/requirements.txt | grep .)"
35+
echo $REQUIREMENTS > requirements.txt
36+
pip install -r requirements.txt
37+
pip install pySoundFile
38+
# Force uninstall torch & related packages, we'll install them using conda later.
39+
pip uninstall -y torch torchvision torchtext
40+
conda install -yq -c pytorch "cudatoolkit=10.1" pytorch torchvision torchtext torchaudio
41+
python -m spacy download de
42+
python -m spacy download en
43+
pushd ${PROJECT_DIR}
44+
DIR=.jenkins
45+
export NUM_WORKERS=4
46+
47+
if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
48+
python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
49+
python $DIR/remove_runnable_code.py advanced_source/static_quantization_tutorial.py advanced_source/static_quantization_tutorial.py || true
50+
python $DIR/remove_runnable_code.py beginner_source/hyperparameter_tuning_tutorial.py beginner_source/hyperparameter_tuning_tutorial.py || true
51+
python $DIR/remove_runnable_code.py beginner_source/audio_preprocessing_tutorial.py beginner_source/audio_preprocessing_tutorial.py || true
52+
python $DIR/remove_runnable_code.py beginner_source/dcgan_faces_tutorial.py beginner_source/dcgan_faces_tutorial.py || true
53+
python $DIR/remove_runnable_code.py intermediate_source/tensorboard_profiler_tutorial.py intermediate_source/tensorboard_profiler_tutorial.py || true
54+
# Temp remove for mnist download issue. (Re-enabled for 1.8.1)
55+
# python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py beginner_source/fgsm_tutorial.py || true
56+
57+
export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
58+
count=0
59+
FILES_TO_RUN=()
60+
for work_directory in beginner_source intermediate_source advanced_source recipes_source prototype_source; do
61+
for filename in $(find $work_directory -name '\*.py' -not -path '\*/data/\*'); do
62+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
63+
echo "Removing runnable code from "$filename
64+
python $DIR/remove_runnable_code.py $filename $filename
65+
else
66+
echo "Keeping "$filename
67+
FILES_TO_RUN+=($(basename $filename .py))
68+
fi
69+
count=$((count+1))
70+
done
71+
done
72+
echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
73+
fi
74+
75+
if [[ ! -d advanced_source/data || ! -d beginner_source/data || ! -d intermediate_source/data || ! -d prototype_source/data ]];then
76+
make download
77+
fi
78+
79+
make html

‎.circleci/scripts/install_conda.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%

‎.circleci/scripts/windows_cuda_install.sh

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
5+
7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
6+
cd cuda_10.1.243_426.00_win10
7+
mkdir cuda_install_logs
8+
9+
set +e
10+
11+
./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
12+
13+
set -e
14+
15+
curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
16+
7z x NvToolsExt.7z -oNvToolsExt
17+
mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
18+
cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
19+
export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
20+
21+
if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
22+
then
23+
echo "CUDA installation failed"
24+
mkdir -p /c/w/build-results
25+
7z a "c:\\w\\build-results\\cuda_install_logs.7z" cuda_install_logs
26+
exit 1
27+
fi
28+
29+
cd ..
30+
rm -rf ./cuda_10.1.243_426.00_win10
31+
rm -f ./cuda_10.1.243_426.00_win10.exe

‎.devcontainer/Dockerfile

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FROM mcr.microsoft.com/vscode/devcontainers/python:3.8
2+
3+
COPY requirements.txt /tmp/pip-tmp/
4+
5+
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
6+
&& apt-get install git gcc unzip make -y \
7+
&& pip3 install --disable-pip-version-check --no-cache-dir -r /tmp/pip-tmp/requirements.txt \
8+
&& rm -rf /tmp/pip-tmp

‎.devcontainer/devcontainer.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"name": "PyTorch Tutorials",
3+
"build": {
4+
"context": "..",
5+
"dockerfile": "Dockerfile",
6+
"args": {}
7+
},
8+
"settings": {
9+
"terminal.integrated.shell.linux": "/bin/bash",
10+
"workbench.startupEditor": "none",
11+
"files.autoSave": "afterDelay",
12+
"python.dataScience.enabled": true,
13+
"python.dataScience.alwaysTrustNotebooks": true,
14+
"python.insidersChannel": "weekly",
15+
"python.showStartPage": false
16+
},
17+
"extensions": ["ms-python.python", "lextudio.restructuredtext"]
18+
}

‎.devcontainer/requirements.txt

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Refer to ./jenkins/build.sh for tutorial build instructions
2+
3+
sphinx==1.8.2
4+
sphinx-gallery==0.3.1
5+
tqdm
6+
numpy
7+
matplotlib
8+
torch
9+
torchvision
10+
torchtext
11+
torchaudio
12+
PyHamcrest
13+
bs4
14+
awscli==1.16.35
15+
flask
16+
spacy
17+
ray[tune]
18+
19+
# PyTorch Theme
20+
-e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
21+
22+
ipython
23+
24+
# to run examples
25+
pandas
26+
scikit-image
27+
pillow==8.1.1
28+
wget
29+
30+
# for codespaces env
31+
pylint

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ intermediate
44
advanced
55
pytorch_basics
66
recipes
7+
prototype
78

89
#data things
910
_data/
@@ -117,3 +118,6 @@ ENV/
117118
.DS_Store
118119
cleanup.sh
119120
*.swp
121+
122+
# PyTorch things
123+
*.pt

0 commit comments

Comments
 (0)