Skip to content

Commit c0eda34

Browse files
guyang3532guyang3532
committed
add windows build
1 parent 32e5407 commit c0eda34

File tree

5 files changed

+217
-13
lines changed

5 files changed

+217
-13
lines changed

‎.circleci/config.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
version: 2.1
22

3+
executors:
4+
windows-with-nvidia-gpu:
5+
machine:
6+
resource_class: windows.gpu.nvidia.medium
7+
image: windows-server-2019-nvidia:stable
8+
shell: bash.exe
9+
310
install_official_git_client: &install_official_git_client
411
name: Install Official Git Client
512
no_output_timeout: "1h"
@@ -175,6 +182,20 @@ pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_default
175182
resource_class: medium
176183
<<: *pytorch_tutorial_build_defaults
177184

185+
pytorch_windows_build: &pytorch_windows_build
186+
executor: windows-with-nvidia-gpu
187+
steps:
188+
- checkout
189+
- run:
190+
name: Install Cuda
191+
no_output_timeout: 30m
192+
command: |
193+
.circleci/scripts/windows_cuda_install.sh
194+
- run:
195+
name: test
196+
command: |
197+
.circleci/scripts/build_for_windows.sh
198+
178199
jobs:
179200
pytorch_tutorial_pr_build_worker_0:
180201
<<: *pytorch_tutorial_build_worker_defaults
@@ -302,6 +323,30 @@ jobs:
302323
pytorch_tutorial_master_build_manager:
303324
<<: *pytorch_tutorial_build_manager_defaults
304325

326+
pytorch_windows_build_worker_0:
327+
<<: *pytorch_windows_build
328+
329+
pytorch_windows_build_worker_1:
330+
<<: *pytorch_windows_build
331+
332+
pytorch_windows_build_worker_2:
333+
<<: *pytorch_windows_build
334+
335+
pytorch_windows_build_worker_3:
336+
<<: *pytorch_windows_build
337+
338+
pytorch_windows_build_worker_4:
339+
<<: *pytorch_windows_build
340+
341+
pytorch_windows_build_worker_5:
342+
<<: *pytorch_windows_build
343+
344+
pytorch_windows_build_worker_6:
345+
<<: *pytorch_windows_build
346+
347+
pytorch_windows_build_worker_7:
348+
<<: *pytorch_windows_build
349+
305350
workflows:
306351
build:
307352
jobs:
@@ -538,3 +583,19 @@ workflows:
538583
branches:
539584
only:
540585
- master
586+
- pytorch_windows_build_worker_0:
587+
name: win_test_worker_0
588+
- pytorch_windows_build_worker_1:
589+
name: win_test_worker_1
590+
- pytorch_windows_build_worker_2:
591+
name: win_test_worker_2
592+
- pytorch_windows_build_worker_3:
593+
name: win_test_worker_3
594+
- pytorch_windows_build_worker_4:
595+
name: win_test_worker_4
596+
- pytorch_windows_build_worker_5:
597+
name: win_test_worker_5
598+
- pytorch_windows_build_worker_6:
599+
name: win_test_worker_6
600+
- pytorch_windows_build_worker_7:
601+
name: win_test_worker_7

‎.circleci/scripts/build_for_windows.sh

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
retry () {
5+
$* || (sleep 1 && $*) || (sleep 2 && $*)
6+
}
7+
8+
SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
9+
PROJECT_DIR="${SOURCE_DIR}/../.."
10+
pushd $SOURCE_DIR
11+
12+
#install wget and make
13+
curl -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
14+
unzip mingw32.zip -d mingw32
15+
mingw32/bin/mingw-get.exe install mingw32-make
16+
mingw32/bin/mingw-get.exe install msys-wget
17+
mingw32/bin/mingw-get.exe install msys-findutils
18+
mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
19+
export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
20+
21+
#install anaconda3
22+
export CONDA_HOME="${SOURCE_DIR}/conda"
23+
export tmp_conda="${SOURCE_DIR}/conda"
24+
export miniconda_exe="${SOURCE_DIR}/miniconda.exe"
25+
rm -rf conda miniconda.exe
26+
curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
27+
./install_conda.bat
28+
export PATH="${tmp_conda}:${tmp_conda}/Library/usr/bin:${tmp_conda}/Library/bin:${tmp_conda}/Scripts:${tmp_conda}/bin:$PATH"
29+
30+
eval "$(conda shell.bash hook)"
31+
conda create -qyn testenv python=3.7
32+
conda activate testenv
33+
34+
conda install sphinx
35+
conda install cmake
36+
pip install sphinx_gallery==0.3.1 flask pandas spacy ipython scipy pySoundFile scikit-image
37+
pip install -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
38+
conda install -yq -c pytorch "cudatoolkit=10.1" pytorch torchvision torchtext
39+
conda install torchaudio -c pytorch-nightly
40+
python -m spacy download de
41+
python -m spacy download en
42+
pushd ${PROJECT_DIR}
43+
DIR=.jenkins
44+
python $DIR/remove_runnable_code.py beginner_source/aws_distributed_training_tutorial.py beginner_source/aws_distributed_training_tutorial.py || true
45+
python $DIR/remove_runnable_code.py beginner_source/data_loading_tutorial.py beginner_source/data_loading_tutorial.py || true
46+
python $DIR/remove_runnable_code.py beginner_source/dcgan_faces_tutorial.py beginner_source/dcgan_faces_tutorial.py || true
47+
python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
48+
49+
# Decide whether to parallelize tutorial builds, based on $CIRCLE_JOB
50+
export NUM_WORKERS=8
51+
echo $CIRCLE_JOB
52+
if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
53+
export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
54+
count=0
55+
FILES_TO_RUN=()
56+
for filename in $(find beginner_source/ -name '\*.py' -not -path '*/data/*'); do
57+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
58+
echo "Removing runnable code from "$filename
59+
python $DIR/remove_runnable_code.py $filename $filename
60+
else
61+
echo "Keeping "$filename
62+
FILES_TO_RUN+=($(basename $filename .py))
63+
fi
64+
count=$((count+1))
65+
done
66+
for filename in $(find intermediate_source/ -name '\*.py' -not -path '*/data/*'); do
67+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
68+
echo "Removing runnable code from "$filename
69+
python $DIR/remove_runnable_code.py $filename $filename
70+
else
71+
echo "Keeping "$filename
72+
FILES_TO_RUN+=($(basename $filename .py))
73+
fi
74+
count=$((count+1))
75+
done
76+
for filename in $(find advanced_source/ -name '\*.py' -not -path '*/data/*'); do
77+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
78+
echo "Removing runnable code from "$filename
79+
python $DIR/remove_runnable_code.py $filename $filename
80+
else
81+
echo "Keeping "$filename
82+
FILES_TO_RUN+=($(basename $filename .py))
83+
fi
84+
count=$((count+1))
85+
done
86+
for filename in $(find recipes_source/ -name '\*.py' -not -path '*/data/*'); do
87+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
88+
echo "Removing runnable code from "$filename
89+
python $DIR/remove_runnable_code.py $filename $filename
90+
else
91+
echo "Keeping "$filename
92+
FILES_TO_RUN+=($(basename $filename .py))
93+
fi
94+
count=$((count+1))
95+
done
96+
for filename in $(find prototype_source/ -name '\*.py' -not -path '*/data/*'); do
97+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
98+
echo "Removing runnable code from "$filename
99+
python $DIR/remove_runnable_code.py $filename $filename
100+
else
101+
echo "Keeping "$filename
102+
FILES_TO_RUN+=($(basename $filename .py))
103+
fi
104+
count=$((count+1))
105+
done
106+
echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
107+
make docs
108+
fi
109+
110+
111+

‎.circleci/scripts/install_conda.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%

‎.circleci/scripts/windows_cuda_install.sh

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
5+
7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
6+
cd cuda_10.1.243_426.00_win10
7+
mkdir cuda_install_logs
8+
9+
set +e
10+
11+
./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
12+
13+
set -e
14+
15+
curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
16+
7z x NvToolsExt.7z -oNvToolsExt
17+
mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
18+
cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
19+
export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
20+
21+
if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
22+
then
23+
echo "CUDA installation failed"
24+
mkdir -p /c/w/build-results
25+
7z a "c:\\w\\build-results\\cuda_install_logs.7z" cuda_install_logs
26+
exit 1
27+
fi
28+
29+
cd ..
30+
rm -rf ./cuda_10.1.243_426.00_win10
31+
rm -f ./cuda_10.1.243_426.00_win10.exe

‎Makefile

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,18 +45,18 @@ download:
4545
mkdir -p intermediate_source/data
4646

4747
# transfer learning tutorial data
48-
wget -N https://download.pytorch.org/tutorial/hymenoptera_data.zip -P $(DATADIR)
48+
wget -N http://download.pytorch.org/tutorial/hymenoptera_data.zip -P $(DATADIR)
4949
unzip $(ZIPOPTS) $(DATADIR)/hymenoptera_data.zip -d beginner_source/data/
5050

5151
# nlp tutorial data
52-
wget -N https://download.pytorch.org/tutorial/data.zip -P $(DATADIR)
52+
wget -N http://download.pytorch.org/tutorial/data.zip -P $(DATADIR)
5353
unzip $(ZIPOPTS) $(DATADIR)/data.zip -d intermediate_source/ # This will unzip all files in data.zip to intermediate_source/data/ folder
5454

5555
# data loader tutorial
56-
wget -N https://download.pytorch.org/tutorial/faces.zip -P $(DATADIR)
56+
wget -N http://download.pytorch.org/tutorial/faces.zip -P $(DATADIR)
5757
unzip $(ZIPOPTS) $(DATADIR)/faces.zip -d beginner_source/data/
5858

59-
wget -N https://download.pytorch.org/models/tutorials/4000_checkpoint.tar -P $(DATADIR)
59+
wget -N http://download.pytorch.org/models/tutorials/4000_checkpoint.tar -P $(DATADIR)
6060
cp $(DATADIR)/4000_checkpoint.tar beginner_source/data/
6161

6262
# neural style images
@@ -65,39 +65,39 @@ download:
6565
cp -r _static/img/neural-style/ advanced_source/data/images/
6666

6767
# Download dataset for beginner_source/dcgan_faces_tutorial.py
68-
wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/img_align_celeba.zip -P $(DATADIR)
68+
wget -N http://s3.amazonaws.com/pytorch-tutorial-assets/img_align_celeba.zip -P $(DATADIR)
6969
unzip $(ZIPOPTS) $(DATADIR)/img_align_celeba.zip -d beginner_source/data/celeba
7070

7171
# Download dataset for beginner_source/hybrid_frontend/introduction_to_hybrid_frontend_tutorial.py
72-
wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/iris.data -P $(DATADIR)
72+
wget -N http://s3.amazonaws.com/pytorch-tutorial-assets/iris.data -P $(DATADIR)
7373
cp $(DATADIR)/iris.data beginner_source/data/
7474

7575
# Download dataset for beginner_source/chatbot_tutorial.py
76-
wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/cornell_movie_dialogs_corpus.zip -P $(DATADIR)
76+
wget -N http://s3.amazonaws.com/pytorch-tutorial-assets/cornell_movie_dialogs_corpus.zip -P $(DATADIR)
7777
unzip $(ZIPOPTS) $(DATADIR)/cornell_movie_dialogs_corpus.zip -d beginner_source/data/
7878

7979
# Download dataset for beginner_source/audio_classifier_tutorial.py
80-
wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/UrbanSound8K.tar.gz -P $(DATADIR)
80+
wget -N http://s3.amazonaws.com/pytorch-tutorial-assets/UrbanSound8K.tar.gz -P $(DATADIR)
8181
tar $(TAROPTS) -xzf $(DATADIR)/UrbanSound8K.tar.gz -C ./beginner_source/data/
8282

8383
# Download model for beginner_source/fgsm_tutorial.py
84-
wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/lenet_mnist_model.pth -P $(DATADIR)
84+
wget -N http://s3.amazonaws.com/pytorch-tutorial-assets/lenet_mnist_model.pth -P $(DATADIR)
8585
cp $(DATADIR)/lenet_mnist_model.pth ./beginner_source/data/lenet_mnist_model.pth
8686

8787
# Download model for advanced_source/dynamic_quantization_tutorial.py
88-
wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/word_language_model_quantize.pth -P $(DATADIR)
88+
wget -N http://s3.amazonaws.com/pytorch-tutorial-assets/word_language_model_quantize.pth -P $(DATADIR)
8989
cp $(DATADIR)/word_language_model_quantize.pth advanced_source/data/word_language_model_quantize.pth
9090

9191
# Download data for advanced_source/dynamic_quantization_tutorial.py
92-
wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/wikitext-2.zip -P $(DATADIR)
92+
wget -N http://s3.amazonaws.com/pytorch-tutorial-assets/wikitext-2.zip -P $(DATADIR)
9393
unzip $(ZIPOPTS) $(DATADIR)/wikitext-2.zip -d advanced_source/data/
9494

9595
# Download model for advanced_source/static_quantization_tutorial.py
96-
wget -N https://download.pytorch.org/models/mobilenet_v2-b0353104.pth -P $(DATADIR)
96+
wget -N http://download.pytorch.org/models/mobilenet_v2-b0353104.pth -P $(DATADIR)
9797
cp $(DATADIR)/mobilenet_v2-b0353104.pth advanced_source/data/mobilenet_pretrained_float.pth
9898

9999
# Download dataset for advanced_source/static_quantization_tutorial.py
100-
wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/imagenet_1k.zip -P $(DATADIR)
100+
wget -N http://s3.amazonaws.com/pytorch-tutorial-assets/imagenet_1k.zip -P $(DATADIR)
101101
unzip $(ZIPOPTS) $(DATADIR)/imagenet_1k.zip -d advanced_source/data/
102102

103103
docs:

0 commit comments

Comments
 (0)