Skip to content

Commit 84c6357

Browse files
jcaipjcaip
authored
Merge branch 'master' into quant-fix
2 parents 335108c + 4918fe4 commit 84c6357

File tree

141 files changed

+7334
-2697
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

141 files changed

+7334
-2697
lines changed

‎.circleci/config.yml

Lines changed: 144 additions & 180 deletions
Large diffs are not rendered by default.

‎.circleci/config.yml.in

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
# run python regenerate.py to generate config.yml from config.yml.in
2+
3+
version: 2.1
4+
5+
executors:
6+
windows-with-nvidia-gpu:
7+
machine:
8+
resource_class: windows.gpu.nvidia.medium
9+
image: windows-server-2019-nvidia:stable
10+
shell: bash.exe
11+
12+
install_official_git_client: &install_official_git_client
13+
name: Install Official Git Client
14+
no_output_timeout: "1h"
15+
command: |
16+
set -e
17+
sudo apt-get -qq update
18+
sudo apt-get -qq install openssh-client git
19+
20+
# This system setup script is meant to run before the CI-related scripts, e.g.,
21+
# installing Git client, checking out code, setting up CI env, and
22+
# building/testing.
23+
setup_linux_system_environment: &setup_linux_system_environment
24+
name: Set Up System Environment
25+
no_output_timeout: "1h"
26+
command: |
27+
set -ex
28+
29+
# Stop background apt updates. Hypothetically, the kill should not
30+
# be necessary, because stop is supposed to send a kill signal to
31+
# the process, but we've added it for good luck. Also
32+
# hypothetically, it's supposed to be unnecessary to wait for
33+
# the process to block. We also have that line for good luck.
34+
# If you like, try deleting them and seeing if it works.
35+
sudo systemctl stop apt-daily.service || true
36+
sudo systemctl kill --kill-who=all apt-daily.service || true
37+
38+
sudo systemctl stop unattended-upgrades.service || true
39+
sudo systemctl kill --kill-who=all unattended-upgrades.service || true
40+
41+
# wait until `apt-get update` has been killed
42+
while systemctl is-active --quiet apt-daily.service
43+
do
44+
sleep 1;
45+
done
46+
while systemctl is-active --quiet unattended-upgrades.service
47+
do
48+
sleep 1;
49+
done
50+
51+
# See if we actually were successful
52+
systemctl list-units --all | cat
53+
54+
sudo apt-get purge -y unattended-upgrades
55+
56+
cat /etc/apt/sources.list
57+
58+
ps auxfww | grep [a]pt
59+
ps auxfww | grep dpkg
60+
61+
pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
62+
machine:
63+
image: ubuntu-2004-cuda-11.4:202110-01
64+
steps:
65+
- checkout
66+
- run:
67+
<<: *setup_linux_system_environment
68+
- run:
69+
name: Set Up CI Environment
70+
no_output_timeout: "1h"
71+
command: |
72+
set -e
73+
74+
sudo apt-get -y update
75+
sudo apt-get -y install expect-dev moreutils
76+
77+
sudo pip3 -q install awscli==1.16.35
78+
79+
if [ -n "${CUDA_VERSION}" ]; then
80+
nvidia-smi
81+
fi
82+
83+
# This IAM user only allows read-write access to ECR
84+
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_ONLY}
85+
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY}
86+
eval $(aws ecr get-login --region us-east-1 --no-include-email)
87+
- run:
88+
name: Build
89+
no_output_timeout: "20h"
90+
command: |
91+
set -e
92+
93+
# for some reason, pip installs it in a different place than what is looked at in the py file
94+
sudo pip3 install requests --target=/opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages
95+
export pyTorchDockerImageTag=$(python3 .jenkins/get_docker_tag.py)
96+
echo "PyTorchDockerImageTag: "${pyTorchDockerImageTag}
97+
98+
cat >/home/circleci/project/ci_build_script.sh \<<EOL
99+
# =================== The following code will be executed inside Docker container ===================
100+
set -ex
101+
102+
.jenkins/build.sh
103+
# =================== The above code will be executed inside Docker container ===================
104+
EOL
105+
chmod +x /home/circleci/project/ci_build_script.sh
106+
107+
export DOCKER_IMAGE=${DOCKER_IMAGE}:${pyTorchDockerImageTag}
108+
echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
109+
docker pull ${DOCKER_IMAGE} >/dev/null
110+
if [ -n "${CUDA_VERSION}" ]; then
111+
export id=$(docker run --gpus all -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
112+
else
113+
export id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
114+
fi
115+
116+
echo "declare -x JOB_BASE_NAME=${CIRCLE_JOB}" > /home/circleci/project/env
117+
echo "declare -x COMMIT_ID=${CIRCLE_SHA1}" >> /home/circleci/project/env
118+
echo "declare -x COMMIT_SOURCE=${CIRCLE_BRANCH}" >> /home/circleci/project/env
119+
# DANGER! DO NOT REMOVE THE `set +x` SETTING HERE!
120+
set +x
121+
if [[ "$CIRCLE_BRANCH" == master ]]; then
122+
if [ -z "${CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" ]; then exit 1; fi
123+
if [ -z "${CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" ]; then exit 1; fi
124+
if [ -z "${GITHUB_PYTORCHBOT_USERNAME}" ]; then exit 1; fi
125+
if [ -z "${GITHUB_PYTORCHBOT_TOKEN}" ]; then exit 1; fi
126+
127+
echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" >> /home/circleci/project/env
128+
echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" >> /home/circleci/project/env
129+
echo "declare -x GITHUB_PYTORCHBOT_USERNAME=${GITHUB_PYTORCHBOT_USERNAME}" >> /home/circleci/project/env
130+
echo "declare -x GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" >> /home/circleci/project/env
131+
else
132+
echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET}" >> /home/circleci/project/env
133+
echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET}" >> /home/circleci/project/env
134+
fi
135+
set -x
136+
137+
docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
138+
139+
export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
140+
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
141+
142+
pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults
143+
environment:
144+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
145+
CUDA_VERSION: "9"
146+
resource_class: gpu.nvidia.small
147+
<<: *pytorch_tutorial_build_defaults
148+
149+
pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_defaults
150+
environment:
151+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
152+
resource_class: medium
153+
<<: *pytorch_tutorial_build_defaults
154+
{% raw %}
155+
pytorch_windows_build_worker: &pytorch_windows_build_worker
156+
executor: windows-with-nvidia-gpu
157+
steps:
158+
- checkout
159+
- run:
160+
name: Install Cuda
161+
no_output_timeout: 30m
162+
command: |
163+
.circleci/scripts/windows_cuda_install.sh
164+
- run:
165+
name: Generate cache key
166+
# This will refresh cache on Sundays, build should generate new cache.
167+
command: echo "$(date +"%Y-%U")" > .circleci-weekly
168+
- restore_cache:
169+
keys:
170+
- data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
171+
- run:
172+
name: test
173+
no_output_timeout: "1h"
174+
command: |
175+
.circleci/scripts/build_for_windows.sh
176+
- save_cache:
177+
key: data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
178+
paths:
179+
- advanced_source/data
180+
- beginner_source/data
181+
- intermediate_source/data
182+
- prototype_source/data
183+
{% endraw %}
184+
jobs:
185+
{{ jobs("pr") }}
186+
187+
{{ jobs("master") }}
188+
189+
{{ windows_jobs() }}
190+
191+
workflows:
192+
build:
193+
jobs:
194+
# Build jobs that only run on PR
195+
{{ workflows_jobs("pr") }}
196+
# Build jobs that only run on master
197+
{{ workflows_jobs("master") }}
198+
# {{ windows_workflows_jobs() }}

‎.circleci/regenerate.py

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env python3
2+
3+
# regenrates config.yml based on config.yml.in
4+
5+
from copy import deepcopy
6+
import os.path
7+
8+
import jinja2
9+
import yaml
10+
from jinja2 import select_autoescape
11+
12+
WORKFLOWS_JOBS_PR = {"filters": {"branches": {"ignore": ["master"]}}}
13+
14+
WORKFLOWS_JOBS_MASTER = {
15+
"context": "org-member",
16+
"filters": {"branches": {"only": ["master"]}},
17+
}
18+
19+
20+
def indent(indentation, data_list):
21+
return ("\n" + " " * indentation).join(
22+
yaml.dump(data_list, default_flow_style=False).splitlines()
23+
)
24+
25+
26+
def jobs(pr_or_master, num_workers=20, indentation=2):
27+
jobs = {}
28+
29+
# all tutorials that need gpu.nvidia.small.multi machines will be routed
30+
# by get_files_to_run.py to 0th worker
31+
needs_gpu_nvidia_small_multi = [0]
32+
jobs[f"pytorch_tutorial_{pr_or_master}_build_manager"] = {
33+
"<<": "*pytorch_tutorial_build_manager_defaults"
34+
}
35+
for i in range(num_workers):
36+
job_info = {"<<": "*pytorch_tutorial_build_worker_defaults"}
37+
if i in needs_gpu_nvidia_small_multi:
38+
job_info["resource_class"] = "gpu.nvidia.small.multi"
39+
jobs[f"pytorch_tutorial_{pr_or_master}_build_worker_{i}"] = job_info
40+
41+
return indent(indentation, jobs).replace("'", "")
42+
43+
44+
def workflows_jobs(pr_or_master, indentation=6, num_workers=20):
45+
jobs = []
46+
job_info = deepcopy(
47+
WORKFLOWS_JOBS_PR if pr_or_master == "pr" else WORKFLOWS_JOBS_MASTER
48+
)
49+
50+
for i in range(num_workers):
51+
jobs.append(
52+
{f"pytorch_tutorial_{pr_or_master}_build_worker_{i}": deepcopy(job_info)}
53+
)
54+
55+
job_info["requires"] = [
56+
f"pytorch_tutorial_{pr_or_master}_build_worker_{i}" for i in range(num_workers)
57+
]
58+
jobs.append({f"pytorch_tutorial_{pr_or_master}_build_manager": deepcopy(job_info)})
59+
return indent(indentation, jobs)
60+
61+
62+
def windows_jobs(indentation=2, num_workers=4):
63+
jobs = {}
64+
for i in range(num_workers):
65+
jobs[f"pytorch_tutorial_windows_pr_build_worker_{i}"] = {
66+
"<<": "*pytorch_windows_build_worker"
67+
}
68+
jobs[f"pytorch_tutorial_windows_master_build_worker_{i}"] = {
69+
"<<": "*pytorch_windows_build_worker"
70+
}
71+
return indent(indentation, jobs).replace("'", "")
72+
73+
74+
def windows_workflows_jobs(indentation=6, num_workers=4):
75+
jobs = []
76+
job_info = WORKFLOWS_JOBS_PR
77+
for i in range(num_workers):
78+
jobs.append(
79+
{f"pytorch_tutorial_windows_pr_build_worker_{i}": deepcopy(job_info)}
80+
)
81+
82+
job_info = WORKFLOWS_JOBS_MASTER
83+
for i in range(num_workers):
84+
jobs.append(
85+
{f"pytorch_tutorial_windows_master_build_worker_{i}": deepcopy(job_info)}
86+
)
87+
88+
return ("\n#").join(indent(indentation, jobs).splitlines())
89+
90+
91+
if __name__ == "__main__":
92+
93+
directory = os.path.dirname(__file__)
94+
env = jinja2.Environment(
95+
loader=jinja2.FileSystemLoader(directory),
96+
lstrip_blocks=True,
97+
autoescape=select_autoescape(enabled_extensions=("html", "xml")),
98+
keep_trailing_newline=True,
99+
)
100+
with open(os.path.join(directory, "config.yml"), "w") as f:
101+
f.write(
102+
env.get_template("config.yml.in").render(
103+
jobs=jobs,
104+
workflows_jobs=workflows_jobs,
105+
windows_jobs=windows_jobs,
106+
windows_workflows_jobs=windows_workflows_jobs,
107+
)
108+
)

‎.circleci/scripts/build_for_windows.sh

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,20 @@ PROJECT_DIR="${SOURCE_DIR}/../.."
1010
pushd $SOURCE_DIR
1111

1212
#install wget and make
13-
curl -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
13+
curl --retry 3 -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
1414
unzip mingw32.zip -d mingw32
1515
mingw32/bin/mingw-get.exe install mingw32-make
1616
mingw32/bin/mingw-get.exe install msys-findutils
1717
mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
18-
curl -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
18+
curl --retry 3 -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
1919
export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
2020

2121
#install anaconda3
2222
export CONDA_HOME="${SOURCE_DIR}/conda"
2323
export tmp_conda="${SOURCE_DIR}/conda"
2424
export miniconda_exe="${SOURCE_DIR}/miniconda.exe"
2525
rm -rf conda miniconda.exe
26-
curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
26+
curl --retry 3 -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
2727
./install_conda.bat
2828
export PATH="${tmp_conda}:${tmp_conda}/Library/usr/bin:${tmp_conda}/Library/bin:${tmp_conda}/Scripts:${tmp_conda}/bin:$PATH"
2929

@@ -38,8 +38,8 @@ pip install pySoundFile
3838
# Force uninstall torch & related packages, we'll install them using conda later.
3939
pip uninstall -y torch torchvision torchtext
4040
conda install -yq -c pytorch "cudatoolkit=10.2" pytorch torchvision torchtext torchaudio
41-
python -m spacy download de
42-
python -m spacy download en
41+
python -m spacy download de_core_news_sm
42+
python -m spacy download en_core_web_sm
4343
pushd ${PROJECT_DIR}
4444
DIR=.jenkins
4545
export NUM_WORKERS=4

‎.devcontainer/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ ipython
2424
# to run examples
2525
pandas
2626
scikit-image
27-
pillow==9.0.0
27+
pillow==9.0.1
2828
wget
2929

3030
# for codespaces env

‎.github/pytorch-probot.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
tracking_issue: 1896

0 commit comments

Comments
 (0)