Merge branch 'master' into translation_transformer_fix

Author: Svetlana Karslioglu

.circleci/config.yml

@@ -26,9 +26,6 @@ setup_linux_system_environment: &setup_linux_system_environment
   command: |
     set -ex
 
-    # Set up CircleCI GPG keys for apt, if needed
-    curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
-
     # Stop background apt updates.  Hypothetically, the kill should not
     # be necessary, because stop is supposed to send a kill signal to
     # the process, but we've added it for good luck.  Also

.circleci/config.yml.in

@@ -26,9 +26,6 @@ setup_linux_system_environment: &setup_linux_system_environment
   command: |
     set -ex
 
-    # Set up CircleCI GPG keys for apt, if needed
-    curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
-
     # Stop background apt updates.  Hypothetically, the kill should not
     # be necessary, because stop is supposed to send a kill signal to
     # the process, but we've added it for good luck.  Also

.circleci/scripts/build_for_windows.sh

@@ -10,20 +10,20 @@ PROJECT_DIR="${SOURCE_DIR}/../.."
 pushd $SOURCE_DIR
 
 #install wget and make
-curl -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
+curl --retry 3 -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
 unzip mingw32.zip -d mingw32
 mingw32/bin/mingw-get.exe install mingw32-make
 mingw32/bin/mingw-get.exe install msys-findutils
 mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
-curl -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
+curl --retry 3 -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
 export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
 
 #install anaconda3
 export CONDA_HOME="${SOURCE_DIR}/conda"
 export tmp_conda="${SOURCE_DIR}/conda"
 export miniconda_exe="${SOURCE_DIR}/miniconda.exe"
 rm -rf conda miniconda.exe
-curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
+curl --retry 3 -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
 ./install_conda.bat
 export PATH="${tmp_conda}:${tmp_conda}/Library/usr/bin:${tmp_conda}/Library/bin:${tmp_conda}/Scripts:${tmp_conda}/bin:$PATH"
 
@@ -38,8 +38,8 @@ pip install pySoundFile
 # Force uninstall torch & related packages, we'll install them using conda later.
 pip uninstall -y torch torchvision torchtext
 conda install -yq -c pytorch "cudatoolkit=10.2" pytorch torchvision torchtext torchaudio
-python -m spacy download de
-python -m spacy download en
+python -m spacy download de_core_news_sm
+python -m spacy download en_core_web_sm
 pushd ${PROJECT_DIR}
 DIR=.jenkins
 export NUM_WORKERS=4

.jenkins/build.sh

@@ -21,9 +21,6 @@ export PATH=/opt/conda/bin:$PATH
 rm -rf src
 pip install -r $DIR/../requirements.txt
 
-# export PATH=/opt/conda/bin:$PATH
-# pip install sphinx==1.8.2 pandas
-
 #Install PyTorch Nightly for test.
 # Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
 # RC Link
@@ -32,17 +29,9 @@ pip install -r $DIR/../requirements.txt
 # pip uninstall -y torch torchvision torchaudio torchtext
 # pip install -f https://download.pytorch.org/whl/test/cu111/torch_test.html torch torchvision torchaudio torchtext
 
-# For Tensorboard. Until 1.14 moves to the release channel.
-pip install tb-nightly
-
 # Install two language tokenizers for Translation with TorchText tutorial
-python -m spacy download en
-python -m spacy download de
-
-# PyTorch Theme
-rm -rf src
-pip install -e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
-pip install sphinx-gallery==0.3.1 tqdm matplotlib ipython pillow==9.0.1
+python -m spacy download en_core_web_sm
+python -m spacy download de_core_news_sm
 
 awsv2 -i
 awsv2 configure set default.s3.multipart_threshold 5120MB
@@ -63,7 +52,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
   # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true
 
   # Remove runnable code from tensorboard_profiler_tutorial.py as it frequently crashes, see https://github.com/pytorch/pytorch/issues/74139
-  python $DIR/remove_runnable_code.py intermediate_source/tensorboard_profiler_tutorial.py intermediate_source/tensorboard_profiler_tutorial.py || true
+  # python $DIR/remove_runnable_code.py intermediate_source/tensorboard_profiler_tutorial.py intermediate_source/tensorboard_profiler_tutorial.py || true
 
 
   # Step 2: Keep certain tutorials based on file count, and remove runnable code in all other tutorials
@@ -77,45 +66,54 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
 
   # Step 4: If any of the generated files are not related the tutorial files we want to run,
   # then we remove them
+  set +x
   for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.html'); do
     file_basename=$(basename $filename .html)
     if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
+      echo "removing $filename"
       rm $filename
     fi
   done
   for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.rst'); do
     file_basename=$(basename $filename .rst)
     if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
+      echo "removing $filename"
       rm $filename
     fi
   done
   for filename in $(find docs/_downloads -name '*.py'); do
     file_basename=$(basename $filename .py)
     if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
+      echo "removing $filename"
       rm $filename
     fi
   done
   for filename in $(find docs/_downloads -name '*.ipynb'); do
     file_basename=$(basename $filename .ipynb)
     if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
+      echo "removing $filename"
       rm $filename
     fi
   done
   for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced docs/_sources/recipes -name '*.rst.txt'); do
     file_basename=$(basename $filename .rst.txt)
     if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
+      echo "removing $filename"
       rm $filename
     fi
   done
   for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced docs/.doctrees/recipes docs/.doctrees/prototype -name '*.doctree'); do
     file_basename=$(basename $filename .doctree)
     if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then
+      echo "removing $filename"
       rm $filename
     fi
   done
+  set -x
 
   # Step 5: Remove INVISIBLE_CODE_BLOCK from .html/.rst.txt/.ipynb/.py files
   bash $DIR/remove_invisible_code_block_batch.sh docs
+  python .jenkins/validate_tutorials_built.py
 
   # Step 6: Copy generated files to S3, tag with commit ID
   7z a worker_${WORKER_ID}.7z docs
@@ -149,6 +147,7 @@ elif [[ "${JOB_BASE_NAME}" == *manager ]]; then
 
   # Step 5: Remove INVISIBLE_CODE_BLOCK from .html/.rst.txt/.ipynb/.py files
   bash $DIR/remove_invisible_code_block_batch.sh docs
+  python .jenkins/validate_tutorials_built.py
 
   # Step 6: Copy generated HTML files and static files to S3
   7z a manager.7z docs

.jenkins/get_files_to_run.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import Any, Dict, List, Optional, Tuple
 import json
 import os
 from pathlib import Path
@@ -14,17 +14,21 @@ def get_all_files() -> List[str]:
     return [str(x) for x in sources]
 
 
-def calculate_shards(all_files, num_shards=20):
+def read_metadata() -> Dict[str, Any]:
     with (REPO_BASE_DIR / ".jenkins" / "metadata.json").open() as fp:
-        metadata = json.load(fp)
-    sharded_files = [(0.0, []) for _ in range(num_shards)]
+        return json.load(fp)
 
-    def get_duration(file):
+
+def calculate_shards(all_files: List[str], num_shards: int = 20) -> List[List[str]]:
+    sharded_files: List[Tuple[float, List[str]]] = [(0.0, []) for _ in range(num_shards)]
+    metadata = read_metadata()
+
+    def get_duration(file: str) -> int:
         # tutorials not listed in the metadata.json file usually take
         # <3min to run, so we'll default to 1min if it's not listed
         return metadata.get(file, {}).get("duration", 60)
 
-    def get_needs_machine(file):
+    def get_needs_machine(file: str) -> Optional[str]:
         return metadata.get(file, {}).get("needs", None)
 
     def add_to_shard(i, filename):
@@ -55,19 +59,38 @@ def add_to_shard(i, filename):
     return [x[1] for x in sharded_files]
 
 
-def remove_other_files(all_files, files_to_run) -> None:
+def compute_files_to_keep(files_to_run: List[str]) -> List[str]:
+    metadata = read_metadata()
+    files_to_keep = list(files_to_run)
+    for file in files_to_run:
+        extra_files = metadata.get(file, {}).get("extra_files", [])
+        files_to_keep.extend(extra_files)
+    return files_to_keep
+
+
+def remove_other_files(all_files, files_to_keep) -> None:
+
     for file in all_files:
-        if file not in files_to_run:
+        if file not in files_to_keep:
             remove_runnable_code(file, file)
 
 
+def parse_args() -> Any:
+    from argparse import ArgumentParser
+    parser = ArgumentParser("Select files to run")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--num-shards", type=int, default=int(os.environ.get("NUM_WORKERS", 20)))
+    parser.add_argument("--shard-num", type=int, default=int(os.environ.get("WORKER_ID", 0)))
+    return parser.parse_args()
+
+
 def main() -> None:
-    num_shards = int(os.environ.get("NUM_WORKERS", 20))
-    shard_num = int(os.environ.get("WORKER_ID", 0))
+    args = parse_args()
 
     all_files = get_all_files()
-    files_to_run = calculate_shards(all_files, num_shards=num_shards)[shard_num]
-    remove_other_files(all_files, files_to_run)
+    files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num]
+    if not args.dry_run:
+        remove_other_files(all_files, compute_files_to_keep(files_to_run))
     stripped_file_names = [Path(x).stem for x in files_to_run]
     print(" ".join(stripped_file_names))
 

.jenkins/metadata.json

@@ -1,12 +1,16 @@
 {
+  "intermediate_source/ax_multiobjective_nas_tutorial.py": {
+    "extra_files": ["intermediate_source/mnist_train_nas.py"],
+    "duration": 2000
+  },
   "beginner_source/dcgan_faces_tutorial.py": {
     "duration": 2000
   },
   "intermediate_source/seq2seq_translation_tutorial.py": {
     "duration": 1200
   },
   "beginner_source/hyperparameter_tuning_tutorial.py": {
-    "duration": 910
+    "duration": 0
   },
   "advanced_source/dynamic_quantization_tutorial.py": {
     "duration": 380

.jenkins/validate_tutorials_built.py

@@ -0,0 +1,96 @@
+from pathlib import Path
+from typing import List
+
+from bs4 import BeautifulSoup
+
+REPO_ROOT = Path(__file__).parent.parent
+
+# For every tutorial on this list, we should determine if it is ok to not run the tutorial (add a comment after
+# the file name to explain why, like intro.html), or fix the tutorial and remove it from this list).
+
+NOT_RUN = [
+    "basics/intro",  # no code
+    "translation_transformer",
+    "profiler",
+    "saving_loading_models",
+    "introyt/captumyt",
+    "introyt/trainingyt",
+    "examples_nn/polynomial_module",
+    "examples_nn/dynamic_net",
+    "examples_nn/polynomial_optim",
+    "former_torchies/autograd_tutorial_old",
+    "former_torchies/tensor_tutorial_old",
+    "examples_autograd/polynomial_autograd",
+    "examples_autograd/polynomial_custom_function",
+    "parametrizations",
+    "mnist_train_nas",  # used by ax_multiobjective_nas_tutorial.py
+    "fx_conv_bn_fuser",
+    "super_resolution_with_onnxruntime",
+    "ddp_pipeline",  # requires 4 gpus
+    "fx_graph_mode_ptq_dynamic",
+    "vmap_recipe",
+    "torchscript_freezing",
+    "nestedtensor",
+    "recipes/saving_and_loading_models_for_inference",
+    "recipes/saving_multiple_models_in_one_file",
+    "recipes/loading_data_recipe",
+    "recipes/tensorboard_with_pytorch",
+    "recipes/what_is_state_dict",
+    "recipes/profiler_recipe",
+    "recipes/save_load_across_devices",
+    "recipes/warmstarting_model_using_parameters_from_a_different_model",
+    "recipes/dynamic_quantization",
+    "recipes/saving_and_loading_a_general_checkpoint",
+    "recipes/benchmark",
+    "recipes/tuning_guide",
+    "recipes/zeroing_out_gradients",
+    "recipes/defining_a_neural_network",
+    "recipes/timer_quick_start",
+    "recipes/amp_recipe",
+    "recipes/Captum_Recipe",
+    "hyperparameter_tuning_tutorial",
+    "flask_rest_api_tutorial",
+]
+
+
+def tutorial_source_dirs() -> List[Path]:
+    return [
+        p.relative_to(REPO_ROOT).with_name(p.stem[:-7])
+        for p in REPO_ROOT.glob("*_source")
+    ]
+
+
+def main() -> None:
+    docs_dir = REPO_ROOT / "docs"
+    html_file_paths = []
+    for tutorial_source_dir in tutorial_source_dirs():
+        glob_path = f"{tutorial_source_dir}/**/*.html"
+        html_file_paths += docs_dir.glob(glob_path)
+
+    did_not_run = []
+    for html_file_path in html_file_paths:
+        with open(html_file_path, "r", encoding="utf-8") as html_file:
+            html = html_file.read()
+        html_soup = BeautifulSoup(html, "html.parser")
+        elems = html_soup.find_all("p", {"class": "sphx-glr-timing"})
+        for elem in elems:
+            if (
+                "Total running time of the script: ( 0 minutes  0.000 seconds)"
+                in elem.text
+                and not any(
+                    html_file_path.match(file) for file in NOT_RUN
+                )
+            ):
+                did_not_run.append(html_file_path.as_posix())
+
+    if len(did_not_run) != 0:
+        raise RuntimeError(
+            "The following file(s) are not known bad but ran in 0.000 sec, meaning that any "
+            + "python code in this tutorial probably didn't run:\n{}".format(
+                "\n".join(did_not_run)
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()