diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 50535cdab58..3aca5ba2a01 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -66,42 +66,50 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # Step 4: If any of the generated files are not related the tutorial files we want to run, # then we remove them + set +x for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.html'); do file_basename=$(basename $filename .html) if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then + echo "removing $filename" rm $filename fi done for filename in $(find docs/beginner docs/intermediate docs/advanced docs/recipes docs/prototype -name '*.rst'); do file_basename=$(basename $filename .rst) if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then + echo "removing $filename" rm $filename fi done for filename in $(find docs/_downloads -name '*.py'); do file_basename=$(basename $filename .py) if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then + echo "removing $filename" rm $filename fi done for filename in $(find docs/_downloads -name '*.ipynb'); do file_basename=$(basename $filename .ipynb) if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then + echo "removing $filename" rm $filename fi done for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced docs/_sources/recipes -name '*.rst.txt'); do file_basename=$(basename $filename .rst.txt) if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then + echo "removing $filename" rm $filename fi done for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced docs/.doctrees/recipes docs/.doctrees/prototype -name '*.doctree'); do file_basename=$(basename $filename .doctree) if [[ ! " ${FILES_TO_RUN} " =~ " ${file_basename} " ]]; then + echo "removing $filename" rm $filename fi done + set -x # Step 5: Remove INVISIBLE_CODE_BLOCK from .html/.rst.txt/.ipynb/.py files bash $DIR/remove_invisible_code_block_batch.sh docs diff --git a/.jenkins/metadata.json b/.jenkins/metadata.json index 5f933f161f8..095e2a65ccb 100644 --- a/.jenkins/metadata.json +++ b/.jenkins/metadata.json @@ -1,6 +1,7 @@ { "intermediate_source/ax_multiobjective_nas_tutorial.py": { - "extra_files": ["intermediate_source/mnist_train_nas.py"] + "extra_files": ["intermediate_source/mnist_train_nas.py"], + "duration": 2000 }, "beginner_source/dcgan_faces_tutorial.py": { "duration": 2000 @@ -9,7 +10,7 @@ "duration": 1200 }, "beginner_source/hyperparameter_tuning_tutorial.py": { - "duration": 910 + "duration": 0 }, "advanced_source/dynamic_quantization_tutorial.py": { "duration": 380 diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 09b35f64905..4c013c9c511 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -5,57 +5,54 @@ REPO_ROOT = Path(__file__).parent.parent -# files that are ok to have 0 min 0 sec time, probably because they don't have any python code -OK_TO_NOT_RUN = [ - "beginner/basics/intro.html", # no code -] +# For every tutorial on this list, we should determine if it is ok to not run the tutorial (add a comment after +# the file name to explain why, like intro.html), or fix the tutorial and remove it from this list). -# when the tutorial is fixed, remove it from this list -KNOWN_BAD = [ - "beginner/translation_transformer.html", - "beginner/profiler.html", - "beginner/saving_loading_models.html", - "beginner/introyt/captumyt.html", - "beginner/introyt/trainingyt.html", - "beginner/examples_nn/polynomial_module.html", - "beginner/examples_nn/dynamic_net.html", - "beginner/examples_nn/polynomial_optim.html", - "beginner/examples_nn/polynomial_nn.html", - "beginner/examples_tensor/polynomial_numpy.html", - "beginner/examples_tensor/polynomial_tensor.html", - "beginner/former_torchies/autograd_tutorial_old.html", - "beginner/former_torchies/tensor_tutorial_old.html", - "beginner/examples_autograd/polynomial_autograd.html", - "beginner/examples_autograd/polynomial_custom_function.html", - "intermediate/forward_ad_usage.html", - "intermediate/parametrizations.html", - "intermediate/reinforcement_q_learning.html", - "intermediate/text_to_speech_with_torchaudio.html", - "intermediate/mnist_train_nas.html", - "intermediate/fx_conv_bn_fuser.html", - "advanced/super_resolution_with_onnxruntime.html", - "advanced/ddp_pipeline.html", - "prototype/fx_graph_mode_ptq_dynamic.html", - "prototype/vmap_recipe.html", - "prototype/torchscript_freezing.html", - "prototype/nestedtensor.html", - "recipes/recipes/saving_and_loading_models_for_inference.html", - "recipes/recipes/saving_multiple_models_in_one_file.html", - "recipes/recipes/loading_data_recipe.html", - "recipes/recipes/tensorboard_with_pytorch.html", - "recipes/recipes/what_is_state_dict.html", - "recipes/recipes/profiler_recipe.html", - "recipes/recipes/save_load_across_devices.html", - "recipes/recipes/warmstarting_model_using_parameters_from_a_different_model.html", - "recipes/recipes/dynamic_quantization.html", - "recipes/recipes/saving_and_loading_a_general_checkpoint.html", - "recipes/recipes/benchmark.html", - "recipes/recipes/tuning_guide.html", - "recipes/recipes/zeroing_out_gradients.html", - "recipes/recipes/defining_a_neural_network.html", - "recipes/recipes/timer_quick_start.html", - "recipes/recipes/amp_recipe.html", - "recipes/recipes/Captum_Recipe.html", +NOT_RUN = [ + "basics/intro", # no code + "translation_transformer", + "profiler", + "saving_loading_models", + "introyt/captumyt", + "introyt/trainingyt", + "examples_nn/polynomial_module", + "examples_nn/dynamic_net", + "examples_nn/polynomial_optim", + "former_torchies/autograd_tutorial_old", + "former_torchies/tensor_tutorial_old", + "examples_autograd/polynomial_autograd", + "examples_autograd/polynomial_custom_function", + "forward_ad_usage", + "parametrizations", + "reinforcement_q_learning", + "text_to_speech_with_torchaudio", + "mnist_train_nas", # used by ax_multiobjective_nas_tutorial.py + "fx_conv_bn_fuser", + "super_resolution_with_onnxruntime", + "ddp_pipeline", # requires 4 gpus + "fx_graph_mode_ptq_dynamic", + "vmap_recipe", + "torchscript_freezing", + "nestedtensor", + "recipes/saving_and_loading_models_for_inference", + "recipes/saving_multiple_models_in_one_file", + "recipes/loading_data_recipe", + "recipes/tensorboard_with_pytorch", + "recipes/what_is_state_dict", + "recipes/profiler_recipe", + "recipes/save_load_across_devices", + "recipes/warmstarting_model_using_parameters_from_a_different_model", + "recipes/dynamic_quantization", + "recipes/saving_and_loading_a_general_checkpoint", + "recipes/benchmark", + "recipes/tuning_guide", + "recipes/zeroing_out_gradients", + "recipes/defining_a_neural_network", + "recipes/timer_quick_start", + "recipes/amp_recipe", + "recipes/Captum_Recipe", + "hyperparameter_tuning_tutorial", + "flask_rest_api_tutorial", ] @@ -84,7 +81,7 @@ def main() -> None: "Total running time of the script: ( 0 minutes 0.000 seconds)" in elem.text and not any( - html_file_path.match(file) for file in KNOWN_BAD + OK_TO_NOT_RUN + html_file_path.match(file) for file in NOT_RUN ) ): did_not_run.append(html_file_path.as_posix()) diff --git a/conf.py b/conf.py index 830f5a1d3ea..41e3ee3e56c 100644 --- a/conf.py +++ b/conf.py @@ -30,6 +30,7 @@ import os import sys sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('./.jenkins')) import pytorch_sphinx_theme import torch import glob @@ -37,6 +38,7 @@ from custom_directives import IncludeDirective, GalleryItemDirective, CustomGalleryItemDirective, CustomCalloutItemDirective, CustomCardItemDirective import distutils.file_util import re +from validate_tutorials_built import NOT_RUN import plotly.io as pio pio.renderers.default = 'sphinx_gallery' @@ -84,8 +86,8 @@ 'examples_dirs': ['beginner_source', 'intermediate_source', 'advanced_source', 'recipes_source', 'prototype_source'], 'gallery_dirs': ['beginner', 'intermediate', 'advanced', 'recipes', 'prototype'], - 'filename_pattern': 'tutorial.py', - 'ignore_pattern': r'(hyperparameter_tuning_tutorial|flask_rest_api_tutorial).py$', + 'filename_pattern': '.py', + 'ignore_pattern': re.compile(f"({'|'.join(NOT_RUN)}).py$"), 'backreferences_dir': None }