diff --git a/.jenkins/get_sphinx_filenames.py b/.jenkins/get_sphinx_filenames.py new file mode 100644 index 00000000000..b84267b48a3 --- /dev/null +++ b/.jenkins/get_sphinx_filenames.py @@ -0,0 +1,13 @@ +from pathlib import Path +from typing import List + +from get_files_to_run import get_all_files +from validate_tutorials_built import NOT_RUN + + +def get_files_for_sphinx() -> List[str]: + all_py_files = get_all_files() + return [x for x in all_py_files if all(y not in x for y in NOT_RUN)] + + +SPHINX_SHOULD_RUN = "|".join(get_files_for_sphinx()) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 7811fa0509f..5b136627c92 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -9,47 +9,47 @@ # the file name to explain why, like intro.html), or fix the tutorial and remove it from this list). NOT_RUN = [ - "basics/intro", # no code - "translation_transformer", - "profiler", - "saving_loading_models", - "introyt/captumyt", - "examples_nn/polynomial_module", - "examples_nn/dynamic_net", - "examples_nn/polynomial_optim", - "former_torchies/autograd_tutorial_old", - "former_torchies/tensor_tutorial_old", - "examples_autograd/polynomial_autograd", - "examples_autograd/polynomial_custom_function", - "parametrizations", - "mnist_train_nas", # used by ax_multiobjective_nas_tutorial.py - "fx_conv_bn_fuser", - "super_resolution_with_onnxruntime", - "ddp_pipeline", # requires 4 gpus - "fx_graph_mode_ptq_dynamic", - "vmap_recipe", - "torchscript_freezing", - "nestedtensor", - "recipes/saving_and_loading_models_for_inference", - "recipes/saving_multiple_models_in_one_file", - "recipes/loading_data_recipe", - "recipes/tensorboard_with_pytorch", - "recipes/what_is_state_dict", - "recipes/profiler_recipe", - "recipes/save_load_across_devices", - "recipes/warmstarting_model_using_parameters_from_a_different_model", - "torch_compile_tutorial_", - "recipes/dynamic_quantization", - "recipes/saving_and_loading_a_general_checkpoint", - "recipes/benchmark", - "recipes/tuning_guide", - "recipes/zeroing_out_gradients", - "recipes/defining_a_neural_network", - "recipes/timer_quick_start", - "recipes/amp_recipe", - "recipes/Captum_Recipe", - "flask_rest_api_tutorial", - "text_to_speech_with_torchaudio", + "beginner_source/basics/intro", # no code + "beginner_source/translation_transformer", + "beginner_source/profiler", + "beginner_source/saving_loading_models", + "beginner_source/introyt/captumyt", + "beginner_source/examples_nn/polynomial_module", + "beginner_source/examples_nn/dynamic_net", + "beginner_source/examples_nn/polynomial_optim", + "beginner_source/former_torchies/autograd_tutorial_old", + "beginner_source/former_torchies/tensor_tutorial_old", + "beginner_source/examples_autograd/polynomial_autograd", + "beginner_source/examples_autograd/polynomial_custom_function", + "intermediate_source/parametrizations", + "intermediate_source/mnist_train_nas", # used by ax_multiobjective_nas_tutorial.py + "intermediate_source/fx_conv_bn_fuser", + "advanced_source/super_resolution_with_onnxruntime", + "advanced_source/ddp_pipeline", # requires 4 gpus + "prototype_source/fx_graph_mode_ptq_dynamic", + "prototype_source/vmap_recipe", + "prototype_source/torchscript_freezing", + "prototype_source/nestedtensor", + "recipes_source/recipes/saving_and_loading_models_for_inference", + "recipes_source/recipes/saving_multiple_models_in_one_file", + "recipes_source/recipes/loading_data_recipe", + "recipes_source/recipes/tensorboard_with_pytorch", + "recipes_source/recipes/what_is_state_dict", + "recipes_source/recipes/profiler_recipe", + "recipes_source/recipes/save_load_across_devices", + "recipes_source/recipes/warmstarting_model_using_parameters_from_a_different_model", + "intermediate_source/torch_compile_tutorial_", + "recipes_source/recipes/dynamic_quantization", + "recipes_source/recipes/saving_and_loading_a_general_checkpoint", + "recipes_source/recipes/benchmark", + "recipes_source/recipes/tuning_guide", + "recipes_source/recipes/zeroing_out_gradients", + "recipes_source/recipes/defining_a_neural_network", + "recipes_source/recipes/timer_quick_start", + "recipes_source/recipes/amp_recipe", + "recipes_source/recipes/Captum_Recipe", + "intermediate_source/flask_rest_api_tutorial", + "intermediate_source/text_to_speech_with_torchaudio", ] @@ -67,6 +67,7 @@ def main() -> None: glob_path = f"{tutorial_source_dir}/**/*.html" html_file_paths += docs_dir.glob(glob_path) + should_not_run = [f'{x.replace("_source", "")}.html' for x in NOT_RUN] did_not_run = [] for html_file_path in html_file_paths: with open(html_file_path, "r", encoding="utf-8") as html_file: @@ -77,9 +78,7 @@ def main() -> None: if ( "Total running time of the script: ( 0 minutes 0.000 seconds)" in elem.text - and not any( - html_file_path.match(file) for file in NOT_RUN - ) + and not any(html_file_path.match(file) for file in should_not_run) ): did_not_run.append(html_file_path.as_posix()) diff --git a/conf.py b/conf.py index bd949b12848..9e3c3068dc4 100644 --- a/conf.py +++ b/conf.py @@ -38,7 +38,7 @@ from custom_directives import IncludeDirective, GalleryItemDirective, CustomGalleryItemDirective, CustomCalloutItemDirective, CustomCardItemDirective import distutils.file_util import re -from validate_tutorials_built import NOT_RUN +from get_sphinx_filenames import SPHINX_SHOULD_RUN import plotly.io as pio pio.renderers.default = 'sphinx_gallery' @@ -87,8 +87,7 @@ 'examples_dirs': ['beginner_source', 'intermediate_source', 'advanced_source', 'recipes_source', 'prototype_source'], 'gallery_dirs': ['beginner', 'intermediate', 'advanced', 'recipes', 'prototype'], - 'filename_pattern': '.py', - 'ignore_pattern': re.compile(f"({'|'.join(NOT_RUN)}).py$"), + 'filename_pattern': re.compile(SPHINX_SHOULD_RUN), 'promote_jupyter_magic': True, 'backreferences_dir': None }