Fix tutorials not running (#2233)

* fix

* update

Author: clee2000

.jenkins/get_sphinx_filenames.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+from typing import List
+
+from get_files_to_run import get_all_files
+from validate_tutorials_built import NOT_RUN
+
+
+def get_files_for_sphinx() -> List[str]:
+    all_py_files = get_all_files()
+    return [x for x in all_py_files if all(y not in x for y in NOT_RUN)]
+
+
+SPHINX_SHOULD_RUN = "|".join(get_files_for_sphinx())

.jenkins/validate_tutorials_built.py

@@ -9,47 +9,47 @@
 # the file name to explain why, like intro.html), or fix the tutorial and remove it from this list).
 
 NOT_RUN = [
-    "basics/intro",  # no code
-    "translation_transformer",
-    "profiler",
-    "saving_loading_models",
-    "introyt/captumyt",
-    "examples_nn/polynomial_module",
-    "examples_nn/dynamic_net",
-    "examples_nn/polynomial_optim",
-    "former_torchies/autograd_tutorial_old",
-    "former_torchies/tensor_tutorial_old",
-    "examples_autograd/polynomial_autograd",
-    "examples_autograd/polynomial_custom_function",
-    "parametrizations",
-    "mnist_train_nas",  # used by ax_multiobjective_nas_tutorial.py
-    "fx_conv_bn_fuser",
-    "super_resolution_with_onnxruntime",
-    "ddp_pipeline",  # requires 4 gpus
-    "fx_graph_mode_ptq_dynamic",
-    "vmap_recipe",
-    "torchscript_freezing",
-    "nestedtensor",
-    "recipes/saving_and_loading_models_for_inference",
-    "recipes/saving_multiple_models_in_one_file",
-    "recipes/loading_data_recipe",
-    "recipes/tensorboard_with_pytorch",
-    "recipes/what_is_state_dict",
-    "recipes/profiler_recipe",
-    "recipes/save_load_across_devices",
-    "recipes/warmstarting_model_using_parameters_from_a_different_model",
-    "torch_compile_tutorial_",
-    "recipes/dynamic_quantization",
-    "recipes/saving_and_loading_a_general_checkpoint",
-    "recipes/benchmark",
-    "recipes/tuning_guide",
-    "recipes/zeroing_out_gradients",
-    "recipes/defining_a_neural_network",
-    "recipes/timer_quick_start",
-    "recipes/amp_recipe",
-    "recipes/Captum_Recipe",
-    "flask_rest_api_tutorial",
-    "text_to_speech_with_torchaudio",
+    "beginner_source/basics/intro",  # no code
+    "beginner_source/translation_transformer",
+    "beginner_source/profiler",
+    "beginner_source/saving_loading_models",
+    "beginner_source/introyt/captumyt",
+    "beginner_source/examples_nn/polynomial_module",
+    "beginner_source/examples_nn/dynamic_net",
+    "beginner_source/examples_nn/polynomial_optim",
+    "beginner_source/former_torchies/autograd_tutorial_old",
+    "beginner_source/former_torchies/tensor_tutorial_old",
+    "beginner_source/examples_autograd/polynomial_autograd",
+    "beginner_source/examples_autograd/polynomial_custom_function",
+    "intermediate_source/parametrizations",
+    "intermediate_source/mnist_train_nas",  # used by ax_multiobjective_nas_tutorial.py
+    "intermediate_source/fx_conv_bn_fuser",
+    "advanced_source/super_resolution_with_onnxruntime",
+    "advanced_source/ddp_pipeline",  # requires 4 gpus
+    "prototype_source/fx_graph_mode_ptq_dynamic",
+    "prototype_source/vmap_recipe",
+    "prototype_source/torchscript_freezing",
+    "prototype_source/nestedtensor",
+    "recipes_source/recipes/saving_and_loading_models_for_inference",
+    "recipes_source/recipes/saving_multiple_models_in_one_file",
+    "recipes_source/recipes/loading_data_recipe",
+    "recipes_source/recipes/tensorboard_with_pytorch",
+    "recipes_source/recipes/what_is_state_dict",
+    "recipes_source/recipes/profiler_recipe",
+    "recipes_source/recipes/save_load_across_devices",
+    "recipes_source/recipes/warmstarting_model_using_parameters_from_a_different_model",
+    "intermediate_source/torch_compile_tutorial_",
+    "recipes_source/recipes/dynamic_quantization",
+    "recipes_source/recipes/saving_and_loading_a_general_checkpoint",
+    "recipes_source/recipes/benchmark",
+    "recipes_source/recipes/tuning_guide",
+    "recipes_source/recipes/zeroing_out_gradients",
+    "recipes_source/recipes/defining_a_neural_network",
+    "recipes_source/recipes/timer_quick_start",
+    "recipes_source/recipes/amp_recipe",
+    "recipes_source/recipes/Captum_Recipe",
+    "intermediate_source/flask_rest_api_tutorial",
+    "intermediate_source/text_to_speech_with_torchaudio",
 ]
 
 
@@ -67,6 +67,7 @@ def main() -> None:
         glob_path = f"{tutorial_source_dir}/**/*.html"
         html_file_paths += docs_dir.glob(glob_path)
 
+    should_not_run = [f'{x.replace("_source", "")}.html' for x in NOT_RUN]
     did_not_run = []
     for html_file_path in html_file_paths:
         with open(html_file_path, "r", encoding="utf-8") as html_file:
@@ -77,9 +78,7 @@ def main() -> None:
             if (
                 "Total running time of the script: ( 0 minutes  0.000 seconds)"
                 in elem.text
-                and not any(
-                    html_file_path.match(file) for file in NOT_RUN
-                )
+                and not any(html_file_path.match(file) for file in should_not_run)
             ):
                 did_not_run.append(html_file_path.as_posix())
 

conf.py

@@ -38,7 +38,7 @@
 from custom_directives import IncludeDirective, GalleryItemDirective, CustomGalleryItemDirective, CustomCalloutItemDirective, CustomCardItemDirective
 import distutils.file_util
 import re
-from validate_tutorials_built import NOT_RUN
+from get_sphinx_filenames import SPHINX_SHOULD_RUN
 
 import plotly.io as pio
 pio.renderers.default = 'sphinx_gallery'
@@ -87,8 +87,7 @@
     'examples_dirs': ['beginner_source', 'intermediate_source',
                       'advanced_source', 'recipes_source', 'prototype_source'],
     'gallery_dirs': ['beginner', 'intermediate', 'advanced', 'recipes', 'prototype'],
-    'filename_pattern': '.py',
-    'ignore_pattern': re.compile(f"({'|'.join(NOT_RUN)}).py$"),
+    'filename_pattern': re.compile(SPHINX_SHOULD_RUN),
     'promote_jupyter_magic': True,
     'backreferences_dir': None
 }