Merge branch 'main' into device-recipe

Author: malfet

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,60 @@
+name: 🐛 Bug Report
+description: Create a tutorial bug report
+title: "[BUG] - <title>"
+labels: [
+  "bug"
+]
+
+body:
+- type: markdown
+  attributes:
+    value: >
+      #### Before submitting a bug, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/pytorch/tutorials/issues?q=is%3Aissue+sort%3Acreated-desc+).
+- type: textarea
+  attributes:
+    label: Add Link
+    description: | 
+      **Add the link to the tutorial***
+    placeholder: |
+      Link to the tutorial on the website:
+  validations:
+    required: true 
+- type: textarea
+  attributes:
+    label: Describe the bug
+    description: | 
+      **Add the bug description**      
+    placeholder: |
+      Provide a detailed description of the issue with code samples if relevant
+      ```python
+
+      # Sample code to reproduce the problem if relevant
+      ```
+
+      **Expected Result:** (Describe what you were expecting to see)
+
+
+      **Actual Result:** (Describe the result)
+
+      ```
+      The error message you got, with the full traceback.
+      ```
+
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Describe your environment
+    description: |
+      **Describe the environment you encountered the bug in:**
+    placeholder: |
+      * Platform (i.e macOS, Linux, Google Colab):
+      * CUDA (yes/no, version?):
+      * PyTorch version (run `python -c "import torch; print(torch.__version__)"`):
+
+  validations:
+    required: true
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,37 @@
+name: 🚀 Feature request
+description: Submit a proposal for a new PyTorch tutorial or improvement of an existing tutorial
+title: "💡 [REQUEST] - <title>"
+labels: [
+  "feature"
+]
+
+body:
+- type: textarea
+  attributes:
+    label: 🚀 Descirbe the improvement or the new tutorial
+    description: |
+      **Describe the improvement**
+    placeholder: |
+      Explain why this improvement or new tutorial is important. For example, *"This tutorial will help users to better understand feature X of PyTorch."* If there is a tutorial that you propose to replace, add here. If this is related to another GitHub issue, add a link here.
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Existing tutorials on this topic
+    description: | 
+      **Add a list of existing tutorials on the same topic.**
+    placeholder: |
+      List tutorials that already explain this functionality if exist. On pytorch.org or elsewhere.
+      * Link
+      * Link
+- type: textarea
+  attributes:
+    label: Additional context
+    description: |
+      **Add additional context**
+    placeholder: |
+      Add any other context or screenshots about the feature request.
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

.jenkins/get_sphinx_filenames.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+from typing import List
+
+from get_files_to_run import get_all_files
+from validate_tutorials_built import NOT_RUN
+
+
+def get_files_for_sphinx() -> List[str]:
+    all_py_files = get_all_files()
+    return [x for x in all_py_files if all(y not in x for y in NOT_RUN)]
+
+
+SPHINX_SHOULD_RUN = "|".join(get_files_for_sphinx())

.jenkins/validate_tutorials_built.py

@@ -9,48 +9,49 @@
 # the file name to explain why, like intro.html), or fix the tutorial and remove it from this list).
 
 NOT_RUN = [
-    "basics/intro",  # no code
-    "translation_transformer",
-    "profiler",
-    "saving_loading_models",
-    "introyt/captumyt",
-    "introyt/trainingyt",
-    "examples_nn/polynomial_module",
-    "examples_nn/dynamic_net",
-    "examples_nn/polynomial_optim",
-    "former_torchies/autograd_tutorial_old",
-    "former_torchies/tensor_tutorial_old",
-    "examples_autograd/polynomial_autograd",
-    "examples_autograd/polynomial_custom_function",
-    "parametrizations",
-    "mnist_train_nas",  # used by ax_multiobjective_nas_tutorial.py
-    "fx_conv_bn_fuser",
-    "super_resolution_with_onnxruntime",
-    "ddp_pipeline",  # requires 4 gpus
-    "fx_graph_mode_ptq_dynamic",
-    "vmap_recipe",
-    "torchscript_freezing",
-    "nestedtensor",
-    "recipes/saving_and_loading_models_for_inference",
-    "recipes/saving_multiple_models_in_one_file",
-    "recipes/loading_data_recipe",
-    "recipes/tensorboard_with_pytorch",
-    "recipes/what_is_state_dict",
-    "recipes/profiler_recipe",
-    "recipes/save_load_across_devices",
-    "recipes/warmstarting_model_using_parameters_from_a_different_model",
-    "torch_compile_tutorial_",
-    "recipes/dynamic_quantization",
-    "recipes/saving_and_loading_a_general_checkpoint",
-    "recipes/benchmark",
-    "recipes/tuning_guide",
-    "recipes/zeroing_out_gradients",
-    "recipes/defining_a_neural_network",
-    "recipes/timer_quick_start",
-    "recipes/amp_recipe",
-    "recipes/Captum_Recipe",
-    "flask_rest_api_tutorial",
-    "text_to_speech_with_torchaudio",
+    "beginner_source/basics/intro",  # no code
+    "beginner_source/translation_transformer",
+    "beginner_source/profiler",
+    "beginner_source/saving_loading_models",
+    "beginner_source/introyt/captumyt",
+    "beginner_source/examples_nn/polynomial_module",
+    "beginner_source/examples_nn/dynamic_net",
+    "beginner_source/examples_nn/polynomial_optim",
+    "beginner_source/former_torchies/autograd_tutorial_old",
+    "beginner_source/former_torchies/tensor_tutorial_old",
+    "beginner_source/examples_autograd/polynomial_autograd",
+    "beginner_source/examples_autograd/polynomial_custom_function",
+    "intermediate_source/parametrizations",
+    "intermediate_source/mnist_train_nas",  # used by ax_multiobjective_nas_tutorial.py
+    "intermediate_source/fx_conv_bn_fuser",
+    "advanced_source/super_resolution_with_onnxruntime",
+    "advanced_source/ddp_pipeline",  # requires 4 gpus
+    "prototype_source/fx_graph_mode_ptq_dynamic",
+    "prototype_source/vmap_recipe",
+    "prototype_source/torchscript_freezing",
+    "prototype_source/nestedtensor",
+    "recipes_source/recipes/saving_and_loading_models_for_inference",
+    "recipes_source/recipes/saving_multiple_models_in_one_file",
+    "recipes_source/recipes/loading_data_recipe",
+    "recipes_source/recipes/tensorboard_with_pytorch",
+    "recipes_source/recipes/what_is_state_dict",
+    "recipes_source/recipes/profiler_recipe",
+    "recipes_source/recipes/save_load_across_devices",
+    "recipes_source/recipes/warmstarting_model_using_parameters_from_a_different_model",
+    "intermediate_source/torch_compile_tutorial_",
+    "recipes_source/recipes/dynamic_quantization",
+    "recipes_source/recipes/saving_and_loading_a_general_checkpoint",
+    "recipes_source/recipes/benchmark",
+    "recipes_source/recipes/tuning_guide",
+    "recipes_source/recipes/zeroing_out_gradients",
+    "recipes_source/recipes/defining_a_neural_network",
+    "recipes_source/recipes/timer_quick_start",
+    "recipes_source/recipes/amp_recipe",
+    "recipes_source/recipes/Captum_Recipe",
+    "intermediate_source/flask_rest_api_tutorial",
+    "intermediate_source/text_to_speech_with_torchaudio",
+    "intermediate_source/ax_multiobjective_nas_tutorial",
+    "intermediate_source/tensorboard_profiler_tutorial" # reenable after 2.0 release.
 ]
 
 
@@ -68,6 +69,7 @@ def main() -> None:
         glob_path = f"{tutorial_source_dir}/**/*.html"
         html_file_paths += docs_dir.glob(glob_path)
 
+    should_not_run = [f'{x.replace("_source", "")}.html' for x in NOT_RUN]
     did_not_run = []
     for html_file_path in html_file_paths:
         with open(html_file_path, "r", encoding="utf-8") as html_file:
@@ -78,9 +80,7 @@ def main() -> None:
             if (
                 "Total running time of the script: ( 0 minutes  0.000 seconds)"
                 in elem.text
-                and not any(
-                    html_file_path.match(file) for file in NOT_RUN
-                )
+                and not any(html_file_path.match(file) for file in should_not_run)
             ):
                 did_not_run.append(html_file_path.as_posix())
 

beginner_source/basics/tensorqs_tutorial.py

@@ -133,7 +133,7 @@
 ######################################################################
 # **Joining tensors** You can use ``torch.cat`` to concatenate a sequence of tensors along a given dimension.
 # See also `torch.stack <https://pytorch.org/docs/stable/generated/torch.stack.html>`__,
-# another tensor joining op that is subtly different from ``torch.cat``.
+# another tensor joining option that is subtly different from ``torch.cat``.
 t1 = torch.cat([tensor, tensor, tensor], dim=1)
 print(t1)
 

beginner_source/introyt/autogradyt_tutorial.py

@@ -334,7 +334,7 @@ def forward(self, x):
 
 print(model.layer2.weight.grad[0][0:10])
 
-optimizer.zero_grad()
+optimizer.zero_grad(set_to_none=False)
 
 print(model.layer2.weight.grad[0][0:10])
 

beginner_source/introyt/trainingyt.py

@@ -81,8 +81,8 @@
 validation_set = torchvision.datasets.FashionMNIST('./data', train=False, transform=transform, download=True)
 
 # Create data loaders for our datasets; shuffle for training, not for validation
-training_loader = torch.utils.data.DataLoader(training_set, batch_size=4, shuffle=True, num_workers=2)
-validation_loader = torch.utils.data.DataLoader(validation_set, batch_size=4, shuffle=False, num_workers=2)
+training_loader = torch.utils.data.DataLoader(training_set, batch_size=4, shuffle=True)
+validation_loader = torch.utils.data.DataLoader(validation_set, batch_size=4, shuffle=False)
 
 # Class labels
 classes = ('T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',

beginner_source/text_sentiment_ngrams_tutorial.py

@@ -148,7 +148,7 @@ class TextClassificationModel(nn.Module):
 
     def __init__(self, vocab_size, embed_dim, num_class):
         super(TextClassificationModel, self).__init__()
-        self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=True)
+        self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=False)
         self.fc = nn.Linear(embed_dim, num_class)
         self.init_weights()
 

conf.py

@@ -38,7 +38,7 @@
 from custom_directives import IncludeDirective, GalleryItemDirective, CustomGalleryItemDirective, CustomCalloutItemDirective, CustomCardItemDirective
 import distutils.file_util
 import re
-from validate_tutorials_built import NOT_RUN
+from get_sphinx_filenames import SPHINX_SHOULD_RUN
 
 import plotly.io as pio
 pio.renderers.default = 'sphinx_gallery'
@@ -87,8 +87,7 @@
     'examples_dirs': ['beginner_source', 'intermediate_source',
                       'advanced_source', 'recipes_source', 'prototype_source'],
     'gallery_dirs': ['beginner', 'intermediate', 'advanced', 'recipes', 'prototype'],
-    'filename_pattern': '.py',
-    'ignore_pattern': re.compile(f"({'|'.join(NOT_RUN)}).py$"),
+    'filename_pattern': re.compile(SPHINX_SHOULD_RUN),
     'promote_jupyter_magic': True,
     'backreferences_dir': None
 }

index.rst

@@ -417,6 +417,34 @@ What's new in PyTorch tutorials?
    :link: intermediate/forward_ad_usage.html
    :tags: Frontend-APIs
 
+.. customcarditem::
+   :header: Jacobians, Hessians, hvp, vhp, and more
+   :card_description: Learn how to compute advanced autodiff quantities using torch.func
+   :image: _static/img/thumbnails/cropped/generic-pytorch-logo.png
+   :link: intermediate/jacobians_hessians.html
+   :tags: Frontend-APIs
+
+.. customcarditem::
+   :header: Model Ensembling
+   :card_description: Learn how to ensemble models using torch.vmap
+   :image: _static/img/thumbnails/cropped/generic-pytorch-logo.png
+   :link: intermediate/ensembling.html
+   :tags: Frontend-APIs
+
+.. customcarditem::
+   :header: Per-Sample-Gradients
+   :card_description: Learn how to compute per-sample-gradients using torch.func
+   :image: _static/img/thumbnails/cropped/generic-pytorch-logo.png
+   :link: intermediate/per_sample_grads.html
+   :tags: Frontend-APIs
+
+.. customcarditem::
+   :header: Neural Tangent Kernels
+   :card_description: Learn how to compute neural tangent kernels using torch.func
+   :image: _static/img/thumbnails/cropped/generic-pytorch-logo.png
+   :link: intermediate/neural_tangent_kernels.html
+   :tags: Frontend-APIs
+
 .. Model Optimization
 
 .. customcarditem::
@@ -502,7 +530,7 @@ What's new in PyTorch tutorials?
    :image: _static/img/thumbnails/cropped/generic-pytorch-logo.png
    :link: intermediate/torchserve_with_ipex_2
    :tags: Model-Optimization,Production
-   
+
 .. customcarditem::
    :header: Introduction to nvFuser
    :card_description: An introduction to nvFuser
@@ -524,6 +552,13 @@ What's new in PyTorch tutorials?
    :link: intermediate/torch_compile_tutorial.html
    :tags: Model-Optimization
 
+.. customcarditem::
+   :header: (beta) Implementing High-Performance Transformers with SCALED DOT PRODUCT ATTENTION
+   :card_description: This tutorial explores the new torch.nn.functional.scaled_dot_product_attention and how it can be used to construct Transformer components.
+   :image: _static/img/thumbnails/cropped/pytorch-logo.png
+   :link: intermediate/scaled_dot_product_attention_tutorial.html
+   :tags: Model-Optimization,Attention,Transformer
+
 .. Parallel-and-Distributed-Training
 
 
@@ -870,6 +905,10 @@ Additional Resources
 
    intermediate/memory_format_tutorial
    intermediate/forward_ad_usage
+   intermediate/jacobians_hessians
+   intermediate/ensembling
+   intermediate/per_sample_grads
+   intermediate/neural_tangent_kernels.py
    advanced/cpp_frontend
    advanced/torch-script-parallelism
    advanced/cpp_autograd
@@ -909,6 +948,7 @@ Additional Resources
    intermediate/nvfuser_intro_tutorial
    intermediate/ax_multiobjective_nas_tutorial
    intermediate/torch_compile_tutorial
+   intermediate/scaled_dot_product_attention_tutorial
 
 .. toctree::
    :maxdepth: 2