Merge branch 'main' into arrow-key

Author: Svetlana Karslioglu

.circleci/config.yml

@@ -142,14 +142,14 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
 
 pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults
   environment:
-    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7"
+    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7"
     CUDA_VERSION: "9"
   resource_class: gpu.nvidia.small
   <<: *pytorch_tutorial_build_defaults
 
 pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_defaults
   environment:
-    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7"
+    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7"
   resource_class: medium
   <<: *pytorch_tutorial_build_defaults
 

.circleci/config.yml.in

@@ -142,14 +142,14 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
 
 pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults
   environment:
-    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7"
+    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7"
     CUDA_VERSION: "9"
   resource_class: gpu.nvidia.small
   <<: *pytorch_tutorial_build_defaults
 
 pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_defaults
   environment:
-    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7"
+    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7"
   resource_class: medium
   <<: *pytorch_tutorial_build_defaults
 {% raw %}

.jenkins/get_docker_tag.py

@@ -5,9 +5,14 @@
 }
 
 if __name__ == "__main__":
-    url = "https://api.github.com/repos/pytorch/pytorch/contents/.circleci"
+    url = "https://api.github.com/repos/pytorch/pytorch/contents/.ci"
 
     response = requests.get(url, headers=REQUEST_HEADERS)
-    for file in response.json():
-        if file["name"] == "docker":
-            print(file["sha"])
+    docker_sha = None
+    for finfo in response.json():
+        if finfo["name"] == "docker":
+            docker_sha = finfo["sha"]
+            break
+    if docker_sha is None:
+        raise RuntimeError("Can't find sha sum of docker folder")
+    print(docker_sha)

.jenkins/validate_tutorials_built.py

@@ -49,7 +49,6 @@
     "recipes/timer_quick_start",
     "recipes/amp_recipe",
     "recipes/Captum_Recipe",
-    "hyperparameter_tuning_tutorial",
     "flask_rest_api_tutorial",
     "text_to_speech_with_torchaudio",
 ]

CONTRIBUTING.md

@@ -48,7 +48,7 @@ We welcome your pull requests (PR) for updates and fixes.
 1. If you haven't already, complete the Contributor License Agreement
    ("CLA").
 1. Fork the repo and create a branch from
-   [`master`](https://github.com/pytorch/tutorials).
+   [`main`](https://github.com/pytorch/tutorials).
 1. Test your code.
 1. Lint your code with a tool such as
    [Pylint](https://pylint.pycqa.org/en/latest/).
@@ -88,7 +88,7 @@ commonly-used storage service, such as Amazon S3, and instructing your
 users to download the data at the beginning of your tutorial. 
 
 The
-[Makefile](https://github.com/pytorch/tutorials/blob/master/Makefile)
+[Makefile](https://github.com/pytorch/tutorials/blob/main/Makefile)
 that we use to build the tutorials contains automation that downloads
 required data files.
 
@@ -248,14 +248,14 @@ For Python files, our CI system runs your code during each build.
    [https://github.com/pytorch/tutorials](https://github.com/pytorch/tutorials)
 
 1. Put the tutorial in one of the
-   [`beginner_source`](https://github.com/pytorch/tutorials/tree/master/beginner_source),
-   [`intermediate_source`](https://github.com/pytorch/tutorials/tree/master/intermediate_source),
-   [`advanced_source`](https://github.com/pytorch/tutorials/tree/master/advanced_source)
+   [`beginner_source`](https://github.com/pytorch/tutorials/tree/main/beginner_source),
+   [`intermediate_source`](https://github.com/pytorch/tutorials/tree/main/intermediate_source),
+   [`advanced_source`](https://github.com/pytorch/tutorials/tree/main/advanced_source)
    based on the technical level of the content. For recipes, put the
    recipe in
-   [`recipes_source`](https://github.com/pytorch/tutorials/tree/master/recipes_source).
+   [`recipes_source`](https://github.com/pytorch/tutorials/tree/main/recipes_source).
    In addition, for recipes, add the recipe in the recipes
-   [README.txt](https://github.com/pytorch/tutorials/blob/master/recipes_source/recipes/README.txt)
+   [README.txt](https://github.com/pytorch/tutorials/blob/main/recipes_source/recipes/README.txt)
    file.
 
 
@@ -266,9 +266,9 @@ search, you need to include it in `index.rst`, or for recipes, in
 `recipes_index.rst`. 
 
 1. Open the relevant file
-   [`index.rst`](https://github.com/pytorch/tutorials/blob/master/index.rst)
+   [`index.rst`](https://github.com/pytorch/tutorials/blob/main/index.rst)
    or
-   [`recipes_index.rst`](https://github.com/pytorch/tutorials/blob/master/recipes_source/recipes_index.rst)
+   [`recipes_index.rst`](https://github.com/pytorch/tutorials/blob/main/recipes_source/recipes_index.rst)
 1. Add a _card_ in reStructuredText format similar to the following:
 
 ```
@@ -300,10 +300,10 @@ might fail to build, and the cards will not display properly.
 ### Image ###
 
 Add a thumbnail to the
-[`_static/img/thumbnails/cropped`](https://github.com/pytorch/tutorials/tree/master/_static/img/thumbnails/cropped)
+[`_static/img/thumbnails/cropped`](https://github.com/pytorch/tutorials/tree/main/_static/img/thumbnails/cropped)
 directory. Images that render the best are square--that is, they have
 equal `x` and `y` dimensions--and also have high resolution. [Here is an
-example](https://github.com/pytorch/tutorials/blob/master/_static/img/thumbnails/cropped/loading-data.PNG).
+example](https://github.com/pytorch/tutorials/blob/main/_static/img/thumbnails/cropped/loading-data.PNG).
 
 ## `toctree` ##
 
@@ -344,7 +344,7 @@ test your tutorial when you submit your PR.
 NOTE: Please do not use [ghstack](https://github.com/ezyang/ghstack). We
 do not support ghstack in the [`pytorch/tutorials`](https://github.com/pytorch/tutorials) repo.
 
-Submit the changes as a PR to the master branch of
+Submit the changes as a PR to the main branch of
 [`pytorch/tutorials`](https://github.com/pytorch/tutorials).
 
 1. Add your changes, commit, and push:

README.md

@@ -16,7 +16,7 @@ Here is how you can create a new tutorial (for a detailed description, see [CONT
 1. Create a Python file. If you want it executed while inserted into documentation, save the file with the suffix `tutorial` so that the file name is `your_tutorial.py`.
 2. Put it in one of the `beginner_source`, `intermediate_source`, `advanced_source` directory based on the level of difficulty. If it is a recipe, add it to `recipes_source`. For tutorials demonstrating unstable prototype features, add to the `prototype_source`.
 2. For Tutorials (except if it is a prototype feature), include it in the `toctree` directive and create a `customcarditem` in [index.rst](./index.rst).
-3. For Tutorials (except if it is a prototype feature), create a thumbnail in the [index.rst file](https://github.com/pytorch/tutorials/blob/master/index.rst) using a command like `.. customcarditem:: beginner/your_tutorial.html`. For Recipes, create a thumbnail in the [recipes_index.rst](https://github.com/pytorch/tutorials/blob/master/recipes_source/recipes_index.rst)
+3. For Tutorials (except if it is a prototype feature), create a thumbnail in the [index.rst file](https://github.com/pytorch/tutorials/blob/main/index.rst) using a command like `.. customcarditem:: beginner/your_tutorial.html`. For Recipes, create a thumbnail in the [recipes_index.rst](https://github.com/pytorch/tutorials/blob/main/recipes_source/recipes_index.rst)
 
 If you are starting off with a Jupyter notebook, you can use [this script](https://gist.github.com/chsasank/7218ca16f8d022e02a9c0deb94a310fe) to convert the notebook to Python file. After conversion and addition to the project, please make sure that section headings and other things are in logical order.
 

_templates/layout.html

@@ -48,7 +48,7 @@
       url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/8-inference?WT.mc_id=aiml-7486-cxa";
     }
 
-    $(".pytorch-call-to-action-links").children().first().before("<a href="+url+' data-behavior="call-to-action-event" data-response="Run in Microsoft Learn" target="_blank"><div id="microsoft-learn-link" style="padding-bottom: 0.625rem;border-bottom: 1px solid #f3f4f7;padding-right: 2.5rem;display: -webkit-box;  display: -ms-flexbox; isplay: flex; -webkit-box-align: center;-ms-flex-align: center;align-items: center;"><img class="call-to-action-img" src="../../_static/images/microsoft-logo.svg"/><div class="call-to-action-desktop-view">Run in Microsoft Learn</div><div class="call-to-action-mobile-view">Learn</div></div></a>')
+    $(".pytorch-call-to-action-links").children().first().before("<a href="+url+' data-behavior="call-to-action-event" data-response="Run in Microsoft Learn" target="_blank"><div id="microsoft-learn-link" style="padding-bottom: 0.625rem;border-bottom: 1px solid #f3f4f7;padding-right: 2.5rem;display: -webkit-box;  display: -ms-flexbox; display: flex; -webkit-box-align: center;-ms-flex-align: center;align-items: center;"><img class="call-to-action-img" src="../../_static/images/microsoft-logo.svg"/><div class="call-to-action-desktop-view">Run in Microsoft Learn</div><div class="call-to-action-mobile-view">Learn</div></div></a>')
   }
 
   !function(f,b,e,v,n,t,s)

advanced_source/generic_join.rst

@@ -4,7 +4,7 @@ Distributed Training with Uneven Inputs Using the Join Context Manager
 **Author**\ : `Andrew Gu <https://github.com/andwgu>`_
 
 .. note::
-   |edit| View and edit this tutorial in `github <https://github.com/pytorch/tutorials/blob/master/advanced_source/generic_join.rst>`__.
+   |edit| View and edit this tutorial in `github <https://github.com/pytorch/tutorials/blob/main/advanced_source/generic_join.rst>`__.
 
 .. note:: ``Join`` is introduced in PyTorch 1.10 as a prototype feature. This
     API is subject to change.

advanced_source/rpc_ddp_tutorial.rst

@@ -3,7 +3,7 @@ Combining Distributed DataParallel with Distributed RPC Framework
 **Authors**: `Pritam Damania <https://github.com/pritamdamania87>`_ and `Yi Wang <https://github.com/SciPioneer>`_
 
 .. note::
-   |edit| View and edit this tutorial in `github <https://github.com/pytorch/tutorials/blob/master/advanced_source/rpc_ddp_tutorial.rst>`__.
+   |edit| View and edit this tutorial in `github <https://github.com/pytorch/tutorials/blob/main/advanced_source/rpc_ddp_tutorial.rst>`__.
 
 This tutorial uses a simple example to demonstrate how you can combine
 `DistributedDataParallel <https://pytorch.org/docs/stable/nn.html#torch.nn.parallel.DistributedDataParallel>`__ (DDP)

beginner_source/basics/optimization_tutorial.py

@@ -160,7 +160,7 @@ def train_loop(dataloader, model, loss_fn, optimizer):
         optimizer.step()
 
         if batch % 100 == 0:
-            loss, current = loss.item(), batch * len(X)
+            loss, current = loss.item(), (batch + 1) * len(X)
             print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")
 
 

beginner_source/basics/quickstart_tutorial.py

@@ -87,7 +87,7 @@
 # operations in the neural network, we move it to the GPU if available.
 
 # Get cpu or gpu device for training.
-device = "cuda" if torch.cuda.is_available() else "cpu"
+device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
 print(f"Using {device} device")
 
 # Define model
@@ -151,7 +151,7 @@ def train(dataloader, model, loss_fn, optimizer):
         optimizer.step()
 
         if batch % 100 == 0:
-            loss, current = loss.item(), batch * len(X)
+            loss, current = loss.item(), (batch + 1) * len(X)
             print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")
 
 ##############################################################################

beginner_source/bettertransformer_tutorial.rst

@@ -18,7 +18,7 @@ been updated to use the core library modules to benefit from fastpath accelerati
 
 Better Transformer offers two types of acceleration:
 
-* Native multihead attention implementation for CPU and GPU to improvee overall execution efficiency.  
+* Native multihead attention (MHA) implementation for CPU and GPU to improve overall execution efficiency.  
 * Exploiting sparsity in NLP inference.  Because of variable input lengths, input
   tokens may contain a large number of padding tokens for which processing may be
   skipped, delivering significant speedups.
@@ -124,6 +124,7 @@ Finally, we set the benchmark iteration count:
 2.1  Run and benchmark inference on CPU with and without BT fastpath (native MHA only)
 
 We run the model on CPU, and collect profile information:  
+
 * The first run uses traditional ("slow path") execution.
 * The second run enables BT fastpath execution by putting the model in inference mode using `model.eval()` and disables gradient collection with `torch.no_grad()`.
 
@@ -167,6 +168,7 @@ We disable the BT sparsity:
     
  
 We run the model on DEVICE, and collect profile information for native MHA execution on DEVICE:  
+
 * The first run uses traditional ("slow path") execution.
 * The second run enables BT fastpath execution by putting the model in inference mode using `model.eval()`
   and disables gradient collection with `torch.no_grad()`.

beginner_source/blitz/neural_networks_tutorial.py

@@ -219,7 +219,9 @@ def forward(self, x):
 # The simplest update rule used in practice is the Stochastic Gradient
 # Descent (SGD):
 #
-#      ``weight = weight - learning_rate * gradient``
+# .. code:: python
+#
+#     weight = weight - learning_rate * gradient
 #
 # We can implement this using simple Python code:
 #
@@ -233,18 +235,21 @@ def forward(self, x):
 # update rules such as SGD, Nesterov-SGD, Adam, RMSProp, etc.
 # To enable this, we built a small package: ``torch.optim`` that
 # implements all these methods. Using it is very simple:
-
-import torch.optim as optim
-
-# create your optimizer
-optimizer = optim.SGD(net.parameters(), lr=0.01)
-
-# in your training loop:
-optimizer.zero_grad()   # zero the gradient buffers
-output = net(input)
-loss = criterion(output, target)
-loss.backward()
-optimizer.step()    # Does the update
+#
+# .. code:: python
+#
+#     import torch.optim as optim
+#
+#     # create your optimizer
+#     optimizer = optim.SGD(net.parameters(), lr=0.01)
+#
+#     # in your training loop:
+#     optimizer.zero_grad()   # zero the gradient buffers
+#     output = net(input)
+#     loss = criterion(output, target)
+#     loss.backward()
+#     optimizer.step()    # Does the update
+#
 
 
 ###############################################################

beginner_source/ddp_series_fault_tolerance.rst

@@ -42,8 +42,8 @@ Follow along with the video below or on `youtube <https://www.youtube.com/watch/
 
 In distributed training, a single process failure can
 disrupt the entire training job. Since the susceptibility for failure can be higher here, making your training
-script robust is particularly important here. You might also prefer your training job to be *elastic* i.e. 
-
+script robust is particularly important here. You might also prefer your training job to be *elastic*, for example,
+compute resources can join and leave dynamically over the course of the job.
 
 PyTorch offers a utility called ``torchrun`` that provides fault-tolerance and 
 elastic training. When a failure occurs, ``torchrun`` logs the errors and
@@ -60,7 +60,7 @@ Why use ``torchrun``
 ``torchrun`` handles the minutiae of distributed training so that you
 don't need to. For instance,
 
--  You don't need to set environment variables or explicitly pass the ``rank`` and ``world_size``; torchrun assigns this along with several other `environment variables <https://pytorch.org/docs/stable/elastic/run.html#environment-variables>`__.
+-  You don't need to set environment variables or explicitly pass the ``rank`` and ``world_size``; ``torchrun`` assigns this along with several other `environment variables <https://pytorch.org/docs/stable/elastic/run.html#environment-variables>`__.
 -  No need to call ``mp.spawn`` in your script; you only need a generic ``main()`` entrypoint, and launch the script with ``torchrun``. This way the same script can be run in non-distributed as well as single-node and multinode setups. 
 -  Gracefully restarting training from the last saved training snapshot
 

beginner_source/ddp_series_multigpu.rst

@@ -41,7 +41,7 @@ In this tutorial, we start with a single-GPU training script and migrate that to
 Along the way, we will talk through important concepts in distributed training while implementing them in our code.
 
 .. note:: 
-   If your model contains any ``BatchNorm`` layer, it needs to be converted to ``SyncBatchNorm`` to sync the running stats of ``BatchNorm`` 
+   If your model contains any ``BatchNorm`` layers, it needs to be converted to ``SyncBatchNorm`` to sync the running stats of ``BatchNorm``
    layers across replicas.
 
    Use the helper function 
@@ -57,7 +57,7 @@ Imports
 ~~~~~~~
 -  ``torch.multiprocessing`` is a PyTorch wrapper around Python's native
    multiprocessing
--  The dsitributed process group contains all the processes that can
+-  The distributed process group contains all the processes that can
    communicate and synchronize with each other.
 
 .. code:: diff

beginner_source/ddp_series_theory.rst

@@ -54,8 +54,8 @@ DDP improves upon the architecture in a few ways:
 |                                       | machines                     |
 +---------------------------------------+------------------------------+
 | Slower; uses multithreading on a      | Faster (no GIL contention)   |
-| single process and runs into GIL      | because it uses              |
-| contention                            | multiprocessing              |
+| single process and runs into Global   | because it uses              |
+| Interpreter Lock (GIL) contention     | multiprocessing              |
 +---------------------------------------+------------------------------+
 
 Further Reading

beginner_source/dist_overview.rst

@@ -3,7 +3,7 @@ PyTorch Distributed Overview
 **Author**: `Shen Li <https://mrshenli.github.io/>`_
 
 .. note::
-   |edit| View and edit this tutorial in `github <https://github.com/pytorch/tutorials/blob/master/beginner_source/dist_overview.rst>`__.
+   |edit| View and edit this tutorial in `github <https://github.com/pytorch/tutorials/blob/main/beginner_source/dist_overview.rst>`__.
 
 This is the overview page for the ``torch.distributed`` package. The goal of
 this page is to categorize documents into different topics and briefly

beginner_source/former_torchies/parallelism_tutorial.py

@@ -132,7 +132,7 @@ def forward(self, x):
 # -  `Discuss PyTorch on the Forums`_
 # -  `Chat with other users on Slack`_
 #
-# .. _`Deep Learning with PyTorch: a 60-minute blitz`: https://github.com/pytorch/tutorials/blob/master/Deep%20Learning%20with%20PyTorch.ipynb
+# .. _`Deep Learning with PyTorch: a 60-minute blitz`: https://github.com/pytorch/tutorials/blob/main/Deep%20Learning%20with%20PyTorch.ipynb
 # .. _Train a state-of-the-art ResNet network on imagenet: https://github.com/pytorch/examples/tree/master/imagenet
 # .. _Train a face generator using Generative Adversarial Networks: https://github.com/pytorch/examples/tree/master/dcgan
 # .. _Train a word-level language model using Recurrent LSTM networks: https://github.com/pytorch/examples/tree/master/word_language_model

beginner_source/hyperparameter_tuning_tutorial.py

@@ -424,6 +424,12 @@ def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2):
 
 
 if __name__ == "__main__":
+    # sphinx_gallery_start_ignore
+    # Fixes AttributeError: '_LoggingTee' object has no attribute 'fileno'.
+    # This is only needed to run with sphinx-build.
+    import sys
+    sys.stdout.fileno = lambda: False
+    # sphinx_gallery_end_ignore
     # You can change the number of GPUs per trial here:
     main(num_samples=10, max_num_epochs=10, gpus_per_trial=0)