Merge branch 'main' into main

Author: Svetlana Karslioglu

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,4 +8,4 @@ Fixes #ISSUE_NUMBER
 - [ ] The issue that is being fixed is referred in the description (see above "Fixes #ISSUE_NUMBER")
 - [ ] Only one issue is addressed in this pull request
 - [ ] Labels from the issue that this PR is fixing are added to this pull request
-- [ ] No unnessessary issues are included into this pull request.
+- [ ] No unnecessary issues are included into this pull request.

.github/scripts/docathon-label-sync.py

@@ -14,6 +14,9 @@ def main():
     repo = g.get_repo(f'{repo_owner}/{repo_name}')
     pull_request = repo.get_pull(pull_request_number)
     pull_request_body = pull_request.body
+    # PR without description
+    if pull_request_body is None:
+        return
 
     # get issue number from the PR body
     if not re.search(r'#\d{1,5}', pull_request_body):

advanced_source/neural_style_tutorial.py

@@ -423,6 +423,9 @@ def run_style_transfer(cnn, normalization_mean, normalization_std,
     # We want to optimize the input and not the model parameters so we
     # update all the requires_grad fields accordingly
     input_img.requires_grad_(True)
+    # We also put the model in evaluation mode, so that specific layers 
+    # such as dropout or batch normalization layers behave correctly. 
+    model.eval()
     model.requires_grad_(False)
 
     optimizer = get_input_optimizer(input_img)

advanced_source/super_resolution_with_onnxruntime.py

@@ -16,10 +16,7 @@
 and `ONNX Runtime <https://github.com/microsoft/onnxruntime>`__.
 You can get binary builds of ONNX and ONNX Runtime with
 ``pip install onnx onnxruntime``.
-Note that ONNX Runtime is compatible with Python versions 3.5 to 3.7.
-
-``NOTE``: This tutorial needs PyTorch master branch which can be installed by following
-the instructions `here <https://github.com/pytorch/pytorch#from-source>`__
+ONNX Runtime recommends using the latest stable runtime for PyTorch.
 
 """
 

beginner_source/README.txt

@@ -23,4 +23,4 @@ Beginner Tutorials
 
 6. transformer_translation.py
 	Language Translation with Transformers
-	https://pytorch.org/tutorials/beginner/transformer_tutorial.html
+	https://pytorch.org/tutorials/beginner/translation_transformer.html

beginner_source/deep_learning_60min_blitz.rst

@@ -20,11 +20,12 @@ Goal of this tutorial:
 - Understand PyTorch’s Tensor library and neural networks at a high level.
 - Train a small neural network to classify images
 
-To run the tutorials below, make sure you have the `torch`_ and `torchvision`_
-packages installed.
+To run the tutorials below, make sure you have the `torch`_, `torchvision`_,
+and `matplotlib`_ packages installed.
 
 .. _torch: https://github.com/pytorch/pytorch
 .. _torchvision: https://github.com/pytorch/vision
+.. _matplotlib: https://github.com/matplotlib/matplotlib
 
 .. toctree::
    :hidden:

beginner_source/finetuning_torchvision_models_tutorial.rst

@@ -0,0 +1,10 @@
+Finetuning Torchvision Models
+=============================
+
+This tutorial has been moved to https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html
+
+It will redirect in 3 seconds.
+
+.. raw:: html
+
+   <meta http-equiv="Refresh" content="3; url='https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html'" />

beginner_source/former_torchies/parallelism_tutorial.py

@@ -53,7 +53,10 @@ def forward(self, x):
 
 class MyDataParallel(nn.DataParallel):
     def __getattr__(self, name):
-        return getattr(self.module, name)
+        try:
+            return super().__getattr__(name)
+        except AttributeError:
+            return getattr(self.module, name)
 
 ########################################################################
 # **Primitives on which DataParallel is implemented upon:**

beginner_source/introyt/tensorboardyt_tutorial.py

@@ -64,6 +64,13 @@
 # PyTorch TensorBoard support
 from torch.utils.tensorboard import SummaryWriter
 
+# In case you are using an environment that has TensorFlow installed,
+# such as Google Colab, uncomment the following code to avoid
+# a bug with saving embeddings to your TensorBoard directory
+
+# import tensorflow as tf
+# import tensorboard as tb
+# tf.io.gfile = tb.compat.tensorflow_stub.io.gfile
 
 ######################################################################
 # Showing Images in TensorBoard

beginner_source/introyt/trainingyt.py

@@ -290,15 +290,19 @@ def train_one_epoch(epoch_index, tb_writer):
     model.train(True)
     avg_loss = train_one_epoch(epoch_number, writer)
 
-    # We don't need gradients on to do reporting
-    model.train(False)
-    
+
     running_vloss = 0.0
-    for i, vdata in enumerate(validation_loader):
-        vinputs, vlabels = vdata
-        voutputs = model(vinputs)
-        vloss = loss_fn(voutputs, vlabels)
-        running_vloss += vloss
+    # Set the model to evaluation mode, disabling dropout and using population 
+    # statistics for batch normalization.
+    model.eval()
+
+    # Disable gradient computation and reduce memory consumption.
+    with torch.no_grad():
+        for i, vdata in enumerate(validation_loader):
+            vinputs, vlabels = vdata
+            voutputs = model(vinputs)
+            vloss = loss_fn(voutputs, vlabels)
+            running_vloss += vloss
 
     avg_vloss = running_vloss / (i + 1)
     print('LOSS train {} valid {}'.format(avg_loss, avg_vloss))

beginner_source/nn_tutorial.py

@@ -75,6 +75,11 @@
 import numpy as np
 
 pyplot.imshow(x_train[0].reshape((28, 28)), cmap="gray")
+# ``pyplot.show()`` only if not on Colab
+try:
+    import google.colab
+except ImportError:
+    pyplot.show()
 print(x_train.shape)
 
 ###############################################################################
@@ -790,8 +795,7 @@ def __len__(self):
         return len(self.dl)
 
     def __iter__(self):
-        batches = iter(self.dl)
-        for b in batches:
+        for b in self.dl:
             yield (self.func(*b))
 
 train_dl, valid_dl = get_data(train_ds, valid_ds, bs)

beginner_source/transformer_tutorial.py

@@ -37,7 +37,7 @@
 # ``nn.TransformerEncoder`` consists of multiple layers of
 # `nn.TransformerEncoderLayer <https://pytorch.org/docs/stable/generated/torch.nn.TransformerEncoderLayer.html>`__.
 # Along with the input sequence, a square attention mask is required because the
-# self-attention layers in ``nn.TransformerEncoder`` are only allowed to attend
+# self-attention layers in ``nn.TransformerDecoder`` are only allowed to attend
 # the earlier positions in the sequence. For the language modeling task, any
 # tokens on the future positions should be masked. To produce a probability
 # distribution over output words, the output of the ``nn.TransformerEncoder``
@@ -149,7 +149,7 @@ def forward(self, x: Tensor) -> Tensor:
 # into ``batch_size`` columns. If the data does not divide evenly into
 # ``batch_size`` columns, then the data is trimmed to fit. For instance, with
 # the alphabet as the data (total length of 26) and ``batch_size=4``, we would
-# divide the alphabet into 4 sequences of length 6:
+# divide the alphabet into sequences of length 6, resulting in 4 of such sequences.
 #
 # .. math::
 #   \begin{bmatrix}

intermediate_source/char_rnn_classification_tutorial.py

@@ -2,13 +2,16 @@
 """
 NLP From Scratch: Classifying Names with a Character-Level RNN
 **************************************************************
-**Author**: `Sean Robertson <https://github.com/spro/practical-pytorch>`_
+**Author**: `Sean Robertson <https://github.com/spro>`_
 
-We will be building and training a basic character-level RNN to classify
-words. This tutorial, along with the following two, show how to do
-preprocess data for NLP modeling "from scratch", in particular not using
-many of the convenience functions of `torchtext`, so you can see how
-preprocessing for NLP modeling works at a low level.
+We will be building and training a basic character-level Recurrent Neural
+Network (RNN) to classify words. This tutorial, along with two other
+Natural Language Processing (NLP) "from scratch" tutorials
+:doc:`/intermediate/char_rnn_generation_tutorial` and
+:doc:`/intermediate/seq2seq_translation_tutorial`, show how to
+preprocess data to model NLP. In particular these tutorials do not
+use many of the convenience functions of `torchtext`, so you can see how
+preprocessing to model NLP works at a low level.
 
 A character-level RNN reads words as a series of characters -
 outputting a prediction and "hidden state" at each step, feeding its
@@ -32,13 +35,15 @@
     (-2.68) Dutch
 
 
-**Recommended Reading:**
+Recommended Preparation
+=======================
 
-I assume you have at least installed PyTorch, know Python, and
-understand Tensors:
+Before starting this tutorial it is recommended that you have installed PyTorch,
+and have a basic understanding of Python programming language and Tensors:
 
 -  https://pytorch.org/ For installation instructions
 -  :doc:`/beginner/deep_learning_60min_blitz` to get started with PyTorch in general
+   and learn the basics of Tensors
 -  :doc:`/beginner/pytorch_with_examples` for a wide and deep overview
 -  :doc:`/beginner/former_torchies_tutorial` if you are former Lua Torch user
 
@@ -181,10 +186,6 @@ def lineToTensor(line):
 # is just 2 linear layers which operate on an input and hidden state, with
 # a ``LogSoftmax`` layer after the output.
 #
-# .. figure:: https://i.imgur.com/Z2xbySO.png
-#    :alt:
-#
-#
 
 import torch.nn as nn
 
@@ -195,13 +196,13 @@ def __init__(self, input_size, hidden_size, output_size):
         self.hidden_size = hidden_size
 
         self.i2h = nn.Linear(input_size + hidden_size, hidden_size)
-        self.i2o = nn.Linear(input_size + hidden_size, output_size)
+        self.h2o = nn.Linear(hidden_size, output_size)
         self.softmax = nn.LogSoftmax(dim=1)
 
     def forward(self, input, hidden):
         combined = torch.cat((input, hidden), 1)
         hidden = self.i2h(combined)
-        output = self.i2o(combined)
+        output = self.h2o(hidden)
         output = self.softmax(output)
         return output, hidden
 

intermediate_source/char_rnn_generation_tutorial.py

@@ -2,7 +2,7 @@
 """
 NLP From Scratch: Generating Names with a Character-Level RNN
 *************************************************************
-**Author**: `Sean Robertson <https://github.com/spro/practical-pytorch>`_
+**Author**: `Sean Robertson <https://github.com/spro>`_
 
 This is our second of three tutorials on "NLP From Scratch".
 In the `first tutorial </intermediate/char_rnn_classification_tutorial>`

intermediate_source/dynamic_quantization_bert_tutorial.rst

@@ -68,7 +68,7 @@ built-in F1 score calculation helper function.
 .. code:: shell
 
    pip install sklearn
-   pip install transformers
+   pip install transformers==4.29.2
 
 
 Because we will be using the beta parts of the PyTorch, it is

intermediate_source/mario_rl_tutorial.py

@@ -711,17 +711,18 @@ def record(self, episode, epsilon, step):
                 f"{datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'):>20}\n"
             )
 
-        for metric in ["ep_rewards", "ep_lengths", "ep_avg_losses", "ep_avg_qs"]:
-            plt.plot(getattr(self, f"moving_avg_{metric}"))
-            plt.savefig(getattr(self, f"{metric}_plot"))
+        for metric in ["ep_lengths", "ep_avg_losses", "ep_avg_qs", "ep_rewards"]:
             plt.clf()
+            plt.plot(getattr(self, f"moving_avg_{metric}"), label=f"moving_avg_{metric}")
+            plt.legend()
+            plt.savefig(getattr(self, f"{metric}_plot"))
 
 
 ######################################################################
 # Let’s play!
 # """""""""""""""
 #
-# In this example we run the training loop for 10 episodes, but for Mario to truly learn the ways of
+# In this example we run the training loop for 40 episodes, but for Mario to truly learn the ways of
 # his world, we suggest running the loop for at least 40,000 episodes!
 #
 use_cuda = torch.cuda.is_available()
@@ -735,7 +736,7 @@ def record(self, episode, epsilon, step):
 
 logger = MetricLogger(save_dir)
 
-episodes = 10
+episodes = 40
 for e in range(episodes):
 
     state = env.reset()

intermediate_source/seq2seq_translation_tutorial.py

@@ -2,7 +2,7 @@
 """
 NLP From Scratch: Translation with a Sequence to Sequence Network and Attention
 *******************************************************************************
-**Author**: `Sean Robertson <https://github.com/spro/practical-pytorch>`_
+**Author**: `Sean Robertson <https://github.com/spro>`_
 
 This is the third and final tutorial on doing "NLP From Scratch", where we
 write our own classes and functions to preprocess the data to do our NLP

intermediate_source/tensorboard_profiler_tutorial.py

@@ -18,7 +18,7 @@
 -----
 To install ``torch`` and ``torchvision`` use the following command:
 
-::
+.. code-block::
 
    pip install torch torchvision
 
@@ -160,23 +160,23 @@ def train(data):
 #
 # Install PyTorch Profiler TensorBoard Plugin.
 #
-# ::
+# .. code-block::
 #
 #     pip install torch_tb_profiler
 #
 
 ######################################################################
 # Launch the TensorBoard.
 #
-# ::
+# .. code-block::
 #
 #     tensorboard --logdir=./log
 #
 
 ######################################################################
 # Open the TensorBoard profile URL in Google Chrome browser or Microsoft Edge browser.
 #
-# ::
+# .. code-block::
 #
 #     http://localhost:6006/#pytorch_profiler
 #
@@ -287,7 +287,7 @@ def train(data):
 # In this example, we follow the "Performance Recommendation" and set ``num_workers`` as below,
 # pass a different name such as ``./log/resnet18_4workers`` to ``tensorboard_trace_handler``, and run it again.
 #
-# ::
+# .. code-block::
 #
 #     train_loader = torch.utils.data.DataLoader(train_set, batch_size=32, shuffle=True, num_workers=4)
 #
@@ -316,7 +316,7 @@ def train(data):
 #
 # You can try it by using existing example on Azure
 #
-# ::
+# .. code-block::
 #
 #     pip install azure-storage-blob
 #     tensorboard --logdir=https://torchtbprofiler.blob.core.windows.net/torchtbprofiler/demo/memory_demo_1_10
@@ -366,7 +366,7 @@ def train(data):
 #
 # You can try it by using existing example on Azure:
 #
-# ::
+# .. code-block::
 #
 #     pip install azure-storage-blob
 #     tensorboard --logdir=https://torchtbprofiler.blob.core.windows.net/torchtbprofiler/demo/distributed_bert

intermediate_source/torchvision_tutorial.rst

@@ -145,7 +145,7 @@ Let’s write a ``torch.utils.data.Dataset`` class for this dataset.
            num_objs = len(obj_ids)
            boxes = []
            for i in range(num_objs):
-               pos = np.where(masks[i])
+               pos = np.nonzero(masks[i])
                xmin = np.min(pos[1])
                xmax = np.max(pos[1])
                ymin = np.min(pos[0])

prototype_source/README.txt

@@ -1,8 +1,8 @@
 Prototype Tutorials
 ------------------
 1. distributed_rpc_profiling.rst
-     Profiling PyTorch RPC-Based Workloads
-     https://github.com/pytorch/tutorials/blob/release/1.6/prototype_source/distributed_rpc_profiling.rst
+           Profiling PyTorch RPC-Based Workloads
+           https://github.com/pytorch/tutorials/blob/main/prototype_source/distributed_rpc_profiling.rst
 
 2. graph_mode_static_quantization_tutorial.py
 	   Graph Mode Post Training Static Quantization in PyTorch
@@ -21,8 +21,8 @@ Prototype Tutorials
 	   https://github.com/pytorch/tutorials/blob/main/prototype_source/torchscript_freezing.py
 
 6. vulkan_workflow.rst
-     Vulkan Backend User Workflow
-     https://pytorch.org/tutorials/intermediate/vulkan_workflow.html
+           Vulkan Backend User Workflow
+           https://pytorch.org/tutorials/intermediate/vulkan_workflow.html
 
 7. fx_graph_mode_ptq_static.rst
 	   FX Graph Mode Post Training Static Quantization