Merge branch 'main' into issue_995

Author: NicolasHug

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,4 +8,4 @@ Fixes #ISSUE_NUMBER
 - [ ] The issue that is being fixed is referred in the description (see above "Fixes #ISSUE_NUMBER")
 - [ ] Only one issue is addressed in this pull request
 - [ ] Labels from the issue that this PR is fixing are added to this pull request
-- [ ] No unnessessary issues are included into this pull request.
+- [ ] No unnecessary issues are included into this pull request.

.github/scripts/docathon-label-sync.py

@@ -14,6 +14,9 @@ def main():
     repo = g.get_repo(f'{repo_owner}/{repo_name}')
     pull_request = repo.get_pull(pull_request_number)
     pull_request_body = pull_request.body
+    # PR without description
+    if pull_request_body is None:
+        return
 
     # get issue number from the PR body
     if not re.search(r'#\d{1,5}', pull_request_body):

beginner_source/basics/optimization_tutorial.py

@@ -149,6 +149,9 @@ def forward(self, x):
 
 def train_loop(dataloader, model, loss_fn, optimizer):
     size = len(dataloader.dataset)
+    # Set the model to training mode - important for batch normalization and dropout layers
+    # Unnecessary in this situation but added for best practices
+    model.train()
     for batch, (X, y) in enumerate(dataloader):
         # Compute prediction and loss
         pred = model(X)
@@ -165,10 +168,15 @@ def train_loop(dataloader, model, loss_fn, optimizer):
 
 
 def test_loop(dataloader, model, loss_fn):
+    # Set the model to evaluation mode - important for batch normalization and dropout layers
+    # Unnecessary in this situation but added for best practices
+    model.eval()
     size = len(dataloader.dataset)
     num_batches = len(dataloader)
     test_loss, correct = 0, 0
 
+    # Evaluating the model with torch.no_grad() ensures that no gradients are computed during test mode
+    # also serves to reduce unnecessary gradient computations and memory usage for tensors with requires_grad=True
     with torch.no_grad():
         for X, y in dataloader:
             pred = model(X)

beginner_source/finetuning_torchvision_models_tutorial.rst

@@ -0,0 +1,10 @@
+Finetuning Torchvision Models
+=============================
+
+This tutorial has been moved to https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html
+
+It will redirect in 3 seconds.
+
+.. raw:: html
+
+   <meta http-equiv="Refresh" content="3; url='https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html'" />

beginner_source/former_torchies/parallelism_tutorial.py

@@ -53,7 +53,10 @@ def forward(self, x):
 
 class MyDataParallel(nn.DataParallel):
     def __getattr__(self, name):
-        return getattr(self.module, name)
+        try:
+            return super().__getattr__(name)
+        except AttributeError:
+            return getattr(self.module, name)
 
 ########################################################################
 # **Primitives on which DataParallel is implemented upon:**

beginner_source/introyt/introyt1_tutorial.py

@@ -288,7 +288,7 @@ def num_flat_features(self, x):
 
 transform = transforms.Compose(
     [transforms.ToTensor(),
-     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
+     transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])
 
 
 ##########################################################################
@@ -297,9 +297,28 @@ def num_flat_features(self, x):
 # -  ``transforms.ToTensor()`` converts images loaded by Pillow into 
 #    PyTorch tensors.
 # -  ``transforms.Normalize()`` adjusts the values of the tensor so
-#    that their average is zero and their standard deviation is 0.5. Most
+#    that their average is zero and their standard deviation is 1.0. Most
 #    activation functions have their strongest gradients around x = 0, so
 #    centering our data there can speed learning.
+#    The values passed to the transform are the means (first tuple) and the
+#    standard deviations (second tuple) of the rgb values of the images in
+#    the dataset. You can calculate these values yourself by running these
+#    few lines of code:
+#          ```
+#           from torch.utils.data import ConcatDataset
+#           transform = transforms.Compose([transforms.ToTensor()])
+#           trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
+#                                        download=True, transform=transform)
+#
+#           #stack all train images together into a tensor of shape 
+#           #(50000, 3, 32, 32)
+#           x = torch.stack([sample[0] for sample in ConcatDataset([trainset])])
+#           
+#           #get the mean of each channel            
+#           mean = torch.mean(x, dim=(0,2,3)) #tensor([0.4914, 0.4822, 0.4465])
+#           std = torch.std(x, dim=(0,2,3)) #tensor([0.2470, 0.2435, 0.2616])  
+# 
+#          ```   
 # 
 # There are many more transforms available, including cropping, centering,
 # rotation, and reflection.

beginner_source/introyt/tensorboardyt_tutorial.py

@@ -64,6 +64,13 @@
 # PyTorch TensorBoard support
 from torch.utils.tensorboard import SummaryWriter
 
+# In case you are using an environment that has TensorFlow installed,
+# such as Google Colab, uncomment the following code to avoid
+# a bug with saving embeddings to your TensorBoard directory
+
+# import tensorflow as tf
+# import tensorboard as tb
+# tf.io.gfile = tb.compat.tensorflow_stub.io.gfile
 
 ######################################################################
 # Showing Images in TensorBoard

beginner_source/nn_tutorial.py

@@ -795,8 +795,7 @@ def __len__(self):
         return len(self.dl)
 
     def __iter__(self):
-        batches = iter(self.dl)
-        for b in batches:
+        for b in self.dl:
             yield (self.func(*b))
 
 train_dl, valid_dl = get_data(train_ds, valid_ds, bs)

beginner_source/transformer_tutorial.py

@@ -149,7 +149,7 @@ def forward(self, x: Tensor) -> Tensor:
 # into ``batch_size`` columns. If the data does not divide evenly into
 # ``batch_size`` columns, then the data is trimmed to fit. For instance, with
 # the alphabet as the data (total length of 26) and ``batch_size=4``, we would
-# divide the alphabet into 4 sequences of length 6:
+# divide the alphabet into sequences of length 6, resulting in 4 of such sequences.
 #
 # .. math::
 #   \begin{bmatrix}

intermediate_source/char_rnn_classification_tutorial.py

@@ -4,11 +4,14 @@
 **************************************************************
 **Author**: `Sean Robertson <https://github.com/spro>`_
 
-We will be building and training a basic character-level RNN to classify
-words. This tutorial, along with the following two, show how to do
-preprocess data for NLP modeling "from scratch", in particular not using
-many of the convenience functions of `torchtext`, so you can see how
-preprocessing for NLP modeling works at a low level.
+We will be building and training a basic character-level Recurrent Neural
+Network (RNN) to classify words. This tutorial, along with two other
+Natural Language Processing (NLP) "from scratch" tutorials
+:doc:`/intermediate/char_rnn_generation_tutorial` and
+:doc:`/intermediate/seq2seq_translation_tutorial`, show how to
+preprocess data to model NLP. In particular these tutorials do not
+use many of the convenience functions of `torchtext`, so you can see how
+preprocessing to model NLP works at a low level.
 
 A character-level RNN reads words as a series of characters -
 outputting a prediction and "hidden state" at each step, feeding its
@@ -32,13 +35,15 @@
     (-2.68) Dutch
 
 
-**Recommended Reading:**
+Recommended Preparation
+=======================
 
-I assume you have at least installed PyTorch, know Python, and
-understand Tensors:
+Before starting this tutorial it is recommended that you have installed PyTorch,
+and have a basic understanding of Python programming language and Tensors:
 
 -  https://pytorch.org/ For installation instructions
 -  :doc:`/beginner/deep_learning_60min_blitz` to get started with PyTorch in general
+   and learn the basics of Tensors
 -  :doc:`/beginner/pytorch_with_examples` for a wide and deep overview
 -  :doc:`/beginner/former_torchies_tutorial` if you are former Lua Torch user
 
@@ -181,10 +186,6 @@ def lineToTensor(line):
 # is just 2 linear layers which operate on an input and hidden state, with
 # a ``LogSoftmax`` layer after the output.
 #
-# .. figure:: https://i.imgur.com/Z2xbySO.png
-#    :alt:
-#
-#
 
 import torch.nn as nn
 
@@ -195,13 +196,13 @@ def __init__(self, input_size, hidden_size, output_size):
         self.hidden_size = hidden_size
 
         self.i2h = nn.Linear(input_size + hidden_size, hidden_size)
-        self.i2o = nn.Linear(input_size + hidden_size, output_size)
+        self.h2o = nn.Linear(hidden_size, output_size)
         self.softmax = nn.LogSoftmax(dim=1)
 
     def forward(self, input, hidden):
         combined = torch.cat((input, hidden), 1)
         hidden = self.i2h(combined)
-        output = self.i2o(combined)
+        output = self.h2o(hidden)
         output = self.softmax(output)
         return output, hidden
 

intermediate_source/mario_rl_tutorial.py

@@ -711,17 +711,18 @@ def record(self, episode, epsilon, step):
                 f"{datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'):>20}\n"
             )
 
-        for metric in ["ep_rewards", "ep_lengths", "ep_avg_losses", "ep_avg_qs"]:
-            plt.plot(getattr(self, f"moving_avg_{metric}"))
-            plt.savefig(getattr(self, f"{metric}_plot"))
+        for metric in ["ep_lengths", "ep_avg_losses", "ep_avg_qs", "ep_rewards"]:
             plt.clf()
+            plt.plot(getattr(self, f"moving_avg_{metric}"), label=f"moving_avg_{metric}")
+            plt.legend()
+            plt.savefig(getattr(self, f"{metric}_plot"))
 
 
 ######################################################################
 # Let’s play!
 # """""""""""""""
 #
-# In this example we run the training loop for 10 episodes, but for Mario to truly learn the ways of
+# In this example we run the training loop for 40 episodes, but for Mario to truly learn the ways of
 # his world, we suggest running the loop for at least 40,000 episodes!
 #
 use_cuda = torch.cuda.is_available()
@@ -735,7 +736,7 @@ def record(self, episode, epsilon, step):
 
 logger = MetricLogger(save_dir)
 
-episodes = 10
+episodes = 40
 for e in range(episodes):
 
     state = env.reset()

intermediate_source/tensorboard_profiler_tutorial.py

@@ -18,7 +18,7 @@
 -----
 To install ``torch`` and ``torchvision`` use the following command:
 
-::
+.. code-block::
 
    pip install torch torchvision
 
@@ -160,23 +160,23 @@ def train(data):
 #
 # Install PyTorch Profiler TensorBoard Plugin.
 #
-# ::
+# .. code-block::
 #
 #     pip install torch_tb_profiler
 #
 
 ######################################################################
 # Launch the TensorBoard.
 #
-# ::
+# .. code-block::
 #
 #     tensorboard --logdir=./log
 #
 
 ######################################################################
 # Open the TensorBoard profile URL in Google Chrome browser or Microsoft Edge browser.
 #
-# ::
+# .. code-block::
 #
 #     http://localhost:6006/#pytorch_profiler
 #
@@ -287,7 +287,7 @@ def train(data):
 # In this example, we follow the "Performance Recommendation" and set ``num_workers`` as below,
 # pass a different name such as ``./log/resnet18_4workers`` to ``tensorboard_trace_handler``, and run it again.
 #
-# ::
+# .. code-block::
 #
 #     train_loader = torch.utils.data.DataLoader(train_set, batch_size=32, shuffle=True, num_workers=4)
 #
@@ -316,7 +316,7 @@ def train(data):
 #
 # You can try it by using existing example on Azure
 #
-# ::
+# .. code-block::
 #
 #     pip install azure-storage-blob
 #     tensorboard --logdir=https://torchtbprofiler.blob.core.windows.net/torchtbprofiler/demo/memory_demo_1_10
@@ -366,7 +366,7 @@ def train(data):
 #
 # You can try it by using existing example on Azure:
 #
-# ::
+# .. code-block::
 #
 #     pip install azure-storage-blob
 #     tensorboard --logdir=https://torchtbprofiler.blob.core.windows.net/torchtbprofiler/demo/distributed_bert

prototype_source/README.txt

@@ -1,8 +1,8 @@
 Prototype Tutorials
 ------------------
 1. distributed_rpc_profiling.rst
-     Profiling PyTorch RPC-Based Workloads
-     https://github.com/pytorch/tutorials/blob/release/1.6/prototype_source/distributed_rpc_profiling.rst
+           Profiling PyTorch RPC-Based Workloads
+           https://github.com/pytorch/tutorials/blob/main/prototype_source/distributed_rpc_profiling.rst
 
 2. graph_mode_static_quantization_tutorial.py
 	   Graph Mode Post Training Static Quantization in PyTorch
@@ -21,8 +21,8 @@ Prototype Tutorials
 	   https://github.com/pytorch/tutorials/blob/main/prototype_source/torchscript_freezing.py
 
 6. vulkan_workflow.rst
-     Vulkan Backend User Workflow
-     https://pytorch.org/tutorials/intermediate/vulkan_workflow.html
+           Vulkan Backend User Workflow
+           https://pytorch.org/tutorials/intermediate/vulkan_workflow.html
 
 7. fx_graph_mode_ptq_static.rst
 	   FX Graph Mode Post Training Static Quantization

prototype_source/fx_graph_mode_ptq_static.rst

@@ -214,9 +214,9 @@ Download the `torchvision resnet18 model <https://download.pytorch.org/models/re
     float_model = load_model(saved_model_dir + float_model_file).to("cpu")
     float_model.eval()
 
-    # deepcopy the model since we need to keep the original model around
-    import copy
-    model_to_quantize = copy.deepcopy(float_model)
+    # create another instance of the model since
+    # we need to keep the original model around
+    model_to_quantize = load_model(saved_model_dir + float_model_file).to("cpu")
 
 3. Set model to eval mode
 -------------------------
@@ -408,4 +408,4 @@ Running the model in AIBench (with single threading) gives the following result:
 
 As we can see for resnet18 both FX graph mode and eager mode quantized model get similar speedup over the floating point model,
 which is around 2-4x faster than the floating point model. But the actual speedup over floating point model may vary
-depending on model, device, build, input batch sizes, threading etc.
+depending on model, device, build, input batch sizes, threading etc.